chore: release v3.78.2

* fix(scores): update score validation logic to accept inclusive ranges for score config validation

* chore: typo

.cursor/Dockerfile

@@ -12,8 +12,15 @@ RUN apt-get update && \
       postgresql-client \
       redis-tools \
       less nano \
+      sudo \
     && rm -rf /var/lib/apt/lists/*
 
+# ---------- Docker -----------------------------------------------------------
+# Install Docker for background agents that need container capabilities
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
+    sh get-docker.sh && \
+    rm get-docker.sh
+
 # ---------- pnpm -------------------------------------------------------------
 # Langfuse monorepo relies on pnpm 9.5.0 (see CONTRIBUTING.md)
 ENV PNPM_HOME="/pnpm"
@@ -29,11 +36,11 @@ RUN wget -qO- "https://github.com/golang-migrate/migrate/releases/download/v${MI
     chmod +x /usr/local/bin/migrate
 
 # ---------- Non-root user -----------------------------------------------------
-# Use root for convenience in development containers
-WORKDIR /workspace
-
-# Create non-root user
-RUN useradd -ms /bin/bash ubuntu
+# Create non-root user with sudo privileges and docker group access
+RUN useradd -ms /bin/bash ubuntu && \
+    usermod -aG sudo ubuntu && \
+    usermod -aG docker ubuntu && \
+    echo "ubuntu ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # Pre-create pnpm store and set correct ownership to avoid first-run cost & permission issues
 RUN pnpm store path > /dev/null && \

.cursor/environment.json

@@ -4,5 +4,11 @@
     "context": ".",
     "dockerfile": "Dockerfile"
   },
-  "start": "pnpm dx-f"
+  "start": "sudo service docker start",
+  "terminals": [
+    {
+      "name": "dev server",
+      "command": "pnpm run dx-f"
+    }
+  ]
 }

.cursor/rules/frontend-features.mdc

@@ -11,7 +11,7 @@ alwaysApply: true
 ## File Structure
 
 - We generally put all code related to a net-new feature into a folder within web/src/features.
-- Checkout other features to learn about the common structure.
+- Check out other features to learn about the common structure.
 
 ## API for frontend features
 

.cursor/rules/general-info.mdc

@@ -5,4 +5,5 @@ alwaysApply: true
 ---
 # General rules
 
-- Linting in this repo only works if the development server is running
+- Linting in this repo only works if the development server is running
+- Always run the full mono-repo via `pnpm run dx` (use `pnpm dx-f` when you run this in a background agent). Thereby the database will also be seeded.

.devcontainer.json

@@ -0,0 +1,6 @@
+{
+    "name": "langfuse-development",
+    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+    "onCreateCommand": "npm install -g pnpm@9.5.0",
+    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+}

.env.dev.example

@@ -75,7 +75,11 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 # openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+# For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
+# LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10

.gitignore

@@ -4,6 +4,7 @@
 /node_modules
 /.pnp
 .pnp.js
+migrate
 
 # testing
 /coverage

CLAUDE.md

@@ -0,0 +1,188 @@
+# CLAUDE.md
+
+## Project Overview
+
+Langfuse is an open-source LLM engineering platform that helps teams collaboratively develop, monitor, evaluate, and debug AI applications.
+The main feature areas are tracing, evals and prompt management. Langfuse consists of the web application (this repo), documentation, python SDK and javascript/typescript SDK.
+This repo contains the web application, worker, and supporting packages but notably not the JS nor Python client SDKs.
+
+## Repository Structure
+High level structure. There are more folders (eg for hooks etc).
+```
+langfuse/
+├── web/                     # Next.js 14 frontend/backend application
+│   ├── src/
+│   │   ├── components/     # Reusable UI components (shadcn/ui)
+│   │   ├── features/       # Feature-specific code organized by domain
+│   │   ├── pages/          # Next.js pages (Pages Router)
+│   │   └── server/         # tRPC API routes and server logic
+│   └── public/             # Static assets
+├── worker/                  # Express.js background job processor
+│   └── src/
+│       ├── queues/         # BullMQ job queues
+│       └── services/       # Background processing services
+├── packages/
+│   ├── shared/             # Shared types, schemas, and utilities
+│   │   ├── prisma/         # Database schema and migrations
+│   │   └── src/            # Shared TypeScript code
+│   ├── config-eslint/      # ESLint configuration
+│   └── config-typescript/  # TypeScript configuration
+├── ee/                     # Enterprise Edition features
+├── fern/                   # API documentation and OpenAPI specs
+├── generated/              # Auto-generated client code
+└── scripts/                # Development and deployment scripts
+```
+
+## Repository Architecture
+This is a **pnpm + Turbo monorepo** with the following key packages:
+
+### Core Applications
+- **`/web/`** - Next.js 14 application (Pages Router) providing both frontend UI and backend APIs
+- **`/worker/`** - Express.js background job processing server
+- **`/packages/shared/`** - Shared database schema, types, and utilities
+
+### Supporting Packages
+- **`/ee/`** - Enterprise Edition features (separate licensing)
+- **`/packages/config-eslint/`** - Shared ESLint configuration
+- **`/packages/config-typescript/`** - Shared TypeScript configuration
+
+## Development Commands
+
+### Development
+```sh
+pnpm i               # Install dependencies
+pnpm run dev         # Start all services (web + worker)
+pnpm run dev:web     # Web app only (localhost:3000) - **used in most cases!**
+pnpm run dev:worker  # Worker only
+pnpm run dx          # Full initial setup: install deps, reset DBs, resets node modules, seed data, start dev. USE SPARINGLY AS IT WIPES THE DATABASE & node_modules
+```
+
+### Database Management
+database commands are to be run in the `packages/shared/` folder.
+```sh
+pnpm run db:generate       # Build prisma models
+pnpm run db:migrate        # Run Prisma migrations
+pnpm run db:reset          # Reset and reseed databases
+pnpm run db:seed           # Seed with example data
+```
+
+### Infrastructure
+```sh
+pnpm run infra:dev:up      # Start Docker services (PostgreSQL, ClickHouse, Redis, MinIO)
+pnpm run infra:dev:down    # Stop Docker services
+```
+
+### Building
+```sh
+pnpm --filter=PACKAGE_NAME run build  # Runs the build command, will show real typescript errors etc.
+```
+
+### Testing in Web Package
+The web package uses JEST for unit tests.
+Depending on the file location (sync, async)
+`web` related tests must go into the `web/src/__tests__/` folder.
+```sh
+pnpm test-sync --testPathPattern="$FILE_LOCATION_PATTERN" --testNamePattern="$TEST_NAME_PATTERN"
+pnpm test-async --testPathPattern="$FILE_LOCATION_PATTERN" --testNamePattern="$TEST_NAME_PATTERN"
+```
+
+### Testing in the Worker Package
+The worker uses `vitest` for unit tests.
+```sh
+pnpm run test --filter=worker -- $TEST_FILE_NAME -t "$TEST_NAME"
+```
+
+### Utilities
+```bash
+pnpm run nuke              # Remove all node_modules, build files, wipe database, docker containers. **USE WITH CAUTION**
+```
+
+## Technology Stack
+
+### Web Application (`/web/`)
+- **Framework**: Next.js 14 (Pages Router)
+- **APIs**: tRPC (type-safe client-server communication) + REST APIs for public access
+- **Authentication**: NextAuth.js/Auth.js
+- **Database**: Prisma ORM with PostgreSQL
+- **Analytics Database**: ClickHouse (high-volume trace data)
+- **Validation**: Zod schemas, we use zodv4 (always import from `zod/v4`)
+- **Styling**: Tailwind CSS with CSS variables for theming
+- **Components**: shadcn/ui (Radix UI primitives)
+- **State Management**: TanStack Query (React Query) + tRPC
+- **Charts**: Tremor, Recharts
+
+### Worker Application (`/worker/`)
+- **Framework**: Express.js
+- **Queue System**: BullMQ with Redis
+- **Purpose**: Async processing (data ingestion, evaluations, exports, integrations)
+
+### Infrastructure
+- **Primary Database**: PostgreSQL (via Prisma ORM)
+- **Analytics Database**: ClickHouse
+- **Cache/Queues**: Redis
+- **Blob Storage**: MinIO/S3
+
+## Development Guidelines
+
+### Frontend Features
+- All new features go in `/web/src/features/[feature-name]/`
+- Use tRPC for full-stack features (entry point: `web/src/server/api/root.ts`)
+- Follow existing feature structure for consistency
+- Use shadcn/ui components from `@/src/components/ui`
+- Custom reusable components go in `@/src/components`
+
+### Public API Development
+- All public API routes in `/web/src/pages/api/public`
+- Use `withMiddlewares.ts` wrapper
+- Define types in `/web/src/features/public-api/types` with strict Zod v4 objects
+- Add end-to-end tests (see `datasets-api.servertest.ts`)
+- Manually update Fern API specs in `/fern/`, then regenerate OpenAPI spec via Fern CLI
+
+### Authorization & RBAC
+- Check `/web/src/features/rbac/README.md` for authorization patterns
+- Implement proper entitlements checking (see `/web/src/features/entitlements/README.md`)
+
+### Database
+- **Dual database system**: PostgreSQL (primary) + ClickHouse (analytics)
+- Use `golang-migrate` CLI for database migrations
+- All database operations go through Prisma ORM for PostgreSQL
+- Foreign key relationships may not be enforced in schema to allow unordered ingestion
+
+### Testing
+- Jest for API tests, Playwright for E2E tests
+- For backend/API changes, tests must pass before pushes
+- Add tests for new API endpoints and features
+
+### Code Conventions
+- **Pages Router** (not App Router)
+- Follow conventional commits on main branch
+- Use CSS variables for theming (supports auto dark/light mode)
+- TypeScript throughout
+- Zod v4 for all input validation
+
+## Environment Setup
+
+- **Node.js**: Version 20 (specified in `.nvmrc`)
+- **Package Manager**: pnpm v9.5.0
+- **Database Dependencies**: Docker for local PostgreSQL, ClickHouse, Redis, MinIO
+- **Environment**: Copy `.env.dev.example` to `.env`
+
+## Login for Development
+
+When running locally with seed data:
+- Username: `demo@langfuse.com`
+- Password: `password`
+- Demo project URL: `http://localhost:3000/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a`
+
+## Linear MCP
+To get a project, use the `get_project` capability with the full project name as it is in the title.
+- bad: message-placeholder-in-chat-messages-2beb6f02ec48
+- good: Message placeholder in chat messages
+
+## Front-end Tips
+
+### Window Location Handling
+- Whenever you want to use or do use window.location..., ensure that you also add proper handling for a custom basePath
+
+## TypeScript Best Practices
+- In TypeScript, if possible, don't use the `any` type

CONTRIBUTING.md

@@ -42,7 +42,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
   - NextAuth.js / Auth.js
   - tRPC: Frontend APIs
   - Prisma ORM
-  - Zod
+  - Zod v4
   - Tailwind CSS
   - shadcn/ui tailwind components (using Radix and tanstack)
   - Fern: generate OpenAPI spec and Pydantic models
@@ -55,34 +55,34 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 See this [diagram](https://langfuse.com/self-hosting#architecture) for an overview of the architecture.
 
-### Network Overview
-
-```mermaid
-flowchart TB
-    User["UI, API, SDKs"]
-    subgraph vpc["VPC"]
-        Web["Web Server<br/>(langfuse/langfuse)"]
-        Worker["Async Worker<br/>(langfuse/worker)"]
-        Postgres["Postgres - OLTP<br/>(Transactional Data)"]
-        Cache["Redis/Valkey<br/>(Cache, Queue)"]
-        Clickhouse["Clickhouse - OLAP<br/>(Observability Data)"]
-        S3["S3 / Blob Storage<br/>(Raw events, multi-modal attachments)"]
-    end
-    LLM["LLM API/Gateway<br/>(optional)"]
-
-    User --> Web
-    Web --> S3
-    Web --> Postgres
-    Web --> Cache
-    Web --> Clickhouse
-    Web -.->|"optional for playground"| LLM
-
-    Cache --> Worker
-    Worker --> Clickhouse
-    Worker --> Postgres
-    Worker --> S3
-    Worker -.->|"optional for evals"| LLM
-```
+### Network Overview
+
+```mermaid
+flowchart TB
+    User["UI, API, SDKs"]
+    subgraph vpc["VPC"]
+        Web["Web Server<br/>(langfuse/langfuse)"]
+        Worker["Async Worker<br/>(langfuse/worker)"]
+        Postgres["Postgres - OLTP<br/>(Transactional Data)"]
+        Cache["Redis/Valkey<br/>(Cache, Queue)"]
+        Clickhouse["Clickhouse - OLAP<br/>(Observability Data)"]
+        S3["S3 / Blob Storage<br/>(Raw events, multi-modal attachments)"]
+    end
+    LLM["LLM API/Gateway<br/>(optional)"]
+
+    User --> Web
+    Web --> S3
+    Web --> Postgres
+    Web --> Cache
+    Web --> Clickhouse
+    Web -.->|"optional for playground"| LLM
+
+    Cache --> Worker
+    Worker --> Clickhouse
+    Worker --> Postgres
+    Worker --> S3
+    Worker -.->|"optional for evals"| LLM
+```
 
 ### Database Overview
 
@@ -130,10 +130,11 @@ Requirements
     cp .env.dev.example .env
    ```
 
-4. Run the entire infrastructure in dev mode
+4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it.
 
    ```bash
-   pnpm run dx
+   pnpm run dx # first run only (resets db, node_modules, ...)
+   pnpm run dev # any subsequent runs
    ```
 
    You will be asked whether you want to reset Postgres and ClickHouse. Confirm both with 'Y' and press enter.
@@ -148,6 +149,12 @@ Requirements
    - Username: `demo@langfuse.com`
    - Password: `password`
 
+
+To get comprehensive example data, you can use the `seed` command:
+```sh
+pnpm run db:seed:examples
+```
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -197,23 +204,41 @@ Requirements
 
 On the main branch, we adhere to the best practices of [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/). All pull requests and branches are squash-merged to maintain a clean and readable history. This approach ensures the addition of a conventional commit message when merging contributions.
 
-## Test the public API
+## Running Unit Tests
 
-The API is tested using Jest. With the development server running, you can run the tests with:
+All tests run in the CI and must pass before merging.
+All tests run against a running langfuse instance and **write/delete real data from the database**.
 
-Run all
+### Tests in the `web` package (public API)
+We're using Jest with in the `web` package. Therefore, if you want to provide an argument to the test runner, do it directly without an intermittent ` -- `.
 
-```bash
-npm run test
+There are three types of unit tests:
+- `test-sync`
+- `test-async`
+- `test-client`
+
+To run a specific test, for example the test: `"should handle special characters in prompt names"` in `prompts.v2.servertest.ts`, run:
+```sh
+cd web  # or with --filter=web
+pnpm test-sync --testPathPattern="prompts\.v2\.servertest" --testNamePattern="should handle special characters in prompt names"
 ```
 
-Run interactively in watch mode
-
-```bash
-npm run test:watch
+To run all tests:
+```sh
+pnpm run test
 ```
 
-These tests are also run in CI.
+Run interactively in watch mode (not recommended!)
+```sh
+pnpm run test:watch
+```
+
+### Tests in the `worker` package
+For the `worker` package, we're using `vitest` to run unit tests.
+
+```sh
+pnpm run test --filter=worker -- FILE_YOU_WANT_TO_TEST.ts -t "test name"
+```
 
 ## CI/CD
 
@@ -341,6 +366,20 @@ Please note that
 
 Until the V3 release, both the JSON record must be updated **and** a migration must be created to continue supporting self-hosted users. Note that the migration must updated both the `models` as well as the `prices` table accordingly.
 
+## Updating the OpenAPI Specs & fern SDKs
+
+We maintain the API specifications manually to guarantee a high degree of understandability. If you made changes to the API, please update the respective `.yml` files in `fern/apis/...`.
+
+To generate the respective `openapi.yml` files which power the online API reference & SDKs, run:
+
+```sh
+npx fern-api generate --api server  # for the server API
+npx fern-api generate --api client  # for the client API
+npx fern-api generate --api organizations  # for the organizations API
+```
+
+**Note:** You need a signed in fern account to run those commands.
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

README.cn.md

@@ -26,7 +26,7 @@
     <a href="https://langfuse.com/roadmap"><strong>路线图</strong></a> ·
   </div>
   <br/>
-  <span>Langfuse 使用 <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a> 作为支持和功能请求的平台。</span>
+  <span>Langfuse 使用 <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a> 作为支持和功能请求的平台。</span>
   <br/>
   <span><b>我们正在招聘。</b> <a href="https://langfuse.com/careers"><strong>加入我们</strong></a>，从事产品工程和技术市场职位。</span>
   <br/>

README.ja.md

@@ -26,7 +26,7 @@
       <a href="https://langfuse.com/roadmap"><strong>ロードマップ</strong></a> ·
    </div>
    <br/>
-   <span>Langfuseは、サポートと機能リクエストのために <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a> を利用しています。</span>
+   <span>Langfuseは、サポートと機能リクエストのために <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a> を利用しています。</span>
    <br/>
    <span><b>We're hiring.</b> <a href="https://langfuse.com/careers"><strong>チームに加わる</strong></a> （製品エンジニアリングおよびテクニカルGTMのポジション）への応募をお待ちしています。</span>
    <br/>

README.md

@@ -3,6 +3,9 @@
 <div align="center">
    <div>
       <h3>
+        <a href="https://langfuse.com/blog/2025-06-04-open-sourcing-langfuse-product">
+            <strong>Langfuse Is Doubling Down On Open Source</strong>
+         </a> <br> <br>
          <a href="https://cloud.langfuse.com">
             <strong>Langfuse Cloud</strong>
          </a> · 
@@ -23,7 +26,7 @@
       <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
    </div>
    <br/>
-   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
    <span><b>We're hiring.</b> <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in product engineering and technical go-to-market roles.</span>
    <br/>

fern/apis/server/definition/prompts.yml

@@ -82,7 +82,7 @@ types:
   CreateChatPromptRequest:
     properties:
       name: string
-      prompt: list<ChatMessage>
+      prompt: list<ChatMessageWithPlaceholders>
       config: optional<unknown>
       labels:
         type: optional<list<string>>
@@ -132,6 +132,11 @@ types:
         type: optional<map<string, unknown>>
         docs: The dependency resolution graph for the current prompt. Null if prompt has no dependencies.
 
+  ChatMessageWithPlaceholders:
+    union:
+      chatmessage: ChatMessage
+      placeholder: PlaceholderMessage
+
   ChatMessage:
     properties:
       role:
@@ -139,6 +144,11 @@ types:
       content:
         type: string
 
+  PlaceholderMessage:
+    properties:
+      name:
+        type: string
+
   TextPrompt:
     extends: BasePrompt
     properties:
@@ -147,4 +157,4 @@ types:
   ChatPrompt:
     extends: BasePrompt
     properties:
-      prompt: list<ChatMessage>
+      prompt: list<ChatMessageWithPlaceholders>

fern/apis/server/definition/trace.yml

@@ -63,6 +63,9 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Optional filter for traces where the environment is one of the provided values.
+          fields:
+            type: optional<string>
+            docs: "Comma-separated list of fields to include in the response. Available field groups are 'core' (always included), 'io' (input, output, metadata), 'scores', 'observations', 'metrics'. If not provided, all fields are included. Example: 'core,scores,metrics'"
       response: Traces
     deleteMultiple:
       docs: Delete multiple traces

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.72.1",
+  "version": "3.78.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -25,6 +25,7 @@
     "dev": "turbo run dev",
     "dev:worker": "turbo run dev --filter=worker",
     "dev:web": "turbo run dev --filter=web",
+    "dev:web-turbo": "turbo run dev --filter=web -- --turbo",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "release": "dotenv -e ../.env -- release-it",
@@ -37,7 +38,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.3.3",
     "release-it": "^19.0.3",
-    "turbo": "^1.13.4"
+    "turbo": "^2.5.4"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.4",
+    "eslint-config-turbo": "^2.5.4",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/scripts/seed.ts

@@ -1,70 +0,0 @@
-import { prisma } from "../../src/db";
-import { ObservationRecordReadType, redis } from "../../src/server";
-import { prepareClickhouse } from "../../scripts/prepareClickhouse";
-import { createDatasets } from "../../prisma/seed";
-import { queryClickhouse } from "../../src/server/repositories/clickhouse";
-import { convertObservation } from "../../src/server/repositories/observations_converters";
-
-async function main() {
-  try {
-    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
-    if (
-      await prisma.project.findFirst({
-        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
-      })
-    ) {
-      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
-    }
-    await prepareClickhouse(projectIds, {
-      numberOfDays: 3,
-      totalObservations: 10000,
-    });
-
-    const project1 = await prisma.project.findFirst({
-      where: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-    });
-
-    const project2 =
-      projectIds.length > 1
-        ? await prisma.project.findFirst({
-            where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
-          })
-        : await prisma.project.findFirst();
-
-    const query = `
-        SELECT *
-        FROM observations o
-        WHERE o.project_id IN ({projectIds: Array(String)})
-        LIMIT 2000;
-      `;
-
-    const res = await queryClickhouse<ObservationRecordReadType>({
-      query,
-      params: {
-        projectIds,
-      },
-    });
-
-    await createDatasets(
-      project1!,
-      project2!,
-      (await Promise.all(res.map(convertObservation))).map((o) => ({
-        ...o,
-        metadata: {},
-        modelParameters: {},
-        input: {},
-        output: {},
-      })),
-    );
-
-    console.log("Clickhouse preparation completed successfully.");
-  } catch (error) {
-    console.error("Error during Clickhouse preparation:", error);
-  } finally {
-    await prisma.$disconnect();
-    redis?.disconnect();
-    console.log("Disconnected from Clickhouse.");
-  }
-}
-
-main();

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 70",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -47,11 +47,11 @@
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
     "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed",
-    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' clickhouse/scripts/seed.ts",
-    "load:setup": "dotenv -e ../../.env -- tsx scripts/load-seed.ts"
+    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' scripts/seeder/seed-clickhouse.ts",
+    "load:setup": "dotenv -e ../../.env -- tsx scripts/seeder/load-seed-clickhouse.ts"
   },
   "prisma": {
-    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
+    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} scripts/seeder/seed-postgres.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
@@ -69,8 +69,8 @@
     "@langchain/google-vertexai": "^0.2.12",
     "@langchain/openai": "^0.5.13",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^6.3.0",
-    "@react-email/components": "^0.0.42",
+    "@prisma/client": "^6.10.1",
+    "@react-email/components": "^0.1.0",
     "@react-email/render": "^1.1.2",
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.8.2",
@@ -81,11 +81,12 @@
     "exponential-backoff": "^3.1.1",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
+    "jsonpath-plus": "10.3.0",
     "kysely": "^0.27.4",
     "langchain": "^0.3.28",
-    "langfuse-langchain": "3.37.4",
+    "langfuse-langchain": "3.38.1",
     "lodash": "^4.17.21",
-    "lossless-json": "^4.0.2",
+    "lossless-json": "^4.1.1",
     "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
@@ -111,7 +112,7 @@
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
     "prettier": "^3.3.3",
-    "prisma": "^6.3.0",
+    "prisma": "^6.10.1",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",

packages/shared/prisma/generated/types.ts

@@ -111,7 +111,8 @@ export const DashboardWidgetChartType = {
     VERTICAL_BAR: "VERTICAL_BAR",
     PIE: "PIE",
     NUMBER: "NUMBER",
-    HISTOGRAM: "HISTOGRAM"
+    HISTOGRAM: "HISTOGRAM",
+    PIVOT_TABLE: "PIVOT_TABLE"
 } as const;
 export type DashboardWidgetChartType = (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export type Account = {

packages/shared/prisma/migrations/20250625_add_pivot_table_charttype/migration.sql

@@ -0,0 +1,9 @@
+-- Database migration to add PIVOT_TABLE chart type to DashboardWidgetChartType enum
+-- This enables the creation of pivot table widgets in the dashboard system
+-- 
+-- This migration adds support for tabular data visualization with configurable
+-- row dimensions and metrics, extending the existing widget types (line charts,
+-- bar charts, pie charts, etc.) to include pivot table functionality.
+
+-- AlterEnum
+ALTER TYPE "DashboardWidgetChartType" ADD VALUE 'PIVOT_TABLE'; 

packages/shared/prisma/schema.prisma

@@ -871,7 +871,7 @@ model JobExecution {
   jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
 
   jobTemplateId String?       @map("job_template_id")
-  jobTemplate   EvalTemplate? @relation(fields: [jobTemplateId], references: [id], onDelete: SetNull)
+  jobTemplate   EvalTemplate? @relation(fields: [jobTemplateId], references: [id], onDelete: SetNull, onUpdate: NoAction)
 
   status    JobExecutionStatus
   startTime DateTime?          @map("start_time")
@@ -902,7 +902,7 @@ model DefaultLlmModel {
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
 
   projectId String  @map("project_id")
-  Project   Project @relation(fields: [projectId], references: [id])
+  Project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
   llmApiKeyId String     @map("llm_api_key_id")
   LlmApiKey   LlmApiKeys @relation("LlmApiKeyId", fields: [llmApiKeyId], references: [id], onDelete: Cascade)
@@ -1157,6 +1157,7 @@ enum DashboardWidgetChartType {
   PIE
   NUMBER
   HISTOGRAM
+  PIVOT_TABLE
 }
 
 model DashboardWidget {

packages/shared/scripts/prepareClickhouse.ts

@@ -1,314 +0,0 @@
-import { SEED_PROMPTS } from "../prisma/seed";
-import { prisma } from "../src/db";
-import { clickhouseClient, logger } from "../src/server";
-
-function randn_bm(min: number, max: number, skew: number) {
-  let u = 0,
-    v = 0;
-  while (u === 0) u = Math.random(); //Converting [0,1) to (0,1)
-  while (v === 0) v = Math.random();
-  let num = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
-
-  num = num / 10.0 + 0.5; // Translate to 0 -> 1
-  if (num > 1 || num < 0)
-    num = randn_bm(min, max, skew); // resample between 0 and 1 if out of range
-  else {
-    num = Math.pow(num, skew); // Skew
-    num *= max - min; // Stretch to fill range
-    num += min; // offset to min
-  }
-  return num;
-}
-
-export const prepareClickhouse = async (
-  projectIds: string[],
-  opts: {
-    numberOfDays: number;
-    totalObservations: number;
-  },
-) => {
-  logger.info(
-    `Preparing Clickhouse for ${projectIds.length} projects and ${opts.numberOfDays} days.`,
-  );
-
-  const projectData = projectIds.map((projectId) => {
-    const observationsPerProject = Math.ceil(
-      randn_bm(0, opts.totalObservations, 2),
-    ); // Skew the number of observations
-
-    const tracesPerProject = Math.floor(observationsPerProject / 6); // On average, one trace should have 6 observations
-    const scoresPerProject = tracesPerProject * 10; // On average, one trace should have 10 scores
-    return {
-      projectId,
-      observationsPerProject,
-      tracesPerProject,
-      scoresPerProject,
-    };
-  });
-
-  for (const data of projectData) {
-    const {
-      projectId,
-      tracesPerProject,
-      observationsPerProject,
-      scoresPerProject,
-    } = data;
-    logger.info(
-      `Preparing Clickhouse for ${projectId}: Traces: ${tracesPerProject}, Scores: ${scoresPerProject}, Observations: ${observationsPerProject}`,
-    );
-
-    const tracesQuery = `
-    INSERT INTO traces
-    SELECT toString(number) AS id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-      concat('name_', toString(rand() % 100)) AS name,
-      concat('user_id_', toInt64(randExponential(1 / 100))) AS user_id,
-      map('prototype', 'test') AS metadata,
-      concat('release_', toString(randUniform(0, 100))) AS release,
-      concat('version_', toString(randUniform(0, 100))) AS version,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      if(rand() < 0.8, true, false) as public,
-      if(rand() < 0.8, true, false) as bookmarked,
-      array('tag1', 'tag2') as tags,
-      repeat('input', toInt64(randExponential(1 / 100))) AS input,
-      repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      if(randUniform(0, 1) < 0.2, NULL, concat('session_', toString(rand() % 1000))) AS session_id,
-      timestamp AS created_at,
-      timestamp AS updated_at,
-      timestamp AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${tracesPerProject});
-  `;
-
-    const observationsQuery = `
-    INSERT INTO observations
-    SELECT toString(number) AS id,
-      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
-      toString(rand()) AS parent_observation_id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
-      addSeconds(start_time, if(rand() < 0.6, floor(randUniform(0, 20)), floor(randUniform(0, 3600)))) AS end_time,
-      concat('name', toString(rand() % 100)) AS name,
-      map('prototype', 'test') AS metadata,
-      if(randUniform(0, 1) < 0.9, 'DEFAULT', if(randUniform(0, 1) < 0.5, 'ERROR', if(randUniform(0, 1) < 0.5, 'DEBUG', 'WARNING'))) AS level,
-      'status_message' AS status_message,
-      'version' AS version,
-      repeat('input', toInt64(randExponential(1 / 100))) AS input,
-      repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      case
-        when number % 2 = 0 then 'claude-3-haiku-20230407'
-        else 'gpt-4'
-      end as provided_model_name,
-      case
-        when number % 2 = 0 then 'cltra4wbs0000k1407g0ya3'
-        else '1cmtk9y0000y3y79x9jgxj'
-      end as internal_model_id,
-      if("type" = 'GENERATION',
-        '{"temperature": 0.7, "max_tokens": 150}',
-        '{}') AS model_parameters,
-      if("type" = 'GENERATION',
-        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
-        map()) AS provided_usage_details,
-      if("type" = 'GENERATION',
-        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
-        map()) AS usage_details,
-      if("type" = 'GENERATION',
-        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
-        map()) AS provided_cost_details,
-      if("type" = 'GENERATION',
-        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
-        map()) AS cost_details,
-      if("type" = 'GENERATION',
-        toDecimal64(randUniform(0, 2000), 12),
-        NULL) AS total_cost,
-      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
-      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,
-      array(${SEED_PROMPTS.map((p) => `'${p.name}'`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_name,
-      array(${SEED_PROMPTS.map((p) => `'${p.version}'`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_version,
-      start_time AS created_at,
-      start_time AS updated_at,
-      start_time AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${observationsPerProject});
-  `;
-
-    const scoresQuery = `
-    INSERT INTO scores
-    SELECT toString(number) AS id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
-      NULL AS session_id,
-      NULL AS dataset_run_id,
-      if(
-        rand() > 0.9,
-        toString(floor(randUniform(0, ${observationsPerProject}))),
-        NULL
-      ) AS observation_id,
-      concat('name_', toString(rand() % 10)) AS name,
-      randUniform(0, 100) as value,
-      'API' as source,
-      'comment' as comment,
-      map('prototype', 'test') AS metadata,
-      toString(rand() % 100) as author_user_id,
-      toString(rand() % 100) as config_id,
-      if (rand() < 0.33, 'NUMERIC', if (rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) as data_type,
-      toString(rand() % 100) as string_value,
-      NULL as queue_id,
-      timestamp AS created_at,
-      timestamp AS updated_at,
-      timestamp AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${scoresPerProject});
-  `;
-
-    const queries = [tracesQuery, scoresQuery, observationsQuery];
-
-    for (const query of queries) {
-      logger.info(`Executing query: ${query}`);
-      await clickhouseClient().command({
-        query,
-        clickhouse_settings: {
-          wait_end_of_query: 1,
-        },
-      });
-    }
-    // we also need to upsert trace sessions in postgres
-
-    const sessionQuery = `
-      SELECT session_id, project_id
-      FROM traces 
-      WHERE session_id IS NOT NULL;
-    `;
-    const sessionResult = await clickhouseClient().query({
-      query: sessionQuery,
-      format: "JSONEachRow",
-    });
-
-    const sessionData = await sessionResult.json<{
-      session_id: string;
-      project_id: string;
-    }>();
-
-    const sessionsToScore = sessionData
-      .filter(() => Math.random() < 0.5)
-      .slice(0, Math.min(500, sessionData.length));
-
-    if (sessionsToScore.length > 0) {
-      // Generate session scores query with specific session IDs
-      const sessionScoresQuery = `
-        INSERT INTO scores
-        SELECT
-          concat('session-', toString(number)) AS id,
-          toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-          '${projectId}' AS project_id,
-          'default' AS environment,
-          NULL AS trace_id,
-          arrayElement(['${sessionsToScore.map((s) => s.session_id).join("','")}'], 1 + (number % ${sessionsToScore.length})) AS session_id,
-          NULL AS dataset_run_id,
-          NULL AS observation_id,
-          concat('session_quality_', toString(rand() % 10)) AS name,
-          randUniform(0, 100) AS value,
-          'API' AS source,
-          'Session-level assessment score' AS comment,
-          map('key', 'value') AS metadata,
-          toString(rand() % 100) AS author_user_id,
-          toString(rand() % 100) AS config_id,
-          if(rand() < 0.33, 'NUMERIC', if(rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) AS data_type,
-          toString(rand() % 100) AS string_value,
-          NULL AS queue_id,
-          timestamp AS created_at,
-          timestamp AS updated_at,
-          timestamp AS event_ts,
-          0 AS is_deleted
-        FROM numbers(${sessionsToScore.length})
-      `;
-
-      await clickhouseClient().command({
-        query: sessionScoresQuery,
-        clickhouse_settings: {
-          wait_end_of_query: 1,
-        },
-      });
-    }
-
-    const idProjectIdCombinations = sessionData.map((session) => ({
-      id: session.session_id,
-      projectId: session.project_id,
-      public: Math.random() < 0.1,
-      bookmarked: Math.random() < 0.1,
-    }));
-
-    await prisma.traceSession.createMany({
-      data: idProjectIdCombinations,
-      skipDuplicates: true,
-    });
-  }
-
-  const tables = ["traces", "scores", "observations"];
-  for (const table of tables) {
-    const query = `
-            SELECT
-            project_id,
-            count() AS per_project_count,
-            bar(per_project_count, 0, (
-                SELECT count(*)
-                FROM ${table}
-            ), 50) AS bar_representation
-          FROM ${table}
-          GROUP BY project_id
-          ORDER BY count() desc
-          `;
-
-    const result = await clickhouseClient().query({
-      query,
-      format: "TabSeparated",
-    });
-
-    logger.info(
-      `${table.charAt(0).toUpperCase() + table.slice(1)} per Project: \n` +
-        (await result.text()),
-    );
-  }
-
-  const tablesWithDateColumns = [
-    { name: "traces", dateColumn: "timestamp" },
-    { name: "scores", dateColumn: "timestamp" },
-    { name: "observations", dateColumn: "start_time" },
-  ];
-
-  for (const { name: table, dateColumn } of tablesWithDateColumns) {
-    const query = `
-            SELECT
-            toDate(${dateColumn}) AS event_date,
-            count() AS per_date_count,
-            bar(per_date_count, 0, (
-                SELECT count(*)
-                FROM ${table}
-            ), 50) AS bar_representation
-          FROM ${table}
-          GROUP BY event_date
-          ORDER BY event_date desc
-          `;
-
-    const result = await clickhouseClient().query({
-      query,
-      format: "TabSeparated",
-    });
-
-    logger.info(
-      `${table.charAt(0).toUpperCase() + table.slice(1)} per Date: \n` +
-        (await result.text()),
-    );
-  }
-};

packages/shared/scripts/seeder/load-seed-clickhouse.ts

@@ -1,8 +1,8 @@
 import { randomUUID } from "crypto";
-import { prisma } from "../src/db";
-import { getDisplaySecretKey, hashSecretKey, logger } from "../src/server";
-import { prepareClickhouse } from "./prepareClickhouse";
-import { redis } from "../src/server";
+import { prisma } from "../../src/db";
+import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
+import { prepareClickhouse } from "./prepare-clickhouse";
+import { redis } from "../../src/server";
 
 const createRandomProjectId = () => randomUUID().toString();
 
@@ -117,6 +117,7 @@ async function main() {
     await prepareClickhouse(createdProjectIds, {
       numberOfDays,
       totalObservations: totalObservations ?? 1000,
+      numberOfRuns: 3,
     });
 
     logger.info("Clickhouse preparation completed successfully.");

packages/shared/scripts/seeder/prepare-clickhouse.ts

@@ -0,0 +1,35 @@
+import { SeederOrchestrator } from "./utils/seeder-orchestrator";
+import { SeederOptions } from "./utils/types";
+import { logger } from "../../src/server";
+
+/**
+ * ClickHouse data preparation using the seeder abstraction.
+ */
+export const prepareClickhouse = async (
+  projectIds: string[],
+  opts: {
+    numberOfDays: number;
+    totalObservations: number;
+    numberOfRuns?: number;
+  },
+) => {
+  logger.info(
+    `Preparing ClickHouse for ${projectIds.length} projects and ${opts.numberOfDays} days.`,
+  );
+
+  const formattedOpts: SeederOptions = {
+    numberOfDays: opts.numberOfDays,
+    totalObservations: opts.totalObservations,
+    numberOfRuns: opts.numberOfRuns || 1,
+  };
+
+  const orchestrator = new SeederOrchestrator();
+
+  try {
+    await orchestrator.executeFullSeed(projectIds, formattedOpts);
+    logger.info("ClickHouse preparation completed successfully");
+  } catch (error) {
+    logger.error("ClickHouse preparation failed:", error);
+    throw error;
+  }
+};

packages/shared/scripts/seeder/seed-clickhouse.ts

@@ -0,0 +1,31 @@
+import { prisma } from "../../src/db";
+import { redis } from "../../src/server";
+import { prepareClickhouse } from "./prepare-clickhouse";
+
+async function main() {
+  try {
+    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
+    if (
+      await prisma.project.findFirst({
+        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      })
+    ) {
+      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
+    }
+    await prepareClickhouse(projectIds, {
+      numberOfDays: 3,
+      totalObservations: 1000,
+      numberOfRuns: 3,
+    });
+
+    console.log("Clickhouse preparation completed successfully.");
+  } catch (error) {
+    console.error("Error during Clickhouse preparation:", error);
+  } finally {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    console.log("Disconnected from Clickhouse.");
+  }
+}
+
+main();

packages/shared/scripts/seeder/seed-postgres.ts

@@ -0,0 +1,924 @@
+import { randomUUID } from "node:crypto";
+import { parseArgs } from "node:util";
+import { hash } from "bcryptjs";
+import { v4 } from "uuid";
+import { encrypt } from "../../src/encryption";
+import {
+	type JobConfiguration,
+	JobExecutionStatus,
+	PrismaClient,
+	type Project,
+	ScoreDataType,
+} from "../../src/index";
+import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
+import { redis } from "../../src/server/redis/redis";
+import {EVAL_TRACE_COUNT,
+  FAILED_EVAL_TRACE_INTERVAL,
+  SEED_CHAT_ML_PROMPTS,
+  SEED_DATASETS,
+  SEED_EVALUATOR_CONFIGS,
+  SEED_EVALUATOR_TEMPLATES,
+  SEED_PROMPT_VERSIONS,
+  SEED_TEXT_PROMPTS,
+} from "./utils/postgres-seed-constants";
+import {
+  generateDatasetRunTraceId,
+  generateEvalObservationId,
+  generateEvalScoreId,
+  generateEvalTraceId,
+} from "./utils/seed-helpers";
+
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
+const options = {
+  environment: { type: "string" },
+} as const;
+
+const prisma = new PrismaClient();
+
+async function main() {
+  const environment = parseArgs({
+    options,
+  }).values.environment;
+
+  const seedOrgId = "seed-org-id";
+  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  const seedUserId1 = "user-1"; // Owner of org
+  const seedUserId2 = "user-2"; // Member of org, admin of project
+
+  const user = await prisma.user.upsert({
+    where: { id: seedUserId1 },
+    update: {
+      name: "Demo User",
+      email: "demo@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId1,
+      name: "Demo User",
+      email: "demo@langfuse.com",
+      password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
+    },
+  });
+  const user2 = await prisma.user.upsert({
+    where: { id: seedUserId2 },
+    update: {
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId2,
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+  });
+
+  await prisma.organization.upsert({
+    where: { id: seedOrgId },
+    update: {
+      name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
+    },
+    create: {
+      id: seedOrgId,
+      name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
+    },
+  });
+
+  const project1 = await prisma.project.upsert({
+    where: { id: seedProjectId },
+    update: {
+      name: "llm-app",
+      orgId: seedOrgId,
+    },
+    create: {
+      id: seedProjectId,
+      name: "llm-app",
+      orgId: seedOrgId,
+    },
+  });
+
+  await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user.id,
+      orgId: seedOrgId,
+      role: "OWNER",
+    },
+    update: {},
+  });
+
+  const orgMembership2 = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user2.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user2.id,
+      orgId: seedOrgId,
+      role: "MEMBER",
+    },
+    update: {},
+  });
+
+  await prisma.projectMembership.upsert({
+    where: {
+      projectId_userId: {
+        projectId: project1.id,
+        userId: user2.id,
+      },
+    },
+    create: {
+      userId: user2.id,
+      projectId: project1.id,
+      role: "ADMIN",
+      orgMembershipId: orgMembership2.id,
+    },
+    update: {
+      orgMembershipId: orgMembership2.id,
+    },
+  });
+
+  await prisma.prompt.upsert({
+    where: {
+      projectId_name_version: {
+        projectId: seedProjectId,
+        name: "summary-prompt",
+        version: 1,
+      },
+    },
+    create: {
+      name: "summary-prompt",
+      project: { connect: { id: seedProjectId } },
+      prompt: "prompt {{variable}} {{anotherVariable}}",
+      labels: ["production", "latest"],
+      version: 1,
+      createdBy: "user-1",
+    },
+    update: {},
+  });
+
+  const seedApiKey = {
+    id: "seed-api-key",
+    secret: process.env.SEED_SECRET_KEY ?? "sk-lf-1234567890", // eslint-disable-line turbo/no-undeclared-env-vars
+    public: "pk-lf-1234567890",
+    note: "seeded key",
+  };
+
+  if (!(await prisma.apiKey.findUnique({ where: { id: seedApiKey.id } }))) {
+    await prisma.apiKey.create({
+      data: {
+        note: seedApiKey.note,
+        id: seedApiKey.id,
+        publicKey: seedApiKey.public,
+        hashedSecretKey: await hashSecretKey(seedApiKey.secret),
+        displaySecretKey: getDisplaySecretKey(seedApiKey.secret),
+        scope: "PROJECT",
+        project: {
+          connect: {
+            id: project1.id,
+          },
+        },
+      },
+    });
+  }
+
+  // Do not run the following for local docker compose setup
+  if (environment === "examples" || environment === "load") {
+    const seedOrgIdOrg2 = "demo-org-id";
+    const project2Id = "239ad00f-562f-411d-af14-831c75ddd875";
+    const org2 = await prisma.organization.upsert({
+      where: { id: seedOrgIdOrg2 },
+      update: {
+        name: "Langfuse Demo",
+      },
+      create: {
+        id: seedOrgIdOrg2,
+        name: "Langfuse Demo",
+      },
+    });
+    const project2 = await prisma.project.upsert({
+      where: { id: project2Id },
+      create: {
+        id: project2Id,
+        name: "demo-app",
+        orgId: org2.id,
+      },
+      update: { orgId: seedOrgIdOrg2 },
+    });
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: {
+          userId: user.id,
+          orgId: seedOrgIdOrg2,
+        },
+      },
+      create: {
+        userId: user.id,
+        orgId: seedOrgIdOrg2,
+        role: "VIEWER",
+      },
+      update: {},
+    });
+
+    const secondKey = {
+      id: "seed-api-key-2",
+      secret: process.env.SEED_SECRET_KEY ?? "sk-lf-asdfghjkl", // eslint-disable-line turbo/no-undeclared-env-vars
+      public: "pk-lf-asdfghjkl",
+      note: "seeded key 2",
+    };
+    if (!(await prisma.apiKey.findUnique({ where: { id: secondKey.id } }))) {
+      await prisma.apiKey.create({
+        data: {
+          note: secondKey.note,
+          id: secondKey.id,
+          publicKey: secondKey.public,
+          hashedSecretKey: await hashSecretKey(secondKey.secret),
+          displaySecretKey: getDisplaySecretKey(secondKey.secret),
+          scope: "PROJECT",
+          project: {
+            connect: {
+              id: project2.id,
+            },
+          },
+        },
+      });
+    }
+
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
+    await generateQueuesForProject([project1, project2], configIdsAndNames);
+    await generatePromptsForProject([project1, project2]);
+    await createDatasets(project1, project2);
+    await createTraceSessions(project1, project2);
+
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY; // eslint-disable-line turbo/no-undeclared-env-vars
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+          adapter: "openai",
+        },
+      });
+    } else {
+      logger.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key.",
+      );
+    }
+
+    // add eval objects
+    for (const evalTemplate of SEED_EVALUATOR_TEMPLATES) {
+      await prisma.evalTemplate.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project1.id,
+            name: evalTemplate.name,
+            version: 1,
+          },
+        },
+        create: {
+          id: evalTemplate.id,
+          projectId: project1.id,
+          name: evalTemplate.name,
+          version: evalTemplate.version,
+          prompt: evalTemplate.prompt,
+          model: evalTemplate.model,
+          vars: evalTemplate.vars,
+          provider: evalTemplate.provider,
+          outputSchema: evalTemplate.outputSchema,
+          modelParams: evalTemplate.modelParams,
+        },
+        update: {},
+      });
+    }
+
+    for (const evalConfig of SEED_EVALUATOR_CONFIGS) {
+      await prisma.jobConfiguration.upsert({
+        where: {
+          id: evalConfig.id,
+        },
+        create: {
+          id: evalConfig.id,
+          evalTemplateId: evalConfig.evalTemplateId,
+          projectId: project1.id,
+          jobType: evalConfig.jobType as any,
+          status: evalConfig.status as any,
+          scoreName: evalConfig.scoreName,
+          filter: evalConfig.filter,
+          variableMapping: evalConfig.variableMapping,
+          targetObject: evalConfig.targetObject,
+          sampling: evalConfig.sampling,
+          delay: evalConfig.delay,
+        },
+        update: {},
+      });
+    }
+
+    await generateEvalJobExecutions(
+      [project1, project2],
+      SEED_EVALUATOR_CONFIGS as unknown as Partial<JobConfiguration>[],
+    );
+
+    await createDashboardsAndWidgets([project1, project2]);
+
+    await prisma.llmSchema.createMany({
+      data: [
+        {
+          projectId: project1.id,
+          name: "get_weather",
+          description: "Fetches weather in Celsius for a given location",
+          schema: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+              unit: {
+                type: "string",
+                enum: ["celsius", "fahrenheit"],
+              },
+            },
+            required: ["location", "unit"],
+          },
+        },
+        {
+          projectId: project1.id,
+          name: "calculator",
+          description: "Performs basic arithmetic calculations",
+          schema: {
+            type: "object",
+            properties: {
+              expression: {
+                type: "string",
+                description:
+                  "The mathematical expression to evaluate, e.g. '2 + 2'",
+              },
+            },
+            required: ["expression"],
+          },
+        },
+      ],
+    });
+  }
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+  })
+  .catch(async (e) => {
+    logger.error(e);
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+    process.exit(1);
+  });
+
+async function createDashboardsAndWidgets(projects: Project[]) {
+  logger.info("Creating dashboards and widgets");
+
+  // Process each project
+  for (const project of projects) {
+    const widget = await prisma.dashboardWidget.upsert({
+      where: { id: "cabc" },
+      create: {
+        id: "cabc",
+        projectId: project.id,
+        name: "Trace Counts",
+        description: "Trace Counts by Name Over Time",
+        view: "TRACES",
+        dimensions: [{ field: "name" }],
+        metrics: [{ measure: "count", agg: "count" }],
+        filters: [],
+        chartType: "BAR_TIME_SERIES",
+        chartConfig: {
+          type: "BAR_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    const widget2 = await prisma.dashboardWidget.upsert({
+      where: { id: "cdef" },
+      create: {
+        id: "cdef",
+        projectId: project.id,
+        name: "Observation Latencies by Model",
+        description: "p95 Observation Latencies by Model Name",
+        view: "OBSERVATIONS",
+        dimensions: [{ field: "providedModelName" }],
+        metrics: [{ measure: "count", agg: "sum" }],
+        filters: [],
+        chartType: "LINE_TIME_SERIES",
+        chartConfig: {
+          type: "LINE_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    // Create a dashboard with multiple widgets
+    await prisma.dashboard.upsert({
+      where: { id: "seed-dashboard" },
+      create: {
+        id: "seed-dashboard",
+        projectId: project.id,
+        name: "Performance Overview",
+        description: "Dashboard with various performance metrics",
+        definition: {
+          widgets: [
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget.id,
+              x: 0,
+              y: 0,
+              x_size: 6,
+              y_size: 6,
+            },
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget2.id,
+              x: 6,
+              y: 0,
+              x_size: 6,
+              y_size: 6,
+            },
+          ],
+        },
+      },
+      update: {},
+    });
+  }
+}
+
+export async function createDatasets(
+  project1: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  project2: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+) {
+  for (const data of SEED_DATASETS) {
+    for (const projectId of [project1.id, project2.id]) {
+      const datasetName = data.name;
+
+      // check if ds already exists
+      const dataset =
+        (await prisma.dataset.findFirst({
+          where: {
+            projectId,
+            name: datasetName,
+          },
+        })) ??
+        (await prisma.dataset.create({
+          data: {
+            name: datasetName,
+            description: data.description,
+            projectId,
+            metadata: data.metadata,
+          },
+        }));
+
+      const datasetItemIds: string[] = [];
+      for (let index = 0; index < data.items.length; index++) {
+        const item = data.items[index];
+        const sourceTraceId =
+          Math.random() > 0.3
+            ? `${Math.floor(Math.random() * 100)}`
+            : undefined;
+
+        // Use upsert to prevent duplicates
+        const datasetItem = await prisma.datasetItem.upsert({
+          where: {
+            id_projectId: {
+              id: `${dataset.id}-${index}`,
+              projectId,
+            },
+          },
+          create: {
+            projectId,
+            id: `${dataset.id}-${index}`,
+            datasetId: dataset.id,
+            sourceTraceId: sourceTraceId ?? null,
+            sourceObservationId: null,
+            input: item.input,
+            expectedOutput: item.output,
+            metadata: Math.random() > 0.5 ? { key: "value" } : undefined,
+          },
+          update: {}, // Don't update if it exists
+        });
+        datasetItemIds.push(datasetItem.id);
+      }
+
+      for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
+        const datasetRun = await prisma.datasetRuns.upsert({
+          where: {
+            datasetId_projectId_name: {
+              datasetId: dataset.id,
+              projectId,
+              name: `demo-dataset-run-${datasetRunNumber}`,
+            },
+          },
+          create: {
+            projectId,
+            name: `demo-dataset-run-${datasetRunNumber}`,
+            description: Math.random() > 0.5 ? "Dataset run description" : "",
+            datasetId: dataset.id,
+            metadata: [
+              undefined,
+              "string",
+              100,
+              { key: "value" },
+              ["tag1", "tag2"],
+            ][datasetRunNumber % 5],
+          },
+          update: {},
+        });
+
+        for (let index = 0; index < datasetItemIds.length; index++) {
+          await prisma.datasetRunItems.upsert({
+            where: {
+              id_projectId: {
+                id: `${dataset.id}-${index}-${datasetRunNumber}`,
+                projectId,
+              },
+            },
+            create: {
+              id: `${dataset.id}-${index}-${datasetRunNumber}`,
+              projectId,
+              datasetItemId: datasetItemIds[index],
+              traceId: `${generateDatasetRunTraceId(datasetName, index, projectId, datasetRunNumber)}`,
+              datasetRunId: datasetRun.id,
+            },
+            update: {},
+          });
+        }
+      }
+    }
+  }
+}
+
+async function generateEvalJobExecutions(
+  projects: Project[],
+  evalJobConfigurations: Partial<JobConfiguration>[],
+) {
+  for (const project of projects) {
+    for (let i = 0; i < EVAL_TRACE_COUNT; i++) {
+      const jobConfiguration =
+        evalJobConfigurations[i % evalJobConfigurations.length];
+
+      const isFailed = i % FAILED_EVAL_TRACE_INTERVAL === 0;
+      await prisma.jobExecution.create({
+        data: {
+          projectId: project.id,
+          jobTemplateId: jobConfiguration.evalTemplateId,
+          jobInputTraceId: generateEvalTraceId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+          jobConfigurationId: jobConfiguration.id!,
+          status: isFailed
+            ? JobExecutionStatus.ERROR
+            : JobExecutionStatus.COMPLETED,
+          error: isFailed ? "Error message" : undefined,
+          jobOutputScoreId: generateEvalScoreId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+          jobInputObservationId: generateEvalObservationId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+        },
+      });
+    }
+  }
+}
+
+async function generatePromptsForProject(projects: Project[]) {
+  const promptIds = new Map<string, string[]>();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const promptIdsForProject = await generatePrompts(project);
+      promptIds.set(project.id, promptIdsForProject);
+    }),
+  );
+  return promptIds;
+}
+
+export const PROMPT_IDS: string[] = [];
+
+async function generatePrompts(project: Project) {
+  const promptIds = [];
+  for (const prompt of SEED_TEXT_PROMPTS) {
+    const versions = Math.floor(Math.random() * 20) + 1;
+    for (let i = 1; i <= versions; i++) {
+      const promptId = `prompt-${v4()}`;
+      await prisma.prompt.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project.id,
+            name: prompt.name,
+            version: i,
+          },
+          id: promptId,
+        },
+        create: {
+          id: promptId,
+          projectId: project.id,
+          createdBy: prompt.createdBy,
+          prompt: `${prompt.prompt} version ${i} content`,
+          name: prompt.name,
+          version: i,
+          labels: i === versions ? prompt.labels : [],
+        },
+        update: {
+          id: promptId,
+        },
+      });
+      promptIds.push(promptId);
+    }
+  }
+
+  for (const prompt of SEED_CHAT_ML_PROMPTS) {
+    const promptId = `prompt-${v4()}`;
+    const versions = Math.floor(Math.random() * 20) + 1;
+    for (let i = 1; i <= versions; i++) {
+      const versionAddition = [
+        {
+          role: "user",
+          content: "This is content for version " + i,
+        },
+      ];
+
+      await prisma.prompt.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project.id,
+            name: prompt.name,
+            version: prompt.version,
+          },
+          id: promptId,
+        },
+        create: {
+          id: promptId,
+          projectId: project.id,
+          createdBy: prompt.createdBy,
+          prompt: [...prompt.prompt, ...versionAddition],
+          name: prompt.name,
+          version: i,
+          type: "chat",
+          labels: prompt.labels,
+          tags: prompt.tags,
+        },
+        update: {
+          id: promptId,
+        },
+      });
+      promptIds.push(promptId);
+    }
+  }
+
+  for (const version of SEED_PROMPT_VERSIONS) {
+    const id = `prompt-${v4()}`;
+    await prisma.prompt.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project.id,
+          name: version.name,
+          version: version.version,
+        },
+        id: id,
+      },
+      create: {
+        id: id,
+        projectId: project.id,
+        createdBy: version.createdBy,
+        prompt: version.prompt,
+        name: version.name,
+        config: version.config,
+        version: version.version,
+        labels: version.labels,
+      },
+      update: {
+        id: id,
+      },
+    });
+    promptIds.push(id);
+  }
+
+  return promptIds;
+}
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    }),
+  );
+  return projectIdsToConfigs;
+}
+
+async function createTraceSessions(project1: Project, project2: Project) {
+  for (const project of [project1, project2]) {
+    for (let i = 0; i < 100; i++) {
+      await prisma.traceSession.create({
+        data: {
+          projectId: project.id,
+          id: `session_${i}`,
+          createdAt: new Date(),
+        },
+      });
+    }
+  }
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
+  }
+
+  return configNameAndId;
+}
+
+async function generateQueuesForProject(
+  projects: Project[],
+  configIdsAndNames: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >,
+) {
+  const projectIdsToQueues: Map<string, string[]> = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const queueIds = await generateQueues(
+        project,
+        configIdsAndNames.get(project.id) ?? [],
+      );
+      projectIdsToQueues.set(project.id, queueIds);
+    }),
+  );
+  return projectIdsToQueues;
+}
+
+async function generateQueues(
+  project: Project,
+  configIdsAndNames: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[],
+) {
+  const queue = {
+    id: `queue-${v4()}`,
+    name: "Default",
+    description: "Default queue",
+    scoreConfigIds: configIdsAndNames.map((config) => config.id),
+    projectId: project.id,
+  };
+
+  await prisma.annotationQueue.upsert({
+    where: {
+      projectId_name: {
+        projectId: queue.projectId,
+        name: queue.name,
+      },
+    },
+    create: {
+      ...queue,
+    },
+    update: {
+      id: queue.id,
+    },
+  });
+
+  return [queue.id];
+}

packages/shared/scripts/seeder/utils/README.md

@@ -0,0 +1,160 @@
+# Langfuse Seeder System
+
+System for generating test data in ClickHouse and PostgreSQL for Langfuse development and testing.
+
+## Architecture Overview
+
+```
+seeder/
+├── types.ts                 # Core interfaces and types
+├── data-generators.ts       # Data generation logic
+├── clickhouse-builder.ts    # ClickHouse query building
+├── seeder-orchestrator.ts   # Main orchestration logic
+├── postgres-seed-constants.ts  # PostgreSQL data constants
+├── clickhouse-seed-constants.ts  # ClickHouse data constants
+└── seed-helpers.ts          # Utility functions
+```
+
+## Quick Start
+
+```typescript
+import { SeederOrchestrator } from "./seeder/seeder-orchestrator";
+
+const orchestrator = new SeederOrchestrator();
+
+// Full seed (datasets + evaluation + synthetic data)
+await orchestrator.executeFullSeed(projectIds, {
+  numberOfDays: 30,
+  totalObservations: 10000,
+  numberOfRuns: 3,
+});
+
+// Individual data types
+await orchestrator.createDatasetExperimentData(projectIds, config);
+await orchestrator.createEvaluationData(projectIds);
+await orchestrator.createSyntheticData(projectIds, config);
+```
+
+## Generated Data
+
+### 1. Dataset Experiment Data
+
+- **Purpose**: Realistic experiment traces based on actual datasets
+- **Environment**: `langfuse-prompt-experiments`
+- **Structure**: Each dataset item links to a trace with a single generation observation
+- **ID Pattern**: `trace-dataset-{datasetName}-{itemIndex}-{projectId}-{runNumber}`
+
+### 2. Evaluation Data
+
+- **Purpose**: Evaluation metrics and scoring data - to be linked to evaluation logs
+- **Environment**: `langfuse-evaluation`
+- **Structure**: Traces with multiple observations and comprehensive scoring
+- **ID Pattern**: `trace-eval-{index}-{projectId}`
+
+### 3. Synthetic Data
+
+- **Purpose**: Large-scale realistic tracing data
+- **Environment**: `default`
+- **Structure**: Hierarchical traces with multiple observations and scores
+- **ID Pattern**: `trace-synthetic-{index}-{projectId}`
+
+## Abstraction Architecture
+
+### DataGenerator
+
+Generates realistic data for all three types. If you need to change any clickhouse data, you should modify this class. Key methods:
+
+- `generateDatasetTrace()` - Creates traces from dataset items
+- `generateSyntheticTraces()` - Creates realistic synthetic traces
+- `generateEvaluationTraces()` - Creates evaluation-focused traces
+
+### ClickHouseQueryBuilder
+
+Builds optimized ClickHouse insert queries. No need to edit this file. Handles proper escaping and type handling.
+
+### SeederOrchestrator
+
+Main coordination class that:
+
+- Loads file content for realistic inputs/outputs
+- Coordinates data generation and insertion
+- Handles batching and error recovery
+- Provides logging and statistics
+
+## Making Changes
+
+### Configuration Options
+
+```typescript
+interface SeederConfig {
+  numberOfDays: number; // How far back to generate timestamps
+  numberOfRuns?: number; // How many experiment runs per dataset
+  totalObservations?: number; // Total observations for synthetic data
+}
+```
+
+### Extending the System
+
+#### Adding New Data Types
+
+1. Add interface to `types.ts`
+2. Add generator method to `DataGenerator`
+3. Add query builder method to `ClickHouseQueryBuilder`
+4. Add orchestration method to `SeederOrchestrator`
+5. Update interdependency documentation
+
+#### Adding New File Sources
+
+1. Add file path to `SeederOrchestrator.loadFileContent()`
+2. Add processing logic to `DataGenerator`
+3. Update `FileContent` interface if needed
+
+#### Changing Data Distribution
+
+1. Modify generator methods in `DataGenerator`
+2. Update constants in `clickhouse-seed-constants.ts`
+3. Test with small datasets first
+
+#### Changing ID Generation
+
+1. **Check**: All places that query ClickHouse by ID
+2. **Check**: PostgreSQL foreign key references
+3. **Check**: Dataset run item and evaluation trace creation logic
+4. **Action**: Update `seed-helpers.ts` functions consistently
+
+#### Changing Environment Names
+
+1. **Check**: All ClickHouse queries that filter by environment
+2. **Check**: PostgreSQL dataset and prompt environment fields
+3. **Check**: UI environment filtering logic
+4. **Action**: Update constants in both systems
+
+#### Changing Data Structure
+
+1. **Check**: ClickHouse table schema compatibility
+2. **Check**: PostgreSQL table relationships
+3. **Check**: API response serialization
+4. **Action**: Update both schemas before changing data generation
+
+#### Adding New Data Types
+
+1. **Check**: Whether PostgreSQL needs corresponding tables
+2. **Check**: Whether new foreign key relationships are needed
+3. **Check**: Whether UI needs to handle new data types
+4. **Action**: Plan database migrations carefully
+
+## File Dependencies
+
+### Required Files
+
+```
+packages/shared/clickhouse/
+├── nested_json.json      # Large JSON for realistic inputs
+├── markdown.txt          # Markdown content for document analysis
+└── chat_ml_json.json     # Chat ML format examples
+```
+
+### Constants Files
+
+- `postgres-seed-constants.ts` - Datasets, prompts, and PostgreSQL data
+- `clickhouse-seed-constants.ts` - ClickHouse-specific constants (models, names)

packages/shared/scripts/seeder/utils/chat_ml_json.json

@@ -0,0 +1,135 @@
+{
+  "messages": [
+    {
+      "content": "what is the weather in sf",
+      "additional_kwargs": {},
+      "response_metadata": {},
+      "type": "human",
+      "name": null,
+      "id": "2166b887-b9fb-4282-a9f7-828d4a6cca54",
+      "example": false
+    },
+    {
+      "content": "",
+      "additional_kwargs": {
+        "tool_calls": [
+          {
+            "id": "call_pNLR4DoZiptb5xGlb299QsoN",
+            "function": {
+              "arguments": "{\"query\":\"current weather in San Francisco\"}",
+              "name": "search"
+            },
+            "type": "function"
+          }
+        ],
+        "refusal": null
+      },
+      "response_metadata": {
+        "token_usage": {
+          "completion_tokens": 17,
+          "prompt_tokens": 48,
+          "total_tokens": 65,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          }
+        },
+        "model_name": "gpt-4o-mini-2024-07-18",
+        "system_fingerprint": "fp_34a54ae93c",
+        "finish_reason": "tool_calls",
+        "logprobs": null
+      },
+      "type": "ai",
+      "name": null,
+      "id": "run-14c4801c-ae47-4c4a-8d49-29ba08c5679d-0",
+      "example": false,
+      "tool_calls": [
+        {
+          "name": "search",
+          "args": {
+            "query": "current weather in San Francisco"
+          },
+          "id": "call_pNLR4DoZiptb5xGlb299QsoN",
+          "type": "tool_call"
+        }
+      ],
+      "invalid_tool_calls": [],
+      "usage_metadata": {
+        "input_tokens": 48,
+        "output_tokens": 17,
+        "total_tokens": 65,
+        "input_token_details": {
+          "audio": 0,
+          "cache_read": 0
+        },
+        "output_token_details": {
+          "audio": 0,
+          "reasoning": 0
+        }
+      }
+    },
+    {
+      "content": "It's 60 degrees and foggy.",
+      "additional_kwargs": {},
+      "response_metadata": {},
+      "type": "tool",
+      "name": "search",
+      "id": "220ba511-6816-4d1a-8acf-8e5791c2d88e",
+      "tool_call_id": "call_pNLR4DoZiptb5xGlb299QsoN",
+      "artifact": null,
+      "status": "success"
+    },
+    {
+      "content": "The current weather in San Francisco is 60 degrees and foggy.",
+      "additional_kwargs": {
+        "refusal": null
+      },
+      "response_metadata": {
+        "token_usage": {
+          "completion_tokens": 15,
+          "prompt_tokens": 80,
+          "total_tokens": 95,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          }
+        },
+        "model_name": "gpt-4o-mini-2024-07-18",
+        "system_fingerprint": "fp_34a54ae93c",
+        "finish_reason": "stop",
+        "logprobs": null
+      },
+      "type": "ai",
+      "name": null,
+      "id": "run-f051be3b-2656-4b19-8704-a2e492306131-0",
+      "example": false,
+      "tool_calls": [],
+      "invalid_tool_calls": [],
+      "usage_metadata": {
+        "input_tokens": 80,
+        "output_tokens": 15,
+        "total_tokens": 95,
+        "input_token_details": {
+          "audio": 0,
+          "cache_read": 0
+        },
+        "output_token_details": {
+          "audio": 0,
+          "reasoning": 0
+        }
+      }
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -0,0 +1,242 @@
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../../../src/server";
+import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
+import {
+  createTracesCh,
+  createObservationsCh,
+  createScoresCh,
+} from "../../../src/server";
+import { InsertResult } from "@clickhouse/client";
+
+/**
+ * Builds or executes ClickHouse SQL INSERT queries for seeding test data.
+ *
+ * Use executeXxxInsert() for custom curated data with detailed control.
+ * Use buildBulkXxxInsert() for large datasets (>1000 items) for random distribution of data.
+ */
+export class ClickHouseQueryBuilder {
+  private escapeString(str: string): string {
+    return str.replace(/'/g, "''");
+  }
+
+  /**
+   * Creates INSERT query for trace data using VALUES syntax.
+   * Use for: Small datasets, detailed trace objects with all fields populated.
+   */
+  async executeTracesInsert(
+    traces: TraceRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createTracesCh(traces);
+  }
+
+  /**
+   * Creates INSERT query for observation data using VALUES syntax.
+   * Use for: Small datasets, observations that link to postgres data (e.g. dataset runs)
+   */
+  async executeObservationsInsert(
+    observations: ObservationRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createObservationsCh(observations);
+  }
+
+  /**
+   * Creates INSERT query for score data using VALUES syntax.
+   * Use for: Small datasets, scores with custom values and metadata.
+   */
+  async executeScoresInsert(
+    scores: ScoreRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createScoresCh(scores);
+  }
+
+  /**
+   * Creates INSERT using ClickHouse numbers() function.
+   * Use for: Large datasets (>1000 traces), realistic timestamps, bulk generation.
+   */
+  buildBulkTracesInsert(
+    projectId: string,
+    count: number,
+    environment: string = "default",
+    fileContent?: { heavyMarkdown: string; nestedJson: any; chatMlJson: any },
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    // Escape file content if provided
+    const escapedHeavyMarkdown = fileContent
+      ? this.escapeString(fileContent.heavyMarkdown)
+      : "Sample heavy markdown content";
+    const escapedNestedJson = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.nestedJson))
+      : '{"sample": "nested json"}';
+    const escapedChatMl = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.chatMlJson))
+      : '{"messages": []}';
+
+    return `
+      INSERT INTO traces
+      SELECT 
+        concat('trace-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+        concat('trace-', toString(number % 10)) AS name,
+        if(randUniform(0, 1) < 0.3, concat('user_', toString(rand() % 1000)), NULL) AS user_id,
+        map('generated', 'bulk') AS metadata,
+        NULL AS release,
+        NULL AS version,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        if(rand() < 0.8, true, false) AS public,
+        if(rand() < 0.1, true, false) AS bookmarked,
+        array() AS tags,
+        if(randUniform(0, 1) < 0.3, '${escapedHeavyMarkdown}',
+          '${escapedChatMl}'
+        ) AS input,
+        if(randUniform(0, 1) < 0.2, '${escapedNestedJson}',
+          '${escapedChatMl}'
+        ) AS output,
+        if(randUniform(0, 1) < 0.3, concat('session_', toString(rand() % 100)), NULL) AS session_id,
+        now() AS created_at,
+        now() AS updated_at,
+        now() AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${count});
+    `;
+  }
+
+  /**
+   * Creates observations with automatic prompt linking (10% rate).
+   * Use for: Large datasets, hierarchical observations, cost/latency variation.
+   */
+  buildBulkObservationsInsert(
+    projectId: string,
+    tracesCount: number,
+    observationsPerTrace: number = 5,
+    environment: string = "default",
+    fileContent?: { heavyMarkdown: string; nestedJson: any; chatMlJson: any },
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    const totalObservations = tracesCount * observationsPerTrace;
+
+    // Escape file content if provided
+    const escapedHeavyMarkdown = fileContent
+      ? this.escapeString(fileContent.heavyMarkdown)
+      : "Sample heavy markdown content";
+    const escapedNestedJson = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.nestedJson))
+      : '{"sample": "nested json"}';
+    const escapedChatMl = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.chatMlJson))
+      : '{"messages": []}';
+
+    return `
+      INSERT INTO observations
+      SELECT 
+        concat('obs-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        concat('trace-bulk-', toString(number % ${tracesCount}), '-${projectId.slice(-8)}') AS trace_id,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
+        if(number % 6 = 0, NULL, toString(number - 1)) AS parent_observation_id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
+        addMilliseconds(start_time, 
+          case 
+            when type = 'GENERATION' then floor(randUniform(5, 30))
+            when type = 'SPAN' then floor(randUniform(1, 50))
+            else floor(randUniform(1, 10))
+          end) AS end_time,
+        case
+          when type = 'GENERATION' then concat('generation-', toString(number % 10))
+          when type = 'SPAN' then concat('span-', toString(number % 10))
+          else concat('event-', toString(number % 10))
+        end AS name,
+        map('key', 'value') AS metadata,
+        if(randUniform(0, 1) < 0.85, 'DEFAULT', if(randUniform(0, 1) < 0.7, 'DEBUG', if(randUniform(0, 1) < 0.3, 'ERROR', 'WARNING'))) AS level,
+        NULL AS status_message,
+        NULL AS version,
+        if(type = 'GENERATION', 
+          if(randUniform(0, 1) < 0.4, '${escapedHeavyMarkdown}', '${escapedChatMl}'),
+          NULL) AS input,
+        if(type = 'GENERATION', 
+          if(randUniform(0, 1) < 0.3, '${escapedNestedJson}', '${escapedChatMl}'),
+          NULL) AS output,
+        if(type = 'GENERATION', 'gpt-4', NULL) AS provided_model_name,
+        if(type = 'GENERATION', concat('model_', toString(rand() % 1000)), NULL) AS internal_model_id,
+        if(type = 'GENERATION', '{"temperature": 0.7}', '{}') AS model_parameters,
+        if(type = 'GENERATION', map('input', toUInt64(randUniform(20, 200)), 'output', toUInt64(randUniform(10, 100)), 'total', toUInt64(randUniform(30, 300))), map()) AS provided_usage_details,
+        if(type = 'GENERATION', map('input', toUInt64(randUniform(20, 200)), 'output', toUInt64(randUniform(10, 100)), 'total', toUInt64(randUniform(30, 300))), map()) AS usage_details,
+        if(type = 'GENERATION', map('input', toDecimal64(randUniform(0.00001, 0.001), 8), 'output', toDecimal64(randUniform(0.00001, 0.002), 8), 'total', toDecimal64(randUniform(0.00002, 0.003), 8)), map()) AS provided_cost_details,
+        if(type = 'GENERATION', map('input', toDecimal64(randUniform(0.00001, 0.001), 8), 'output', toDecimal64(randUniform(0.00001, 0.002), 8), 'total', toDecimal64(randUniform(0.00002, 0.003), 8)), map()) AS cost_details,
+        if(type = 'GENERATION', toDecimal64(randUniform(0.00002, 0.003), 8), NULL) AS total_cost,
+        if(type = 'GENERATION', addMilliseconds(start_time, floor(randUniform(100, 500))), NULL) AS completion_start_time,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.id).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_id,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.name).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_name,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.version).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_version,
+        start_time AS created_at,
+        start_time AS updated_at,
+        start_time AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${totalObservations});
+    `;
+  }
+
+  /**
+   * Creates scores with mixed data types (NUMERIC/BOOLEAN/CATEGORICAL).
+   * Use for: Large datasets, varied score distributions, synthetic metrics.
+   */
+  buildBulkScoresInsert(
+    projectId: string,
+    tracesCount: number,
+    scoresPerTrace: number = 2,
+    environment: string = "default",
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    const totalScores = tracesCount * scoresPerTrace;
+
+    return `
+      INSERT INTO scores
+      SELECT 
+        concat('score-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        concat('trace-bulk-', toString(number % ${tracesCount}), '-${projectId.slice(-8)}') AS trace_id,
+        if(randUniform(0, 1) < 0.3, concat('session_', toString(rand() % 100)), NULL) AS session_id,
+        NULL AS dataset_run_id,
+        if(randUniform(0, 1) < 0.1, concat('obs-bulk-', toString(rand() % (${tracesCount} * 5)), '-${projectId.slice(-8)}'), NULL) AS observation_id,
+        concat('metric_', toString((number % ${scoresPerTrace * 5}) + 1)) AS name,
+        case 
+          when (number % 3) = 0 then toDecimal64(randUniform(0, 100), 8)
+          when (number % 3) = 1 then if(randUniform(0, 1) < 0.5, 1, 0)
+          else NULL
+        end AS value,
+        'API' AS source,
+        'Generated synthetic score' AS comment,
+        map() AS metadata,
+        NULL AS author_user_id,
+        NULL AS config_id,
+        case 
+          when (number % 3) = 0 then 'NUMERIC'
+          when (number % 3) = 1 then 'BOOLEAN'
+          else 'CATEGORICAL'
+        end AS data_type,
+        case 
+          when (number % 3) = 1 then if(value = 1, 'true', 'false')
+          when (number % 3) = 2 then concat('category_', toString((rand() % 5) + 1))
+          else NULL
+        end AS string_value,
+        NULL AS queue_id,
+        timestamp AS created_at,
+        timestamp AS updated_at,
+        timestamp AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${totalScores});
+    `;
+  }
+}

packages/shared/scripts/seeder/utils/clickhouse-seed-constants.ts

@@ -0,0 +1,97 @@
+export const REALISTIC_TRACE_NAMES = [
+  "LangGraph",
+  "ChatCompletion",
+  "DocumentAnalysis",
+  "CodeGeneration",
+  "DataProcessing",
+  "QueryExecution",
+  "ModelInference",
+  "WebScraping",
+  "TextSummarization",
+  "ImageClassification",
+];
+
+export const REALISTIC_SPAN_NAMES = [
+  "agent",
+  "tools",
+  "search",
+  "retrieval",
+  "preprocessing",
+  "validation",
+  "transformation",
+  "classification",
+  "extraction",
+  "postprocessing",
+];
+
+export const REALISTIC_GENERATION_NAMES = [
+  "ChatOpenAI",
+  "GPT-4",
+  "Claude-3",
+  "Gemini",
+  "Llama-2",
+  "PaLM",
+  "CodeLlama",
+  "Mistral",
+  "Falcon",
+  "Vicuna",
+];
+
+export const REALISTIC_MODELS = [
+  "gpt-4o-mini-2024-07-18",
+  "gpt-4-turbo-2024-04-09",
+  "claude-3-haiku-20240307",
+  "claude-3-sonnet-20240229",
+  "claude-3-opus-20240229",
+  "gemini-pro",
+  "llama-2-70b-chat",
+  "mistral-7b-instruct",
+  "codellama-34b-instruct",
+];
+
+export const REALISTIC_USER_INPUTS = [
+  "What is the weather in San Francisco?",
+  "Summarize this document for me",
+  "Generate a Python function to sort a list",
+  "Explain quantum computing in simple terms",
+  "Analyze this data and provide insights",
+  "Translate this text to Spanish",
+  "Create a marketing email for our product",
+  "Debug this code and fix the errors",
+  "What are the latest trends in AI?",
+  "Help me plan a trip to Europe",
+];
+
+export const REALISTIC_AI_RESPONSES = [
+  "The current weather in San Francisco is 60 degrees and foggy.",
+  "Here's a summary of the key points from the document...",
+  "Here's a Python function that sorts a list efficiently...",
+  "Quantum computing uses quantum mechanics principles...",
+  "Based on the data analysis, I found the following insights...",
+  "Here's the Spanish translation of your text...",
+  "I've created a compelling marketing email for your product...",
+  "I found several issues in your code and here are the fixes...",
+  "The latest AI trends include large language models...",
+  "Here's a detailed 10-day European itinerary for you...",
+];
+
+export const REALISTIC_METADATA_EXAMPLES = [
+  { thread_id: 42, session_type: "interactive" },
+  { user_id: "user_123", conversation_id: "conv_456" },
+  { model_version: "v2.1", temperature: 0.7 },
+  { request_id: "req_789", timestamp: "2024-05-23T15:42:11.996Z" },
+  { environment: "production", region: "us-west-2" },
+  {
+    document_type: "state_of_union",
+    file_size: "142KB",
+    processing_time: "2.3s",
+  },
+  { data_source: "product_catalog", record_count: 30, format: "nested_json" },
+  { analysis_type: "sentiment", language: "en", confidence: 0.92 },
+  {
+    extraction_task: "policy_analysis",
+    domain: "politics",
+    entities_found: 15,
+  },
+  { file_type: "JSON", validation: "passed", schema_version: "v1.2" },
+];

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -0,0 +1,594 @@
+import { FileContent, DatasetItemInput } from "./types";
+import {
+  REALISTIC_TRACE_NAMES,
+  REALISTIC_SPAN_NAMES,
+  REALISTIC_GENERATION_NAMES,
+  REALISTIC_MODELS,
+} from "./clickhouse-seed-constants";
+import {
+  generateDatasetRunTraceId,
+  generateEvalObservationId,
+  generateEvalScoreId,
+  generateEvalTraceId,
+} from "./seed-helpers";
+import { v4 as uuidv4 } from "uuid";
+import {
+  FAILED_EVAL_TRACE_INTERVAL,
+  SEED_EVALUATOR_CONFIGS,
+} from "./postgres-seed-constants";
+import {
+  createTrace,
+  createObservation,
+  createTraceScore,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+  TraceRecordInsertType,
+} from "../../../src/server";
+
+/**
+ * Generates realistic test data for traces, observations, and scores.
+ *
+ * Use generateXxxTraces() for creating different data types:
+ * - generateDatasetTrace(): For dataset experiment runs (langfuse-prompt-experiments env)
+ * - generateEvaluationTraces(): For evaluation data (langfuse-evaluation env)
+ * - generateSyntheticTraces(): For large-scale synthetic data (default env)
+ */
+export class DataGenerator {
+  private static instance: DataGenerator;
+  private fileContent: FileContent | null = null;
+
+  static getInstance(): DataGenerator {
+    if (!DataGenerator.instance) {
+      DataGenerator.instance = new DataGenerator();
+    }
+    return DataGenerator.instance;
+  }
+
+  setFileContent(content: FileContent) {
+    this.fileContent = content;
+  }
+
+  private randomElement<T>(array: T[]): T {
+    return array[Math.floor(Math.random() * array.length)];
+  }
+
+  private randomBoolean(probability: number = 0.5): boolean {
+    return Math.random() < probability;
+  }
+
+  private randomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
+
+  /**
+   * Creates traces from dataset items for experiment runs.
+   * Use for: Dataset experiments scenarios.
+   */
+  generateDatasetTrace(
+    input: DatasetItemInput,
+    projectId: string,
+  ): TraceRecordInsertType {
+    const traceId = generateDatasetRunTraceId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    let traceInput: string;
+    let traceOutput: string;
+
+    // Transform dataset item based on type
+    if (input.datasetName === "demo-countries-dataset") {
+      const data = input.item as { input: { country: string }; output: string };
+      traceInput = `What is the capital of ${data.input.country}?`;
+      traceOutput = `The capital of ${data.input.country} is ${data.output}.`;
+    } else if (input.datasetName === "demo-english-transcription-dataset") {
+      const data = input.item as { input: { word: string }; output: string };
+      traceInput = `What is the IPA transcription of the word "${data.input.word}"?`;
+      traceOutput = `The IPA transcription of "${data.input.word}" is ${data.output}.`;
+    } else {
+      traceInput = JSON.stringify(input.item.input);
+      traceOutput = JSON.stringify(input.item.output);
+    }
+
+    return createTrace({
+      id: traceId,
+      project_id: projectId,
+      name: `dataset-run-item-${uuidv4()}`,
+      input: traceInput,
+      output: traceOutput,
+      environment: "langfuse-prompt-experiments",
+      metadata: { experimentType: "langfuse-prompt-experiments" },
+      public: false,
+      bookmarked: false,
+      session_id: null,
+      tags: [],
+    });
+  }
+
+  /**
+   * Creates observations for dataset experiment traces with variable costs/latency.
+   * Use for: Dataset experiments requiring detailed observation tracking.
+   */
+  generateDatasetObservation(
+    trace: TraceRecordInsertType,
+    input: DatasetItemInput,
+    projectId: string,
+  ): ObservationRecordInsertType {
+    const observationId = `observation-dataset-${input.datasetName}-${input.itemIndex}-${input.runNumber}-${projectId.slice(-8)}`;
+
+    // Generate variable usage and cost for each observation
+    const inputTokens = this.randomInt(30, 150);
+    const outputTokens = this.randomInt(10, 80);
+    const totalTokens = inputTokens + outputTokens;
+
+    // Cost should be fraction of cents (0.0001-0.01 range)
+    const inputCost = (inputTokens * this.randomInt(1, 5)) / 1000000; // $0.000001-0.000005 per token
+    const outputCost = (outputTokens * this.randomInt(2, 10)) / 1000000; // $0.000002-0.00001 per token
+    const totalCost = inputCost + outputCost;
+
+    return createObservation({
+      id: observationId,
+      trace_id: trace.id,
+      project_id: projectId,
+      type: "GENERATION",
+      name: `dataset-generation-${input.itemIndex}-run-${input.runNumber}`,
+      input: trace.input,
+      output: trace.output,
+      provided_model_name: "gpt-3.5-turbo",
+      model_parameters: JSON.stringify({ temperature: 0.7 }),
+      usage_details: {
+        input: inputTokens,
+        output: outputTokens,
+        total: totalTokens,
+      },
+      provided_usage_details: {
+        input: inputTokens,
+        output: outputTokens,
+        total: totalTokens,
+      },
+      cost_details: {
+        input: Math.round(inputCost * 100000) / 100000, // Round to 5 decimal places
+        output: Math.round(outputCost * 100000) / 100000,
+        total: Math.round(totalCost * 100000) / 100000,
+      },
+      provided_cost_details: {
+        input: Math.round(inputCost * 100000) / 100000,
+        output: Math.round(outputCost * 100000) / 100000,
+        total: Math.round(totalCost * 100000) / 100000,
+      },
+      total_cost: Math.round(totalCost * 100000) / 100000,
+      environment: "langfuse-prompt-experiments",
+    });
+  }
+
+  /**
+   * Creates large-scale synthetic traces for performance testing.
+   * Use for: Load testing, dashboard demos, realistic usage simulation.
+   */
+  generateSyntheticTraces(
+    projectId: string,
+    count: number,
+  ): TraceRecordInsertType[] {
+    const traces: TraceRecordInsertType[] = [];
+
+    for (let i = 0; i < count; i++) {
+      const trace = createTrace({
+        id: `trace-synthetic-${i}-${projectId.slice(-8)}`,
+        project_id: projectId,
+        name: this.randomElement(REALISTIC_TRACE_NAMES),
+        input: this.generateTraceInput(),
+        output: this.generateTraceOutput(),
+        user_id: this.randomBoolean(0.3)
+          ? `user_${this.randomInt(1, 1000)}`
+          : null,
+        session_id: this.randomBoolean(0.3)
+          ? `session_${this.randomInt(1, 100)}`
+          : undefined,
+        environment: "default",
+        metadata: { generated: "synthetic" },
+        tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
+        public: this.randomBoolean(0.8),
+        bookmarked: this.randomBoolean(0.1),
+        release: this.randomBoolean(0.4)
+          ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
+          : undefined,
+        version: this.randomBoolean(0.4)
+          ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
+          : undefined,
+      });
+
+      traces.push(trace);
+    }
+
+    return traces;
+  }
+
+  generateEvaluationObservations(
+    traces: TraceRecordInsertType[],
+    observationsPerTrace: number = 5,
+    projectId: string,
+  ): ObservationRecordInsertType[] {
+    const observations: ObservationRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      traces.forEach((trace, traceIndex) => {
+        for (let i = 0; i < observationsPerTrace; i++) {
+          const obsType = this.randomBoolean(0.47)
+            ? "GENERATION"
+            : this.randomBoolean(0.94)
+              ? "SPAN"
+              : "EVENT";
+
+          const observation: ObservationRecordInsertType = createObservation({
+            id: generateEvalObservationId(
+              evalJobConfiguration.evalTemplateId,
+              traceIndex,
+              projectId,
+            ),
+            trace_id: trace.id,
+            project_id: projectId,
+            parent_observation_id: undefined,
+            type: obsType,
+            name:
+              obsType === "GENERATION"
+                ? this.randomElement(REALISTIC_GENERATION_NAMES)
+                : obsType === "SPAN"
+                  ? this.randomElement(REALISTIC_SPAN_NAMES)
+                  : `event_${i % 10}`,
+            level: this.randomBoolean(0.85)
+              ? "DEFAULT"
+              : this.randomBoolean(0.7)
+                ? "DEBUG"
+                : this.randomBoolean(0.3)
+                  ? "ERROR"
+                  : "WARNING",
+            input:
+              obsType === "GENERATION"
+                ? this.randomBoolean(0.4)
+                  ? this.fileContent?.heavyMarkdown || "Sample input"
+                  : JSON.stringify(this.fileContent?.chatMlJson || {})
+                : undefined,
+            output:
+              obsType === "GENERATION"
+                ? this.randomBoolean(0.3)
+                  ? JSON.stringify(this.fileContent?.nestedJson || {})
+                  : JSON.stringify(this.fileContent?.chatMlJson || {})
+                : undefined,
+            provided_model_name:
+              obsType === "GENERATION"
+                ? this.randomElement(REALISTIC_MODELS)
+                : undefined,
+            model_parameters:
+              obsType === "GENERATION"
+                ? JSON.stringify({ temperature: 0.7 })
+                : undefined,
+            usage_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(20, 200),
+                    output: this.randomInt(10, 100),
+                    total: this.randomInt(30, 300),
+                  }
+                : undefined,
+            provided_usage_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(20, 200),
+                    output: this.randomInt(10, 100),
+                    total: this.randomInt(30, 300),
+                  }
+                : undefined,
+            cost_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(1, 10) / 100000,
+                    output: this.randomInt(1, 20) / 100000,
+                    total: this.randomInt(2, 30) / 100000,
+                  }
+                : undefined,
+            provided_cost_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(1, 10) / 100000,
+                    output: this.randomInt(1, 20) / 100000,
+                    total: this.randomInt(2, 30) / 100000,
+                  }
+                : undefined,
+            environment: "langfuse-evaluation",
+          });
+
+          observations.push(observation);
+        }
+      });
+    }
+
+    return observations;
+  }
+
+  /**
+   * Creates synthetic observations with automatic prompt linking (5% rate).
+   * Use for: Large datasets, hierarchical observation structures, cost variation.
+   */
+  generateSyntheticObservations(
+    traces: TraceRecordInsertType[],
+    observationsPerTrace: number = 5,
+  ): ObservationRecordInsertType[] {
+    const observations: ObservationRecordInsertType[] = [];
+
+    traces.forEach((trace, traceIndex) => {
+      for (let i = 0; i < observationsPerTrace; i++) {
+        const obsType = this.randomElement(["GENERATION", "SPAN", "EVENT"]);
+
+        const observation: ObservationRecordInsertType = createObservation({
+          id: `obs-synthetic-${traceIndex}-${i}`,
+          trace_id: trace.id,
+          project_id: trace.project_id,
+          parent_observation_id:
+            i > 0 ? `obs-synthetic-${traceIndex}-${i - 1}` : undefined,
+          type: obsType as any,
+          name:
+            obsType === "GENERATION"
+              ? this.randomElement(REALISTIC_GENERATION_NAMES)
+              : this.randomElement(REALISTIC_SPAN_NAMES),
+          input:
+            obsType === "GENERATION"
+              ? this.generateObservationInput()
+              : undefined,
+          output:
+            obsType === "GENERATION"
+              ? this.generateObservationOutput()
+              : undefined,
+          provided_model_name:
+            obsType === "GENERATION"
+              ? this.randomElement(REALISTIC_MODELS)
+              : undefined,
+          model_parameters:
+            obsType === "GENERATION"
+              ? JSON.stringify({ temperature: 0.7 })
+              : undefined,
+          usage_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(20, 200),
+                  output: this.randomInt(10, 100),
+                  total: this.randomInt(30, 300),
+                }
+              : undefined,
+          provided_usage_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(20, 200),
+                  output: this.randomInt(10, 100),
+                  total: this.randomInt(30, 300),
+                }
+              : undefined,
+          cost_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(1, 10) / 100000,
+                  output: this.randomInt(1, 20) / 100000,
+                  total: this.randomInt(2, 30) / 100000,
+                }
+              : undefined,
+          provided_cost_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(1, 10) / 100000,
+                  output: this.randomInt(1, 20) / 100000,
+                  total: this.randomInt(2, 30) / 100000,
+                }
+              : undefined,
+          level: this.randomBoolean(0.85)
+            ? "DEFAULT"
+            : this.randomBoolean(0.7)
+              ? "DEBUG"
+              : this.randomBoolean(0.5)
+                ? "WARNING"
+                : "ERROR",
+          environment: trace.environment,
+        });
+
+        observations.push(observation);
+      }
+    });
+
+    return observations;
+  }
+
+  generateSyntheticScores(
+    traces: TraceRecordInsertType[],
+    observations: ObservationRecordInsertType[],
+    scoresPerTrace: number = 2,
+  ): ScoreRecordInsertType[] {
+    const scores: ScoreRecordInsertType[] = [];
+
+    traces.forEach((trace, traceIndex) => {
+      for (let i = 0; i < scoresPerTrace; i++) {
+        const scoreType = this.randomElement([
+          "NUMERIC",
+          "CATEGORICAL",
+          "BOOLEAN",
+        ]);
+
+        let value: number | undefined;
+        let stringValue: string | undefined;
+
+        switch (scoreType) {
+          case "NUMERIC":
+            value = Math.random() * 100;
+            break;
+          case "CATEGORICAL":
+            stringValue = `category_${this.randomInt(1, 5)}`;
+            break;
+          case "BOOLEAN":
+            value = this.randomBoolean() ? 1 : 0;
+            stringValue = value === 1 ? "true" : "false";
+            break;
+        }
+
+        const score: ScoreRecordInsertType = createTraceScore({
+          id: `score-synthetic-${traceIndex}-${i}`,
+          project_id: trace.project_id,
+          trace_id: trace.id,
+          observation_id: this.randomBoolean(0.1)
+            ? this.randomElement(
+                observations.filter((o) => o.trace_id === trace.id),
+              )?.id
+            : undefined,
+          name: `metric_${this.randomInt(1, 10)}`,
+          value,
+          string_value: stringValue,
+          data_type: scoreType as any,
+          source: "API",
+          comment: "Generated score\ntest",
+          environment: trace.environment,
+        });
+
+        scores.push(score);
+      }
+    });
+
+    return scores;
+  }
+
+  /**
+   * Creates evaluation traces for testing evaluator configurations.
+   * Use for: Evaluation testing, score validation, evaluator development.
+   */
+  generateEvaluationTraces(
+    projectId: string,
+    count: number,
+  ): TraceRecordInsertType[] {
+    const traces: TraceRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      for (let i = 0; i < count; i++) {
+        const traceId = generateEvalTraceId(
+          evalJobConfiguration.evalTemplateId,
+          i,
+          projectId,
+        );
+        const trace = createTrace({
+          id: traceId,
+          session_id: null,
+          project_id: projectId,
+          name: this.randomElement(REALISTIC_TRACE_NAMES),
+          input: this.generateEvaluationInput(),
+          output: this.generateEvaluationOutput(),
+          user_id: this.randomBoolean(0.3)
+            ? `user_${this.randomInt(1, 1000)}`
+            : null,
+          environment: "langfuse-evaluation",
+          metadata: { purpose: "evaluation" },
+          tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
+          public: this.randomBoolean(0.8),
+          bookmarked: this.randomBoolean(0.1),
+          release: this.randomBoolean(0.4)
+            ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
+            : null,
+          version: this.randomBoolean(0.4)
+            ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
+            : null,
+        });
+
+        traces.push(trace);
+      }
+    }
+
+    return traces;
+  }
+
+  private generateTraceInput(): string {
+    if (!this.fileContent) return "Sample input";
+
+    // Match original logic: 30% chance of heavy markdown, otherwise chatML
+    return this.randomBoolean(0.3)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateTraceOutput(): string {
+    if (!this.fileContent) return "Sample output";
+
+    // Match original logic: 20% chance of nested JSON, otherwise chatML
+    return this.randomBoolean(0.2)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateObservationInput(): string {
+    if (!this.fileContent) return "Sample observation input";
+
+    // Match original logic: 40% chance of heavy markdown, otherwise chatML
+    return this.randomBoolean(0.4)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateObservationOutput(): string {
+    if (!this.fileContent) return "Sample observation output";
+
+    // Match original logic: 30% chance of nested JSON, otherwise chatML
+    return this.randomBoolean(0.3)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateEvaluationInput(): string {
+    if (!this.fileContent) return "Evaluation input";
+
+    return this.randomBoolean(0.3)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateEvaluationOutput(): string {
+    if (!this.fileContent) return "Evaluation output";
+
+    return this.randomBoolean(0.2)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  /**
+   * Creates exactly one score per evaluation trace with prefixed IDs.
+   * Use for: Evaluation traces that need score validation, evaluator testing.
+   */
+  generateEvaluationScores(
+    traces: TraceRecordInsertType[],
+    _observations: ObservationRecordInsertType[],
+    projectId: string,
+  ): ScoreRecordInsertType[] {
+    const scores: ScoreRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      traces.forEach((trace, traceIndex) => {
+        if (traceIndex % FAILED_EVAL_TRACE_INTERVAL === 0) return;
+        // Create exactly one score per evaluation trace with prefixed ID
+        const score: ScoreRecordInsertType = createTraceScore({
+          id: generateEvalScoreId(
+            evalJobConfiguration.evalTemplateId,
+            traceIndex,
+            projectId,
+          ), // Use prefixed ID pattern
+          project_id: projectId,
+          trace_id: trace.id,
+          observation_id: undefined, // Score is for the entire trace, not a specific observation
+          name: `evaluation_score-${evalJobConfiguration.evalTemplateId}`,
+          value: Math.random() * 100, // Random evaluation score 0-100
+          string_value: undefined,
+          data_type: "NUMERIC",
+          source: "EVAL",
+          comment: "Evaluation trace score",
+          environment: trace.environment,
+        });
+
+        scores.push(score);
+      });
+    }
+
+    return scores;
+  }
+}

packages/shared/scripts/seeder/utils/markdown.txt

@@ -0,0 +1,266 @@
+# 🚀 Advanced AI Development Guide
+
+## 📋 Table of Contents
+- [Getting Started](#getting-started)
+- [Architecture Overview](#architecture)
+- [Implementation Details](#implementation)
+- [Best Practices](#best-practices)
+
+---
+
+## 🎯 Getting Started
+
+### Prerequisites
+\`\`\`bash
+npm install @langfuse/core
+pip install langfuse
+\`\`\`
+
+### Quick Setup
+1. **Initialize your project**
+   \`\`\`typescript
+   import { Langfuse } from 'langfuse'
+   
+   const langfuse = new Langfuse({
+     secretKey: process.env.LANGFUSE_SECRET_KEY,
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
+     baseUrl: 'https://cloud.langfuse.com'
+   })
+   \`\`\`
+
+2. **Create your first trace**
+   \`\`\`python
+   from langfuse import Langfuse
+   
+   langfuse = Langfuse()
+   trace = langfuse.trace(name="chat-application")
+   \`\`\`
+
+---
+
+## 🏗️ Architecture Overview
+
+### System Components
+
+| Component | Description | Status |
+|-----------|-------------|--------|
+| **Core Engine** | Main processing unit | ✅ Active |
+| **API Gateway** | Request routing | ✅ Active |
+| **Data Store** | Persistence layer | ⚠️ Maintenance |
+| **Analytics** | Metrics & insights | 🚧 Development |
+
+### Data Flow
+
+```mermaid
+graph TD
+    A[User Request] --> B[API Gateway]
+    B --> C{Route Decision}
+    C -->|Trace| D[Trace Handler]
+    C -->|Generation| E[Generation Handler]
+    C -->|Score| F[Score Handler]
+    D --> G[Database]
+    E --> G
+    F --> G
+```
+
+---
+
+## ⚙️ Implementation Details
+
+### Trace Management
+
+> **Note:** Traces are the foundation of observability in LLM applications.
+
+#### Creating Traces
+\`\`\`typescript
+// Basic trace creation
+const trace = langfuse.trace({
+  name: "user-query-processing",
+  userId: "user-123",
+  sessionId: "session-456",
+  metadata: {
+    environment: "production",
+    version: "2.1.0"
+  }
+})
+
+// Nested observations
+const span = trace.span({
+  name: "document-retrieval",
+  input: { query: "What is machine learning?" },
+  metadata: { vectorStore: "pinecone" }
+})
+
+const generation = span.generation({
+  name: "answer-generation",
+  model: "gpt-4",
+  input: retrievedDocs,
+  output: generatedAnswer,
+  usage: {
+    promptTokens: 1250,
+    completionTokens: 420,
+    totalTokens: 1670
+  }
+})
+\`\`\`
+
+### Advanced Features
+
+#### 🔄 Async Processing
+\`\`\`python
+import asyncio
+from langfuse import Langfuse
+
+async def process_batch():
+    langfuse = Langfuse()
+    
+    tasks = []
+    for item in batch_items:
+        task = asyncio.create_task(
+            process_item_with_tracing(langfuse, item)
+        )
+        tasks.append(task)
+    
+    results = await asyncio.gather(*tasks)
+    return results
+\`\`\`
+
+#### 🎯 Custom Scoring
+\`\`\`typescript
+// Automated scoring
+trace.score({
+  name: "relevance",
+  value: 0.95,
+  comment: "Highly relevant response"
+})
+
+// Human feedback scoring
+trace.score({
+  name: "user-satisfaction", 
+  value: 1,
+  source: "user-feedback",
+  comment: "User rated 5/5 stars"
+})
+\`\`\`
+
+---
+
+## 🎨 Best Practices
+
+### 📊 Monitoring & Observability
+
+#### Key Metrics to Track
+- **Latency**: P50, P95, P99 response times
+- **Token Usage**: Cost optimization
+- **Error Rates**: System reliability
+- **User Satisfaction**: Quality metrics
+
+#### Dashboard Setup
+\`\`\`yaml
+# monitoring-config.yml
+dashboards:
+  - name: "LLM Performance"
+    panels:
+      - type: "time-series"
+        title: "Response Latency"
+        query: "avg(response_time) by (model)"
+      - type: "stat"
+        title: "Daily Token Usage" 
+        query: "sum(tokens_used)"
+      - type: "table"
+        title: "Top Errors"
+        query: "topk(10, count by (error_type))"
+\`\`\`
+
+### 🔐 Security Considerations
+
+> ⚠️ **Important**: Never log sensitive user data in traces
+
+#### Data Sanitization
+\`\`\`python
+def sanitize_input(data):
+    """Remove PII from trace data"""
+    sanitized = data.copy()
+    
+    # Remove email addresses
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
+                      '[EMAIL_REDACTED]', sanitized)
+    
+    # Remove phone numbers
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
+                      '[PHONE_REDACTED]', sanitized)
+    
+    return sanitized
+\`\`\`
+
+### 🚀 Performance Optimization
+
+#### Batch Processing
+\`\`\`typescript
+// Efficient batch uploads
+const batchSize = 100
+const traces = []
+
+for (let i = 0; i < data.length; i += batchSize) {
+  const batch = data.slice(i, i + batchSize)
+  const processedBatch = await Promise.all(
+    batch.map(item => processWithLangfuse(item))
+  )
+  traces.push(...processedBatch)
+}
+
+// Flush all traces at once
+await langfuse.flushAsync()
+\`\`\`
+
+---
+
+## 📚 Advanced Examples
+
+### Multi-Agent System Tracing
+\`\`\`python
+class MultiAgentTracer:
+    def __init__(self):
+        self.langfuse = Langfuse()
+    
+    async def orchestrate_agents(self, task):
+        # Main orchestration trace
+        main_trace = self.langfuse.trace(
+            name="multi-agent-orchestration",
+            input={"task": task}
+        )
+        
+        # Agent 1: Research
+        research_span = main_trace.span(name="research-agent")
+        research_result = await self.research_agent.process(task)
+        research_span.end(output=research_result)
+        
+        # Agent 2: Analysis  
+        analysis_span = main_trace.span(name="analysis-agent")
+        analysis_result = await self.analysis_agent.process(research_result)
+        analysis_span.end(output=analysis_result)
+        
+        # Agent 3: Synthesis
+        synthesis_span = main_trace.span(name="synthesis-agent")
+        final_result = await self.synthesis_agent.process(analysis_result)
+        synthesis_span.end(output=final_result)
+        
+        main_trace.end(output=final_result)
+        return final_result
+\`\`\`
+
+---
+
+## 🎉 Conclusion
+
+With proper implementation of Langfuse tracing, you can:
+
+- ✅ **Monitor** your LLM applications in real-time
+- ✅ **Debug** issues with detailed trace information  
+- ✅ **Optimize** performance and costs
+- ✅ **Scale** your applications with confidence
+
+### Next Steps
+1. Review the [official documentation](https://langfuse.com/docs)
+2. Join our [Discord community](https://discord.gg/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/scripts/seeder/utils/postgres-seed-constants.ts

@@ -0,0 +1,432 @@
+// Datasets
+const SEED_DATASET_ITEMS_COUNTRIES = [
+  { input: { country: "France" }, output: "Paris" },
+  { input: { country: "Germany" }, output: "Berlin" },
+  { input: { country: "Italy" }, output: "Rome" },
+  { input: { country: "Spain" }, output: "Madrid" },
+  { input: { country: "United Kingdom" }, output: "London" },
+  { input: { country: "Japan" }, output: "Tokyo" },
+  { input: { country: "China" }, output: "Beijing" },
+  { input: { country: "India" }, output: "New Delhi" },
+  { input: { country: "Brazil" }, output: "Brasília" },
+  { input: { country: "Canada" }, output: "Ottawa" },
+  { input: { country: "Australia" }, output: "Canberra" },
+  { input: { country: "South Africa" }, output: "Pretoria" },
+  { input: { country: "Mexico" }, output: "Mexico City" },
+  { input: { country: "Russia" }, output: "Moscow" },
+  { input: { country: "Egypt" }, output: "Cairo" },
+  { input: { country: "Turkey" }, output: "Ankara" },
+  { input: { country: "Indonesia" }, output: "Jakarta" },
+  { input: { country: "South Korea" }, output: "Seoul" },
+  { input: { country: "Saudi Arabia" }, output: "Riyadh" },
+  { input: { country: "Argentina" }, output: "Buenos Aires" },
+  { input: { country: "Nigeria" }, output: "Abuja" },
+  { input: { country: "Pakistan" }, output: "Islamabad" },
+  { input: { country: "Thailand" }, output: "Bangkok" },
+  { input: { country: "Vietnam" }, output: "Hanoi" },
+  { input: { country: "Malaysia" }, output: "Kuala Lumpur" },
+  { input: { country: "Philippines" }, output: "Manila" },
+  { input: { country: "Singapore" }, output: "Singapore" },
+  { input: { country: "New Zealand" }, output: "Wellington" },
+  { input: { country: "Sweden" }, output: "Stockholm" },
+  { input: { country: "Norway" }, output: "Oslo" },
+  { input: { country: "Denmark" }, output: "Copenhagen" },
+  { input: { country: "Finland" }, output: "Helsinki" },
+  { input: { country: "Netherlands" }, output: "Amsterdam" },
+  { input: { country: "Belgium" }, output: "Brussels" },
+  { input: { country: "Switzerland" }, output: "Bern" },
+  { input: { country: "Austria" }, output: "Vienna" },
+  { input: { country: "Portugal" }, output: "Lisbon" },
+  { input: { country: "Greece" }, output: "Athens" },
+  { input: { country: "Poland" }, output: "Warsaw" },
+  { input: { country: "Ukraine" }, output: "Kyiv" },
+  { input: { country: "Romania" }, output: "Bucharest" },
+  { input: { country: "Hungary" }, output: "Budapest" },
+  { input: { country: "Czech Republic" }, output: "Prague" },
+  { input: { country: "Slovakia" }, output: "Bratislava" },
+  { input: { country: "Croatia" }, output: "Zagreb" },
+  { input: { country: "Serbia" }, output: "Belgrade" },
+  { input: { country: "Bulgaria" }, output: "Sofia" },
+  { input: { country: "Ireland" }, output: "Dublin" },
+  { input: { country: "Iceland" }, output: "Reykjavik" },
+  { input: { country: "Estonia" }, output: "Tallinn" },
+  { input: { country: "Latvia" }, output: "Riga" },
+  { input: { country: "Lithuania" }, output: "Vilnius" },
+];
+
+const SEED_DATASET_ITEMS_IPA = [
+  { input: { word: "the" }, output: "/ðə/" },
+  { input: { word: "be" }, output: "/bi/" },
+  { input: { word: "to" }, output: "/tu/" },
+  { input: { word: "of" }, output: "/əv/" },
+  { input: { word: "and" }, output: "/ænd/" },
+  { input: { word: "a" }, output: "/ə/" },
+  { input: { word: "in" }, output: "/ɪn/" },
+  { input: { word: "that" }, output: "/ðæt/" },
+  { input: { word: "have" }, output: "/hæv/" },
+  { input: { word: "I" }, output: "/aɪ/" },
+  { input: { word: "it" }, output: "/ɪt/" },
+  { input: { word: "for" }, output: "/fɔr/" },
+  { input: { word: "not" }, output: "/nɑt/" },
+  { input: { word: "on" }, output: "/ɑn/" },
+  { input: { word: "with" }, output: "/wɪð/" },
+  { input: { word: "he" }, output: "/hi/" },
+  { input: { word: "as" }, output: "/æz/" },
+  { input: { word: "you" }, output: "/ju/" },
+  { input: { word: "do" }, output: "/du/" },
+  { input: { word: "at" }, output: "/æt/" },
+  { input: { word: "this" }, output: "/ðɪs/" },
+  { input: { word: "but" }, output: "/bʌt/" },
+  { input: { word: "his" }, output: "/hɪz/" },
+  { input: { word: "by" }, output: "/baɪ/" },
+  { input: { word: "from" }, output: "/frʌm/" },
+  { input: { word: "they" }, output: "/ðeɪ/" },
+  { input: { word: "we" }, output: "/wi/" },
+  { input: { word: "say" }, output: "/seɪ/" },
+  { input: { word: "her" }, output: "/hər/" },
+  { input: { word: "she" }, output: "/ʃi/" },
+  { input: { word: "or" }, output: "/ɔr/" },
+  { input: { word: "an" }, output: "/æn/" },
+  { input: { word: "will" }, output: "/wɪl/" },
+  { input: { word: "my" }, output: "/maɪ/" },
+  { input: { word: "one" }, output: "/wʌn/" },
+  { input: { word: "all" }, output: "/ɔl/" },
+  { input: { word: "would" }, output: "/wʊd/" },
+  { input: { word: "there" }, output: "/ðɛr/" },
+  { input: { word: "their" }, output: "/ðɛr/" },
+  { input: { word: "what" }, output: "/wʌt/" },
+  { input: { word: "so" }, output: "/soʊ/" },
+  { input: { word: "up" }, output: "/ʌp/" },
+  { input: { word: "out" }, output: "/aʊt/" },
+  { input: { word: "if" }, output: "/ɪf/" },
+  { input: { word: "about" }, output: "/əˈbaʊt/" },
+  { input: { word: "who" }, output: "/hu/" },
+  { input: { word: "get" }, output: "/gɛt/" },
+  { input: { word: "which" }, output: "/wɪtʃ/" },
+  { input: { word: "go" }, output: "/goʊ/" },
+  { input: { word: "me" }, output: "/mi/" },
+  { input: { word: "when" }, output: "/wɛn/" },
+  { input: { word: "make" }, output: "/meɪk/" },
+  { input: { word: "can" }, output: "/kæn/" },
+  { input: { word: "like" }, output: "/laɪk/" },
+  { input: { word: "time" }, output: "/taɪm/" },
+  { input: { word: "no" }, output: "/noʊ/" },
+  { input: { word: "just" }, output: "/dʒʌst/" },
+  { input: { word: "him" }, output: "/hɪm/" },
+  { input: { word: "know" }, output: "/noʊ/" },
+  { input: { word: "take" }, output: "/teɪk/" },
+  { input: { word: "person" }, output: "/ˈpərsən/" },
+  { input: { word: "into" }, output: "/ˈɪntu/" },
+  { input: { word: "year" }, output: "/jɪr/" },
+  { input: { word: "your" }, output: "/jʊər/" },
+  { input: { word: "good" }, output: "/gʊd/" },
+  { input: { word: "some" }, output: "/sʌm/" },
+  { input: { word: "could" }, output: "/kʊd/" },
+  { input: { word: "them" }, output: "/ðɛm/" },
+  { input: { word: "see" }, output: "/si/" },
+  { input: { word: "other" }, output: "/ˈʌðər/" },
+  { input: { word: "than" }, output: "/ðæn/" },
+  { input: { word: "then" }, output: "/ðɛn/" },
+  { input: { word: "now" }, output: "/naʊ/" },
+  { input: { word: "look" }, output: "/lʊk/" },
+  { input: { word: "only" }, output: "/ˈoʊnli/" },
+  { input: { word: "come" }, output: "/kʌm/" },
+  { input: { word: "its" }, output: "/ɪts/" },
+  { input: { word: "over" }, output: "/ˈoʊvər/" },
+  { input: { word: "think" }, output: "/θɪŋk/" },
+  { input: { word: "also" }, output: "/ˈɔlsoʊ/" },
+  { input: { word: "back" }, output: "/bæk/" },
+  { input: { word: "after" }, output: "/ˈæftər/" },
+  { input: { word: "use" }, output: "/juz/" },
+  { input: { word: "two" }, output: "/tu/" },
+  { input: { word: "how" }, output: "/haʊ/" },
+  { input: { word: "our" }, output: "/aʊər/" },
+  { input: { word: "work" }, output: "/wɜrk/" },
+  { input: { word: "first" }, output: "/fɜrst/" },
+  { input: { word: "well" }, output: "/wɛl/" },
+  { input: { word: "way" }, output: "/weɪ/" },
+  { input: { word: "even" }, output: "/ˈivɪn/" },
+  { input: { word: "new" }, output: "/nu/" },
+  { input: { word: "want" }, output: "/wɑnt/" },
+  { input: { word: "because" }, output: "/bɪˈkɔz/" },
+  { input: { word: "any" }, output: "/ˈɛni/" },
+  { input: { word: "these" }, output: "/ðiz/" },
+  { input: { word: "give" }, output: "/gɪv/" },
+  { input: { word: "day" }, output: "/deɪ/" },
+  { input: { word: "most" }, output: "/moʊst/" },
+  { input: { word: "us" }, output: "/ʌs/" },
+];
+
+export const SEED_DATASETS = [
+  {
+    name: "demo-countries-dataset",
+    description: "Dataset for countries",
+    metadata: {
+      key: "value",
+    },
+    items: SEED_DATASET_ITEMS_COUNTRIES,
+  },
+  {
+    name: "demo-english-transcription-dataset",
+    description:
+      "Dataset for english transcription, where words are represented in their international phonetic alphabet (IPA)",
+    metadata: {
+      key: "value",
+    },
+    items: SEED_DATASET_ITEMS_IPA,
+  },
+];
+
+// Prompts
+export const SEED_TEXT_PROMPTS = [
+  {
+    id: `prompt-parent`,
+    createdBy: "user-1",
+    prompt:
+      'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format. Refer to the respective links of the documentation.\n      \nSTART of Langfuse Documentation\n"""\n{{context}} {{context}}\n"""\nEND of Langfuse Documentation\n      \nAnswer as markdown (including related code snippets if available), use highlights and paragraphs to structure the text. Use emojis in your answers. Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers. Only use information that is available in the context, do not make up any code that is not in the context. If you are unsure and the answer is not explicitly written in the documentation, say "Sorry, I don\'t know how to help with that." If the user is having problems using Langfuse, tell her to reach out to the founders directly via the chat widget. Make it crisp.\n\n@@@langfusePrompt:name=child-prompt|label=production@@@',
+    name: "parent-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-child`,
+    createdBy: "user-1",
+    prompt: `Please follow these guidelines:
+- Refer to the respective links of the documentation
+- Be kind.
+- Include emojis where it makes sense.
+- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.
+- Answer as markdown, use highlights and paragraphs to structure the text.
+- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.`,
+    name: "child-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-123`,
+    createdBy: "user-1",
+    prompt: "Prompt 1 content",
+    name: "prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-456`,
+    createdBy: "user-1",
+    prompt: "Prompt 2 content",
+    name: "prompt-2",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-789`,
+    createdBy: "API",
+    prompt: "Prompt 3 content",
+    name: "prompt-3-by-api",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: "Prompt 4 content",
+    name: "prompt-4",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `countries-experiment-prompt`,
+    createdBy: "user-1",
+    prompt: "What is the capital of {{country}}?",
+    name: "countries-experiment-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-customer-prompt-1`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 1 content",
+    name: "folder/customer/prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-customer-prompt-2`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 2 content",
+    name: "folder/customer/prompt-2",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-prompt-1`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 1 content",
+    name: "folder/prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
+
+export const SEED_CHAT_ML_PROMPTS = [
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: [
+      {
+        role: "system",
+        content:
+          'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format.\n\nPlease follow these guidelines:\n- Refer to the respective links of the documentation and select quality examples\n- Be kind.\n- Include emojis where it makes sense.\n- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.\n- Answer as markdown, use highlights and paragraphs to structure the text.\n- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.\n- Only use information that is available in the context, do not make up any code that is not in the context.\n- Always put an empji at the end of the message.',
+      },
+      {
+        role: "assistant",
+        content:
+          "All right, what is the documentation that I am meant to exclusively use to answer the question?",
+      },
+      {
+        role: "user",
+        content: "<documentation>\n```\n{{context}}\n```\n</documentation>",
+      },
+      {
+        role: "assistant",
+        content:
+          "Answering in next message based on your instructions only. What is the question?",
+      },
+      {
+        role: "user",
+        content: "{{question}}",
+      },
+    ],
+    name: "prompt-chat-ml",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `prompt-chat-placeholder`,
+    createdBy: "user-1",
+    prompt: [
+      {
+        role: "system",
+        content:
+          'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format.\n\nPlease follow these guidelines:\n- Refer to the respective links of the documentation and select quality examples\n- Be kind.\n- Include emojis where it makes sense.\n- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.\n- Answer as markdown, use highlights and paragraphs to structure the text.\n- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.\n- Only use information that is available in the context, do not make up any code that is not in the context.\n- Always put an empji at the end of the message.',
+      },
+      {
+        type: "placeholder",
+        name: "message_history",
+      },
+      {
+        role: "assistant",
+        content:
+          "Answering in next message based on your instructions only. What is the question?",
+      },
+      {
+        role: "user",
+        content: "{{question}}",
+      },
+    ],
+    name: "prompt-chat-ml-with-placeholder",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
+
+export const SEED_PROMPT_VERSIONS = [
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 1 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+    },
+    version: 1,
+  },
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 2 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+      topP: 0.9,
+    },
+    version: 2,
+    labels: ["production"],
+  },
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 3 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+      topP: 0.9,
+      frequencyPenalty: 0.5,
+    },
+    version: 3,
+    labels: ["production", "latest"],
+  },
+];
+
+// evaluators
+export const SEED_EVALUATOR_TEMPLATES = [
+  {
+    id: "toxicity-template",
+    name: "toxicity-template",
+    version: 1,
+    prompt:
+      "Please evaluate the toxicity of the following text {{input}} {{output}}",
+    model: "gpt-3.5-turbo",
+    vars: ["input", "output"],
+    provider: "openai",
+    outputSchema: {
+      score: "provide a score between 0 and 1",
+      reasoning: "one sentence reasoning for the score",
+    },
+    modelParams: {
+      temperature: 0.7,
+      outputTokenLimit: 100,
+      topP: 0.9,
+    },
+  },
+];
+
+export const SEED_EVALUATOR_CONFIGS = [
+  {
+    id: "toxicity-job",
+    evalTemplateId: "toxicity-template",
+    jobType: "EVAL",
+    status: "ACTIVE",
+    scoreName: "toxicity",
+    filter: [
+      {
+        type: "string",
+        value: "user",
+        column: "User ID",
+        operator: "contains",
+      },
+    ],
+    variableMapping: [
+      {
+        langfuseObject: "trace",
+        selectedColumnId: "input",
+        templateVariable: "input",
+      },
+      {
+        langfuseObject: "trace",
+        selectedColumnId: "metadata",
+        templateVariable: "output",
+      },
+    ],
+    targetObject: "trace",
+    sampling: 1,
+    delay: 5_000,
+  },
+];
+
+export const EVAL_TRACE_COUNT = 100;
+export const FAILED_EVAL_TRACE_INTERVAL = 10;

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -0,0 +1,32 @@
+export const generateDatasetRunTraceId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `trace-dataset-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateEvalTraceId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `trace-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};
+
+export const generateEvalObservationId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `observation-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};
+
+export const generateEvalScoreId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `score-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -0,0 +1,328 @@
+import { FileContent, SeederOptions } from "./types";
+import { DataGenerator } from "./data-generators";
+import { ClickHouseQueryBuilder } from "./clickhouse-builder";
+import { EVAL_TRACE_COUNT, SEED_DATASETS } from "./postgres-seed-constants";
+import {
+  clickhouseClient,
+  logger,
+  ObservationRecordInsertType,
+  TraceRecordInsertType,
+} from "../../../src/server";
+import path from "path";
+import { readFileSync } from "fs";
+
+/**
+ * Orchestrates seeding operations across ClickHouse and PostgreSQL.
+ *
+ * Use createXxxData() for specific data types:
+ * - createDatasetExperimentData(): Dataset runs in langfuse-prompt-experiments env
+ * - createEvaluationData(): Evaluation data in langfuse-evaluation env
+ * - createSyntheticData(): Large synthetic data in default env
+ * - executeFullSeed(): All data types together
+ */
+export class SeederOrchestrator {
+  private dataGenerator: DataGenerator;
+  private queryBuilder: ClickHouseQueryBuilder;
+  private fileContent: FileContent | null = null;
+
+  constructor() {
+    this.dataGenerator = DataGenerator.getInstance();
+    this.queryBuilder = new ClickHouseQueryBuilder();
+    this.loadFileContent();
+  }
+
+  private loadFileContent() {
+    try {
+      const nestedJsonPath = path.join(__dirname, "./nested_json.json");
+      const heavyMarkdownPath = path.join(__dirname, "./markdown.txt");
+      const chatMlJsonPath = path.join(__dirname, "./chat_ml_json.json");
+
+      const nestedJsonContent = JSON.parse(
+        readFileSync(nestedJsonPath, "utf-8"),
+      );
+      const heavyMarkdownContent = readFileSync(heavyMarkdownPath, "utf-8");
+      const chatMlJsonContent = JSON.parse(
+        readFileSync(chatMlJsonPath, "utf-8"),
+      );
+
+      // Truncate large content for reasonable test data size
+      const truncatedNestedJson = {
+        ...nestedJsonContent,
+        products: nestedJsonContent.products?.slice(0, 3) || [],
+      };
+
+      const truncatedChatMlJson = {
+        ...chatMlJsonContent,
+        messages: chatMlJsonContent.messages?.slice(0, 4) || [],
+      };
+
+      this.fileContent = {
+        nestedJson: truncatedNestedJson,
+        heavyMarkdown: heavyMarkdownContent,
+        chatMlJson: truncatedChatMlJson,
+      };
+
+      this.dataGenerator.setFileContent(this.fileContent);
+    } catch (error) {
+      logger.warn(
+        "Could not load file content for seeding, using fallback data",
+        error,
+      );
+    }
+  }
+
+  /**
+   * Creates dataset experiment data for A/B testing and prompt comparisons.
+   * Use for: Experiment tracking, dataset-based evaluations, prompt testing.
+   */
+  async createDatasetExperimentData(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info(
+      `Creating dataset experiment data for ${projectIds.length} projects.`,
+    );
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing project ${projectId}`);
+
+      const numberOfRuns = opts.numberOfRuns || 1;
+
+      for (let runNumber = 0; runNumber < numberOfRuns; runNumber++) {
+        logger.info(
+          `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
+        );
+
+        const traces: TraceRecordInsertType[] = [];
+        const observations: ObservationRecordInsertType[] = [];
+
+        for (const seedDataset of SEED_DATASETS) {
+          for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // Generate trace data
+            const trace = this.dataGenerator.generateDatasetTrace(
+              {
+                datasetName: seedDataset.name,
+                itemIndex,
+                item: datasetItem,
+                runNumber,
+              },
+              projectId,
+            );
+
+            // Generate observation data
+            const observation = this.dataGenerator.generateDatasetObservation(
+              trace,
+              {
+                datasetName: seedDataset.name,
+                itemIndex,
+                item: datasetItem,
+                runNumber,
+              },
+              projectId,
+            );
+
+            traces.push(trace);
+            observations.push(observation);
+          }
+        }
+
+        try {
+          await this.queryBuilder.executeTracesInsert(traces);
+          await this.queryBuilder.executeObservationsInsert(observations);
+        } catch (error) {
+          logger.error(`✗ Insert failed:`, error);
+          throw error;
+        }
+      }
+    }
+  }
+
+  /**
+   * Creates evaluation data for testing evaluator configurations.
+   * Use for: Evaluator development, score validation, evaluation testing.
+   */
+  async createEvaluationData(projectIds: string[]): Promise<void> {
+    logger.info(`Creating evaluation data for ${projectIds.length} projects.`);
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing evaluation data for project ${projectId}`);
+
+      const evalTracesPerProject = EVAL_TRACE_COUNT;
+      const evalObservationsPerTrace = 10;
+
+      // Generate evaluation traces
+      const traces = this.dataGenerator.generateEvaluationTraces(
+        projectId,
+        evalTracesPerProject,
+      );
+
+      // Generate evaluation observations
+      const observations = this.dataGenerator.generateEvaluationObservations(
+        traces,
+        evalObservationsPerTrace,
+        projectId,
+      );
+
+      // Generate scores - exactly one score per evaluation trace
+      const scores = this.dataGenerator.generateEvaluationScores(
+        traces,
+        observations,
+        projectId,
+      );
+
+      await this.queryBuilder.executeTracesInsert(traces);
+      await this.queryBuilder.executeObservationsInsert(observations);
+      await this.queryBuilder.executeScoresInsert(scores);
+    }
+  }
+
+  /**
+   * Creates large-scale synthetic data for performance testing and demos.
+   * Use for: Load testing, dashboard demos, realistic usage simulation.
+   */
+  async createSyntheticData(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info(`Creating synthetic data for ${projectIds.length} projects.`);
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing synthetic data for project ${projectId}`);
+
+      const observationsPerTrace = 15;
+      const tracesPerProject = Math.floor(
+        (opts.totalObservations || 1000) / observationsPerTrace,
+      );
+      const scoresPerTrace = 10;
+
+      // For large datasets, use bulk generation for better performance
+      if (tracesPerProject > 100) {
+        logger.info(`Using bulk generation for ${tracesPerProject} traces`);
+
+        const traceQuery = this.queryBuilder.buildBulkTracesInsert(
+          projectId,
+          tracesPerProject,
+          "default",
+          this.fileContent || undefined,
+          { numberOfDays: opts.numberOfDays },
+        );
+        const observationQuery = this.queryBuilder.buildBulkObservationsInsert(
+          projectId,
+          tracesPerProject,
+          observationsPerTrace,
+          "default",
+          this.fileContent || undefined,
+          { numberOfDays: opts.numberOfDays },
+        );
+        const scoreQuery = this.queryBuilder.buildBulkScoresInsert(
+          projectId,
+          tracesPerProject,
+          scoresPerTrace,
+          "default",
+          { numberOfDays: opts.numberOfDays },
+        );
+
+        await this.executeQuery(traceQuery);
+        await this.executeQuery(observationQuery);
+        await this.executeQuery(scoreQuery);
+      } else {
+        // Use detailed generation for smaller datasets
+        const traces = this.dataGenerator.generateSyntheticTraces(
+          projectId,
+          tracesPerProject,
+        );
+        const observations = this.dataGenerator.generateSyntheticObservations(
+          traces,
+          observationsPerTrace,
+        );
+        const scores = this.dataGenerator.generateSyntheticScores(
+          traces,
+          observations,
+          scoresPerTrace,
+        );
+
+        await this.queryBuilder.executeTracesInsert(traces);
+        await this.queryBuilder.executeObservationsInsert(observations);
+        await this.queryBuilder.executeScoresInsert(scores);
+      }
+    }
+  }
+
+  /**
+   * Executes complete seeding: datasets + evaluation + synthetic data.
+   * Use for: Full system setup, comprehensive testing, complete data reset.
+   */
+  async executeFullSeed(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info("Starting full seed process");
+
+    try {
+      // Create dataset experiment data
+      await this.createDatasetExperimentData(projectIds, opts);
+
+      // Create evaluation data
+      await this.createEvaluationData(projectIds);
+
+      // Create synthetic data
+      await this.createSyntheticData(projectIds, opts);
+
+      // Log completion statistics (commented out to reduce terminal noise)
+      await this.logStatistics();
+
+      logger.info("Full seed process completed successfully");
+    } catch (error) {
+      logger.error("Seed process failed:", error);
+      throw error;
+    }
+  }
+
+  private async executeQuery(query: string): Promise<void> {
+    try {
+      await clickhouseClient().command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        },
+      });
+    } catch (error) {
+      logger.error("Query execution failed:", error);
+      logger.error("Failed query:", query);
+      throw error;
+    }
+  }
+
+  private async logStatistics(): Promise<void> {
+    const tables = ["traces", "scores", "observations"];
+
+    for (const table of tables) {
+      try {
+        const query = `
+          SELECT
+            project_id,
+            count() AS per_project_count,
+            bar(per_project_count, 0, (
+              SELECT count(*)
+              FROM ${table}
+            ), 50) AS bar_representation
+          FROM ${table}
+          GROUP BY project_id
+          ORDER BY count() desc
+        `;
+
+        const result = await clickhouseClient().query({
+          query,
+          format: "TabSeparated",
+        });
+
+        logger.info(
+          `${table.charAt(0).toUpperCase() + table.slice(1)} per Project: \n` +
+            (await result.text()),
+        );
+      } catch (error) {
+        logger.warn(`Could not log statistics for ${table}:`, error);
+      }
+    }
+  }
+}

packages/shared/scripts/seeder/utils/types.ts

@@ -0,0 +1,18 @@
+export interface SeederOptions {
+  numberOfDays: number;
+  numberOfRuns?: number;
+  totalObservations?: number;
+}
+
+export interface DatasetItemInput {
+  datasetName: string;
+  itemIndex: number;
+  item: any;
+  runNumber?: number;
+}
+
+export interface FileContent {
+  nestedJson: any;
+  heavyMarkdown: string;
+  chatMlJson: any;
+}

packages/shared/src/db.ts

@@ -88,7 +88,7 @@ declare const globalThis: {
   kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
 } & typeof global;
 
-if (process.env.NODE_ENV === "development") {
+if (process.env.NODE_ENV === "development") { // eslint-disable-line turbo/no-undeclared-env-vars
   globalThis.prismaGlobal ??= createPrismaInstance(); // regular instantiation
   globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
     kyselyExtension({

packages/shared/src/domain/table-view-presets.ts

@@ -3,11 +3,11 @@ import { orderBy } from "../interfaces/orderBy";
 import z from "zod/v4";
 
 export enum TableViewPresetTableName {
-  Traces = "traces",
-  Observations = "observations",
-  Scores = "scores",
-  Sessions = "sessions",
-  Datasets = "datasets",
+  Traces = "traces", // eslint-disable-line no-unused-vars
+  Observations = "observations", // eslint-disable-line no-unused-vars
+  Scores = "scores", // eslint-disable-line no-unused-vars
+  Sessions = "sessions", // eslint-disable-line no-unused-vars
+  Datasets = "datasets", // eslint-disable-line no-unused-vars
 }
 
 const TableViewPresetDomainSchema = z.object({

packages/shared/src/env.ts

@@ -45,10 +45,7 @@ const EnvSchema = z.object({
     .number()
     .nonnegative()
     .default(15_000),
-  LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce
-    .number()
-    .positive()
-    .default(1),
+  LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -91,6 +88,7 @@ const EnvSchema = z.object({
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
+  LANGFUSE_S3_LIST_MAX_KEYS: z.coerce.number().positive().default(200),
   LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
@@ -107,9 +105,11 @@ const EnvSchema = z.object({
     .number()
     .default(80e6), // 80MB
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
+  LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
+  LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
 });
 
 export const env: z.infer<typeof EnvSchema> =
-  process.env.DOCKER_BUILD === "1"
+  process.env.DOCKER_BUILD === "1" // eslint-disable-line turbo/no-undeclared-env-vars
     ? (process.env as any)
     : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/errors/index.ts

@@ -7,3 +7,4 @@ export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
 export { InternalServerError } from "./InternalServerError";
 export { LangfuseConflictError } from "./ConflictError";
+export { QUEUE_ERROR_MESSAGES } from "./utils/constants";

packages/shared/src/errors/utils/constants.ts

@@ -0,0 +1,10 @@
+export const QUEUE_ERROR_MESSAGES = {
+  API_KEY_ERROR: "API key for provider",
+  NO_DEFAULT_MODEL_ERROR: "No default model or custom model found for project",
+  MAPPED_DATA_ERROR:
+    "Please ensure the mapped data exists and consider extending the job delay.",
+  INVALID_JSON_ERROR: "is not valid JSON",
+  TOO_LOW_MAX_TOKENS_ERROR: "Error: Unterminated string in JSON at position",
+  OUTPUT_TOKENS_TOO_LONG_ERROR:
+    "Could not parse response content as the length limit was reached",
+};

packages/shared/src/features/batchExport/types.ts

@@ -5,18 +5,19 @@ import { BatchExport } from "@prisma/client";
 import { singleFilter } from "../../interfaces/filters";
 import { orderBy } from "../../interfaces/orderBy";
 import { BatchTableNames } from "../../interfaces/tableNames";
+import { TracingSearchType } from "../../interfaces/search";
 
 export enum BatchExportStatus {
-  QUEUED = "QUEUED",
-  PROCESSING = "PROCESSING",
-  COMPLETED = "COMPLETED",
-  FAILED = "FAILED",
+  QUEUED = "QUEUED", // eslint-disable-line no-unused-vars
+  PROCESSING = "PROCESSING", // eslint-disable-line no-unused-vars
+  COMPLETED = "COMPLETED", // eslint-disable-line no-unused-vars
+  FAILED = "FAILED", // eslint-disable-line no-unused-vars
 }
 
 export enum BatchExportFileFormat {
-  JSON = "JSON",
-  CSV = "CSV",
-  JSONL = "JSONL",
+  JSON = "JSON", // eslint-disable-line no-unused-vars
+  CSV = "CSV", // eslint-disable-line no-unused-vars
+  JSONL = "JSONL", // eslint-disable-line no-unused-vars
 }
 
 // Use shared BatchTableNames enum for consistency across batch operations
@@ -43,6 +44,8 @@ export const exportOptions: Record<
 export const BatchExportQuerySchema = z.object({
   tableName: z.enum(BatchTableNames),
   filter: z.array(singleFilter).nullable(),
+  searchQuery: z.string().optional(),
+  searchType: z.array(TracingSearchType).optional(),
   orderBy,
   limit: z.number().optional(),
   page: z.number().optional(),

packages/shared/src/features/evals/utilities.ts

@@ -1,4 +1,5 @@
 import z from "zod/v4";
+import { JSONPath } from "jsonpath-plus";
 import { variableMapping } from "./types";
 
 /**
@@ -27,40 +28,44 @@ export const parseUnknownToString = (value: unknown): string => {
 };
 
 function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
-  // Front-end friendly JSON path extraction
-  const parsedJson =
-    typeof selectedColumn === "string"
-      ? JSON.parse(selectedColumn)
-      : selectedColumn;
+  const result = JSONPath({
+    path: jsonSelector,
+    json:
+      typeof selectedColumn === "string"
+        ? JSON.parse(selectedColumn)
+        : selectedColumn,
+  });
 
-  // Simple path extraction (could use a library)
-  return jsonSelector
-    .split(".")
-    .reduce((o, key) => (o as any)?.[key], parsedJson);
+  return result.length > 0 ? result[0] : undefined;
 }
 
 export function extractValueFromObject(
   obj: Record<string, unknown>,
   mapping: z.infer<typeof variableMapping>,
-  parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown,
-): string {
+  parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown, // eslint-disable-line no-unused-vars
+): { value: string; error: Error | null } {
   const selectedColumn = obj[mapping.selectedColumnId];
   const jsonParser = parseJson || parseJsonDefault;
 
   let jsonSelectedColumn;
+  let error: Error | null = null;
+
   if (mapping.jsonSelector && selectedColumn) {
     try {
       jsonSelectedColumn = jsonParser(selectedColumn, mapping.jsonSelector);
-    } catch (error) {
-      console.error(
-        `Error parsing JSON selector: ${mapping.jsonSelector}`,
-        error,
-      );
-      jsonSelectedColumn = selectedColumn;
+    } catch (err) {
+      error =
+        err instanceof Error
+          ? err
+          : new Error("There was an unknown error parsing the JSON");
+      jsonSelectedColumn = selectedColumn; // Fallback to original value
     }
   } else {
     jsonSelectedColumn = selectedColumn;
   }
 
-  return parseUnknownToString(jsonSelectedColumn);
+  return {
+    value: parseUnknownToString(jsonSelectedColumn),
+    error,
+  };
 }

packages/shared/src/features/prompts/types.ts

@@ -1,15 +1,17 @@
 import { z } from "zod/v4";
-import { jsonSchema, PromptNameSchema } from "@langfuse/shared";
-import type { Prompt } from "@langfuse/shared";
-import { COMMIT_MESSAGE_MAX_LENGTH } from "@/src/features/prompts/constants";
+import type { Prompt } from "../../../prisma/generated/types";
+import { jsonSchema } from "../../utils/zod";
+import { COMMIT_MESSAGE_MAX_LENGTH } from "./constants";
+import { PromptChatMessageSchema } from "../../server/llm/types";
+import { PromptNameSchema } from "./validation";
 
-export const ChatMessageSchema = z.object({
-  role: z.string(),
-  content: z.string(),
-});
+export const SingleChatMessageSchema = PromptChatMessageSchema;
+export type SingleChatMessage = z.infer<typeof SingleChatMessageSchema>;
 
 export enum PromptType {
+  // eslint-disable-next-line no-unused-vars
   Chat = "chat",
+  // eslint-disable-next-line no-unused-vars
   Text = "text",
 }
 
@@ -41,7 +43,7 @@ const BaseCreateChatPromptSchema = z.object({
   name: PromptNameSchema,
   labels: z.array(PromptLabelSchema).default([]),
   type: z.literal(PromptType.Chat),
-  prompt: z.array(ChatMessageSchema),
+  prompt: z.array(PromptChatMessageSchema),
   config: jsonSchema.nullable().default({}),
   tags: z.array(z.string()).nullish(),
 });
@@ -134,7 +136,7 @@ export const BaseChatPromptSchema = z.object({
   tags: z.array(z.string()),
   labels: z.array(PromptLabelSchema),
   type: z.literal(PromptType.Chat),
-  prompt: z.array(ChatMessageSchema),
+  prompt: z.array(PromptChatMessageSchema),
   config: jsonSchema,
 });
 

packages/shared/src/features/scores/interfaces/ingestion/validation.ts

@@ -37,13 +37,13 @@ const ScorePropsAgainstConfigNumeric = z
     dataType: z.literal("NUMERIC"),
   })
   .superRefine((data, ctx) => {
-    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+    if (isPresent(data.maxValue) && data.value > data.maxValue) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,
         message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
       });
     }
-    if (isPresent(data.minValue) && data.value <= data.minValue) {
+    if (isPresent(data.minValue) && data.value < data.minValue) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,
         message: `Value is below minimum value of ${data.minValue} defined in config`,

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void,
+  onParseError?: (error: z.ZodError) => void, // eslint-disable-line no-unused-vars
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);

packages/shared/src/index.ts

@@ -13,6 +13,7 @@ export * from "./utils/json";
 export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
+export * from "./utils/prompts";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
@@ -43,6 +44,9 @@ export * from "./features/experiments/utils";
 // prompts
 export * from "./features/prompts/parsePromptDependencyTags";
 export * from "./features/prompts/validation";
+export * from "./features/prompts/types";
+export * from "./features/prompts/constants";
+export { compileChatMessages, compileChatMessagesWithIds, isPlaceholder, type MessagePlaceholderValues, type PromptMessage as ServerPromptMessage } from "./server/llm/compileChatMessages";
 
 // export db types only
 export * from "@prisma/client";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -11,6 +11,9 @@ export const BedrockCredentialSchema = z
   .optional();
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
 
+export const VertexAIConfigSchema = z.object({ location: z.string() });
+export type VertexAIConfig = z.infer<typeof VertexAIConfigSchema>;
+
 export const GCPServiceAccountKeySchema = z.object({
   type: z.literal("service_account"),
   project_id: z.string(),

packages/shared/src/interfaces/filters.ts

@@ -1,5 +1,6 @@
 import { z } from "zod/v4";
 
+// Make sure to update the InMemoryFilterService if you add new filter types
 export const filterOperators = {
   datetime: [">", "<", ">=", "<="],
   string: ["=", "contains", "does not contain", "starts with", "ends with"],

packages/shared/src/interfaces/tableNames.ts

@@ -4,10 +4,11 @@
  * to avoid coupling between different batch operation types.
  */
 export enum BatchTableNames {
-  Scores = "scores",
-  Sessions = "sessions",
-  Traces = "traces",
-  Observations = "observations",
-  DatasetRunItems = "dataset_run_items",
-  AuditLogs = "audit_logs",
+  Scores = "scores", // eslint-disable-line no-unused-vars
+  Sessions = "sessions", // eslint-disable-line no-unused-vars
+  Traces = "traces", // eslint-disable-line no-unused-vars
+  Observations = "observations", // eslint-disable-line no-unused-vars
+  DatasetRunItems = "dataset_run_items", // eslint-disable-line no-unused-vars
+  DatasetItems = "dataset_items", // eslint-disable-line no-unused-vars
+  AuditLogs = "audit_logs", // eslint-disable-line no-unused-vars
 }

packages/shared/src/server/headerPropagation.ts

@@ -30,6 +30,21 @@ export const contextWithLangfuseProps = (
         value: strValue,
       });
     });
+
+    // get x-langfuse-xxx headers and add them to the span
+    Object.keys(props.headers).forEach((name) => {
+      if (
+        name.toLowerCase().startsWith("x-langfuse") ||
+        name.toLowerCase().startsWith("x_langfuse")
+      ) {
+        const value = props.headers![name];
+        if (!value) return;
+        const strValue = Array.isArray(value) ? JSON.stringify(value) : value;
+        baggage = baggage.setEntry(`langfuse.header.${name}`, {
+          value: strValue,
+        });
+      }
+    });
   }
   if (props.userId) {
     baggage = baggage.setEntry("langfuse.user.id", { value: props.userId });

packages/shared/src/server/index.ts

@@ -4,12 +4,14 @@ export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
 export * from "./services/traces-ui-table-service";
+export * from "./services/InMemoryFilterService";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
 export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/utils";
 export * from "./llm/types";
+export * from "./llm/compileChatMessages";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
 export * from "./clickhouse/client";

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -45,7 +45,7 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
   return s3StorageServiceClient;
 };
 
-export type TokenCountDelegate = (p: {
+export type TokenCountDelegate = (p: { // eslint-disable-line no-unused-vars
   model: Model;
   text: unknown;
 }) => number | undefined;
@@ -229,12 +229,21 @@ export const processEventBatch = async (
     throw new Error("Redis not initialized, aborting event processing");
   }
 
+  const projectIdsToSkipS3List =
+    env.LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS?.split(",") ?? [];
+
   await Promise.all(
     Object.keys(sortedBatchByEventBodyId).map(async (id) => {
       const eventData = sortedBatchByEventBodyId[id];
       const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
       const queue = IngestionQueue.getInstance({ shardingKey });
 
+      const shouldSkipS3List =
+        getClickhouseEntityType(eventData.type) === "observation" &&
+        authCheck.scope.projectId !== null &&
+        (projectIdsToSkipS3List.includes(authCheck.scope.projectId) ||
+          source === "otel");
+
       return queue
         ? queue.add(
             QueueJobs.IngestionJob,
@@ -247,9 +256,7 @@ export const processEventBatch = async (
                   type: eventData.type,
                   eventBodyId: eventData.eventBodyId,
                   fileKey: eventData.key,
-                  skipS3List:
-                    source === "otel" &&
-                    getClickhouseEntityType(eventData.type) === "observation",
+                  skipS3List: shouldSkipS3List,
                 },
                 authCheck: authCheck as {
                   validKey: true;

packages/shared/src/server/instrumentation/index.ts

@@ -22,7 +22,7 @@ export type SpanCtx = {
   traceContext?: TCarrier;
 };
 
-type AsyncCallbackFn<T> = (span: opentelemetry.Span) => Promise<T>;
+type AsyncCallbackFn<T> = (span: opentelemetry.Span) => Promise<T>; // eslint-disable-line no-unused-vars
 
 export async function instrumentAsync<T>(
   ctx: SpanCtx,
@@ -64,7 +64,7 @@ export async function instrumentAsync<T>(
   );
 }
 
-type SyncCallbackFn<T> = (span: opentelemetry.Span) => T;
+type SyncCallbackFn<T> = (span: opentelemetry.Span) => T; // eslint-disable-line no-unused-vars
 
 export function instrumentSync<T>(
   ctx: SpanCtx,
@@ -157,6 +157,61 @@ export const traceException = (
   });
 };
 
+export const addUserToSpan = (
+  attributes: {
+    userId?: string;
+    projectId?: string;
+    email?: string;
+    orgId?: string;
+    plan?: string;
+  },
+  span?: opentelemetry.Span,
+) => {
+  const activeSpan = span ?? getCurrentSpan();
+
+  if (!activeSpan) {
+    return;
+  }
+
+  const ctx = opentelemetry.context.active();
+  let baggage =
+    opentelemetry.propagation.getBaggage(ctx) ??
+    opentelemetry.propagation.createBaggage();
+
+  if (attributes.userId) {
+    baggage = baggage.setEntry("user.id", {
+      value: attributes.userId,
+    });
+    activeSpan.setAttribute("user.id", attributes.userId);
+  }
+  if (attributes.email) {
+    baggage = baggage.setEntry("user.email", {
+      value: attributes.email,
+    });
+    activeSpan.setAttribute("user.email", attributes.email);
+  }
+  if (attributes.projectId) {
+    baggage = baggage.setEntry("langfuse.project.id", {
+      value: attributes.projectId,
+    });
+    activeSpan.setAttribute("langfuse.project.id", attributes.projectId);
+  }
+  if (attributes.orgId) {
+    baggage = baggage.setEntry("langfuse.org.id", {
+      value: attributes.orgId,
+    });
+    activeSpan.setAttribute("langfuse.org.id", attributes.orgId);
+  }
+  if (attributes.plan) {
+    baggage = baggage.setEntry("langfuse.org.plan", {
+      value: attributes.plan,
+    });
+    activeSpan.setAttribute("langfuse.org.plan", attributes.plan);
+  }
+
+  return opentelemetry.propagation.setBaggage(ctx, baggage);
+};
+
 export const getTracer = (name: string) => opentelemetry.trace.getTracer(name);
 
 const cloudWatchClient = new CloudWatchClient();

packages/shared/src/server/llm/compileChatMessages.ts

@@ -0,0 +1,116 @@
+import { z } from "zod/v4";
+import { v4 as uuidv4 } from "uuid";
+import { type ChatMessage, type PlaceholderMessage, ChatMessageType, type PromptChatMessageSchema, type ChatMessageWithId, type ChatMessageWithIdNoPlaceholders, ChatMessageSchema } from "./types";
+
+export type MessagePlaceholderValues = Record<string, ChatMessage[]>;
+export type PromptMessage = z.infer<typeof PromptChatMessageSchema>;
+
+export function isPlaceholder(message: PromptMessage): message is PlaceholderMessage {
+  return "type" in message && message.type === ChatMessageType.Placeholder;
+}
+
+function validateMessage(message: unknown): message is ChatMessage {
+  return ChatMessageSchema.safeParse(message).success;
+}
+
+function replaceTextVariables(
+  content: string,
+  textVariables: Record<string, string>
+): string {
+  let result = content;
+  for (const [varName, varValue] of Object.entries(textVariables)) {
+    // Create regex that handles optional whitespace around variable name
+    const variablePattern = new RegExp(`{{\\s*${varName}\\s*}}`, "g");
+    result = result.replace(variablePattern, varValue);
+  }
+  return result;
+}
+
+function expandPlaceholder(
+  placeholder: PlaceholderMessage,
+  placeholderValues: MessagePlaceholderValues
+): ChatMessage[] {
+  const replacementMessages = placeholderValues[placeholder.name];
+
+  if (!replacementMessages) {
+    throw new Error(`Missing value for message placeholder: ${placeholder.name}`);
+  }
+
+  if (!Array.isArray(replacementMessages)) {
+    throw new Error(`Placeholder value for '${placeholder.name}' must be an array of messages`);
+  }
+
+  for (const replacementMsg of replacementMessages) {
+    if (!validateMessage(replacementMsg)) {
+      throw new Error(`Invalid message format in placeholder '${placeholder.name}': messages must have 'role' and 'content' properties`);
+    }
+  }
+  return replacementMessages;
+}
+
+export function compileChatMessages(
+  messages: PromptMessage[],
+  placeholderValues: MessagePlaceholderValues,
+  textVariables?: Record<string, string>
+): ChatMessage[] {
+  const expandedMessages = messages.flatMap((message) =>
+    isPlaceholder(message)
+      ? expandPlaceholder(message, placeholderValues)
+      : [message as ChatMessage]
+  );
+
+  // substitute text variables
+  if (!textVariables || Object.keys(textVariables).length === 0) {
+    return expandedMessages;
+  }
+
+  return expandedMessages.map((message) => {
+    if (!message.content) {
+      return message;
+    }
+
+    return {
+      ...message,
+      content: replaceTextVariables(message.content, textVariables)
+    };
+  });
+}
+
+export function compileChatMessagesWithIds(
+  messages: ChatMessageWithId[],
+  placeholderValues: Record<string, ChatMessage[]>,
+  textVariables?: Record<string, string>
+): ChatMessageWithIdNoPlaceholders[] {
+  // TODO: check, is it even important to retain the IDs?
+  const expandedMessages = messages.flatMap((message) => {
+    if (isPlaceholder(message)) {
+      const expandedMsgs = expandPlaceholder(message, placeholderValues);
+      return expandedMsgs.map(msg => ({ ...msg, id: uuidv4() }));
+    } else {
+      // Preserve message IDs for already non-placeholder messages
+      return [message as ChatMessageWithIdNoPlaceholders];
+    }
+  });
+
+  // substitute text variables
+  if (!textVariables || Object.keys(textVariables).length === 0) {
+    return expandedMessages;
+  }
+
+  return expandedMessages.map((message) => {
+    if (!message.content) {
+      return message;
+    }
+
+    return {
+      ...message,
+      content: replaceTextVariables(message.content, textVariables)
+    };
+  });
+}
+
+export function extractPlaceholderNames(messages: PromptMessage[]): string[] {
+  return messages
+    .filter((msg): msg is PlaceholderMessage => "type" in msg && msg.type === ChatMessageType.Placeholder)
+    .map(msg => msg.name);
+}

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,4 +1,6 @@
-import { type ZodSchema } from "zod/v4";
+// We need to use Zod3 for structured outputs due to a bug in
+// ChatVertexAI. See issue: https://github.com/langfuse/langfuse/issues/7429
+import { type ZodSchema } from "zod/v3";
 
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatVertexAI } from "@langchain/google-vertexai";
@@ -20,6 +22,7 @@ import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
 import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
+  VertexAIConfigSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
@@ -60,7 +63,7 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 };
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & {
+  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
     streaming: true;
   },
 ): Promise<{
@@ -69,7 +72,7 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & {
+  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
     streaming: false;
   },
 ): Promise<{
@@ -78,7 +81,7 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & {
+  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
     streaming: false;
     structuredOutputSchema: ZodSchema;
   },
@@ -88,7 +91,7 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & {
+  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
     tools: LLMToolDefinition[];
     streaming: false;
   },
@@ -251,6 +254,9 @@ export async function fetchLLMCompletion(
     });
   } else if (modelParams.adapter === LLMAdapter.VertexAI) {
     const credentials = GCPServiceAccountKeySchema.parse(JSON.parse(apiKey));
+    const { location } = config
+      ? VertexAIConfigSchema.parse(config)
+      : { location: undefined };
 
     // Requests time out after 60 seconds for both public and private endpoints by default
     // Reference: https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions#send-request
@@ -261,6 +267,7 @@ export async function fetchLLMCompletion(
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
+      location,
       authOptions: {
         projectId: credentials.project_id,
         credentials,

packages/shared/src/server/llm/types.ts

@@ -1,6 +1,9 @@
 import { LlmApiKeys } from "@prisma/client";
 import z from "zod/v4";
-import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
+import {
+  BedrockConfigSchema,
+  VertexAIConfigSchema,
+} from "../../interfaces/customLLMProviderConfigSchemas";
 import { TokenCountDelegate } from "../ingestion/processEventBatch";
 import { AuthHeaderValidVerificationResult } from "../auth/types";
 
@@ -112,6 +115,8 @@ export enum ChatMessageRole {
   Tool = "tool",
 }
 
+// Thought: should placeholder not semantically be part of this, because it can be
+// PublicAPICreated of type? Works for now though.
 export enum ChatMessageType {
   System = "system",
   Developer = "developer",
@@ -120,6 +125,7 @@ export enum ChatMessageType {
   AssistantToolCall = "assistant-tool-call",
   ToolResult = "tool-result",
   PublicAPICreated = "public-api-created",
+  Placeholder = "placeholder",
 }
 
 export const SystemMessageSchema = z.object({
@@ -168,6 +174,12 @@ export const ToolResultMessageSchema = z.object({
 });
 export type ToolResultMessage = z.infer<typeof ToolResultMessageSchema>;
 
+export const PlaceholderMessageSchema = z.object({
+  type: z.literal(ChatMessageType.Placeholder),
+  name: z.string().regex(/^[a-zA-Z][a-zA-Z0-9_]*$/, "Placeholder name must start with a letter and contain only alphanumeric characters and underscores"),
+});
+export type PlaceholderMessage = z.infer<typeof PlaceholderMessageSchema>;
+
 export const ChatMessageDefaultRoleSchema = z.enum(ChatMessageRole);
 export const ChatMessageSchema = z.union([
   SystemMessageSchema,
@@ -190,12 +202,16 @@ export const ChatMessageSchema = z.union([
 ]);
 
 export type ChatMessage = z.infer<typeof ChatMessageSchema>;
-export type ChatMessageWithId = ChatMessage & { id: string };
+export type ChatMessageWithId = (ChatMessage & { id: string }) | (PlaceholderMessage & { id: string });
+export type ChatMessageWithIdNoPlaceholders = (ChatMessage & { id: string });
 
-export const PromptChatMessageSchema = z.object({
-  role: z.string(),
-  content: z.string(),
-});
+export const PromptChatMessageSchema = z.union([
+  z.object({
+    role: z.string(),
+    content: z.string(),
+  }),
+  PlaceholderMessageSchema,
+]);
 export const PromptChatMessageListSchema = z.array(PromptChatMessageSchema);
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
@@ -215,11 +231,11 @@ export const SYSTEM_ROLES: string[] = [
   ChatMessageRole.Developer,
 ];
 
-export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+export const TextPromptContentSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
   PromptChatMessageListSchema,
-  TextPromptSchema,
+  TextPromptContentSchema,
 ]);
 export type PromptContent = z.infer<typeof PromptContentSchema>;
 
@@ -319,10 +335,10 @@ export const anthropicModels = [
 
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const vertexAIModels = [
+  "gemini-2.0-flash",
   "gemini-2.5-pro-preview-05-06",
   "gemini-2.5-flash-preview-05-20",
   "gemini-2.0-pro-exp-02-05",
-  "gemini-2.0-flash",
   "gemini-2.0-flash-001",
   "gemini-2.0-flash-lite-preview-02-05",
   "gemini-2.0-flash-exp",
@@ -379,7 +395,7 @@ export const LLMApiKeySchema = z
     baseURL: z.string().nullable(),
     customModels: z.array(z.string()),
     withDefaultModels: z.boolean(),
-    config: BedrockConfigSchema.nullish(), // currently only Bedrock has additional config
+    config: z.union([BedrockConfigSchema, VertexAIConfigSchema]).nullish(), // Bedrock and VertexAI have additional config
   })
   // strict mode to prevent extra keys. Thorws error otherwise
   // https://github.com/colinhacks/zod?tab=readme-ov-file#strict

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -432,19 +432,19 @@ export class FilterList {
     this.filters.push(...filter);
   }
 
-  find(predicate: (filter: Filter) => boolean) {
+  find(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
     return this.filters.find(predicate);
   }
 
-  filter(predicate: (filter: Filter) => boolean) {
+  filter(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
     return new FilterList(this.filters.filter(predicate));
   }
 
-  some(predicate: (filter: Filter) => boolean) {
+  some(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
     return this.filters.some(predicate);
   }
 
-  forEach(callback: (filter: Filter) => void) {
+  forEach(callback: (filter: Filter) => void) { // eslint-disable-line no-unused-vars
     this.filters.forEach(callback);
   }
 

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -5,13 +5,16 @@ const regexIndefiniteCharacters = "%";
 export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
+  tablePrefix?: string,
 ) => {
+  const prefix = tablePrefix ? `${tablePrefix}.` : "";
+
   const conditions = [
     !searchType || searchType.includes("id")
-      ? `id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR name ILIKE {searchString: String}`
+      ? `${prefix}id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
     searchType && searchType.includes("content")
-      ? `input ILIKE {searchString: String} OR output ILIKE {searchString: String}`
+      ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
       : null,
   ].filter(Boolean);
 

packages/shared/src/server/redis/getQueue.ts

@@ -70,8 +70,9 @@ export function getQueue(
       return ScoreDeleteQueue.getInstance();
     case QueueName.DeadLetterRetryQueue:
       return DeadLetterRetryQueue.getInstance();
-    default:
-      const exhaustiveCheckDefault: never = queueName;
+    default: {
+      const exhaustiveCheckDefault: never = queueName; // eslint-disable-line no-case-declarations, no-unused-vars
       throw new Error(`Queue ${queueName} not found`);
+    }
   }
 }

packages/shared/src/server/redis/redis.ts

@@ -167,9 +167,9 @@ const createRedisClient = () => {
 
 declare global {
   // eslint-disable-next-line no-var
-  var redis: undefined | ReturnType<typeof createRedisClient>;
+  var redis: undefined | ReturnType<typeof createRedisClient>; // eslint-disable-line no-unused-vars
 }
 
-export const redis = globalThis.redis ?? createRedisClient();
+export const redis = globalThis.redis ?? createRedisClient(); // eslint-disable-line no-undef
 
-if (env.NODE_ENV !== "production") globalThis.redis = redis;
+if (env.NODE_ENV !== "production") globalThis.redis = redis; // eslint-disable-line no-undef

packages/shared/src/server/repositories/clickhouse.ts

@@ -8,7 +8,8 @@ import { getTracer, instrumentAsync } from "../instrumentation";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
-import { context, trace } from "@opentelemetry/api";
+import { context, SpanKind, trace } from "@opentelemetry/api";
+import { backOff } from "exponential-backoff";
 import {
   StorageService,
   StorageServiceFactory,
@@ -37,98 +38,103 @@ export async function upsertClickhouse<
 >(opts: {
   table: "scores" | "traces" | "observations";
   records: T[];
-  eventBodyMapper: (body: T) => Record<string, unknown>;
+  eventBodyMapper: (body: T) => Record<string, unknown>; // eslint-disable-line no-unused-vars
   tags?: Record<string, string>;
 }): Promise<void> {
-  return await instrumentAsync({ name: "clickhouse-upsert" }, async (span) => {
-    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
-    span.setAttribute("ch.query.table", opts.table);
+  return await instrumentAsync(
+    { name: "clickhouse-upsert", spanKind: SpanKind.CLIENT },
+    async (span) => {
+      // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+      span.setAttribute("ch.query.table", opts.table);
+      span.setAttribute("db.system", "clickhouse");
+      span.setAttribute("db.operation.name", "UPSERT");
 
-    await Promise.all(
-      opts.records.map(async (record) => {
-        // drop trailing s and pretend it's always a create.
-        // Only applicable to scores and traces.
-        let eventType = `${opts.table.slice(0, -1)}-create`;
-        if (opts.table === "observations") {
-          // @ts-ignore - If it's an observation we now that `type` is a string
-          eventType = `${record["type"].toLowerCase()}-create`;
-        }
+      await Promise.all(
+        opts.records.map(async (record) => {
+          // drop trailing s and pretend it's always a create.
+          // Only applicable to scores and traces.
+          let eventType = `${opts.table.slice(0, -1)}-create`;
+          if (opts.table === "observations") {
+            // @ts-ignore - If it's an observation we now that `type` is a string
+            eventType = `${record["type"].toLowerCase()}-create`;
+          }
 
-        const eventId = randomUUID();
-        const bucketPath = `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${eventId}.json`;
+          const eventId = randomUUID();
+          const bucketPath = `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${eventId}.json`;
 
-        // Write new file directly to ClickHouse. We don't use the ClickHouse writer here as we expect more limited traffic
-        // and are not worried that much about latency.
-        await clickhouseClient().insert({
-          table: "blob_storage_file_log",
-          values: [
-            {
-              id: randomUUID(),
-              project_id: record.project_id,
-              entity_type: getClickhouseEntityType(eventType),
-              entity_id: record.id,
-              event_id: eventId,
-              bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-              bucket_path: bucketPath,
-              event_ts: convertDateToClickhouseDateTime(new Date()),
-              is_deleted: 0,
+          // Write new file directly to ClickHouse. We don't use the ClickHouse writer here as we expect more limited traffic
+          // and are not worried that much about latency.
+          await clickhouseClient().insert({
+            table: "blob_storage_file_log",
+            values: [
+              {
+                id: randomUUID(),
+                project_id: record.project_id,
+                entity_type: getClickhouseEntityType(eventType),
+                entity_id: record.id,
+                event_id: eventId,
+                bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+                bucket_path: bucketPath,
+                event_ts: convertDateToClickhouseDateTime(new Date()),
+                is_deleted: 0,
+              },
+            ],
+            format: "JSONEachRow",
+            clickhouse_settings: {
+              log_comment: JSON.stringify(opts.tags ?? {}),
             },
-          ],
-          format: "JSONEachRow",
-          clickhouse_settings: {
-            log_comment: JSON.stringify(opts.tags ?? {}),
-          },
-        });
+          });
 
-        return getS3StorageServiceClient(
-          env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-        ).uploadJson(bucketPath, [
-          {
-            id: eventId,
-            timestamp: new Date().toISOString(),
-            type: eventType,
-            body: opts.eventBodyMapper(record),
-          },
-        ]);
-      }),
-    );
+          return getS3StorageServiceClient(
+            env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+          ).uploadJson(bucketPath, [
+            {
+              id: eventId,
+              timestamp: new Date().toISOString(),
+              type: eventType,
+              body: opts.eventBodyMapper(record),
+            },
+          ]);
+        }),
+      );
 
-    const res = await clickhouseClient().insert({
-      table: opts.table,
-      values: opts.records.map((record) => ({
-        ...record,
-        event_ts: convertDateToClickhouseDateTime(new Date()),
-      })),
-      format: "JSONEachRow",
-      clickhouse_settings: {
-        log_comment: JSON.stringify(opts.tags ?? {}),
-      },
-    });
-    // same logic as for prisma. we want to see queries in development
-    if (env.NODE_ENV === "development") {
-      logger.info(`clickhouse:insert ${res.query_id} ${opts.table}`);
-    }
-
-    span.setAttribute("ch.queryId", res.query_id);
-
-    // add summary headers to the span. Helps to tune performance
-    const summaryHeader = res.response_headers["x-clickhouse-summary"];
-    if (summaryHeader) {
-      try {
-        const summary = Array.isArray(summaryHeader)
-          ? JSON.parse(summaryHeader[0])
-          : JSON.parse(summaryHeader);
-        for (const key in summary) {
-          span.setAttribute(`ch.${key}`, summary[key]);
-        }
-      } catch (error) {
-        logger.debug(
-          `Failed to parse clickhouse summary header ${summaryHeader}`,
-          error,
-        );
+      const res = await clickhouseClient().insert({
+        table: opts.table,
+        values: opts.records.map((record) => ({
+          ...record,
+          event_ts: convertDateToClickhouseDateTime(new Date()),
+        })),
+        format: "JSONEachRow",
+        clickhouse_settings: {
+          log_comment: JSON.stringify(opts.tags ?? {}),
+        },
+      });
+      // same logic as for prisma. we want to see queries in development
+      if (env.NODE_ENV === "development") {
+        logger.info(`clickhouse:insert ${res.query_id} ${opts.table}`);
       }
-    }
-  });
+
+      span.setAttribute("ch.queryId", res.query_id);
+
+      // add summary headers to the span. Helps to tune performance
+      const summaryHeader = res.response_headers["x-clickhouse-summary"];
+      if (summaryHeader) {
+        try {
+          const summary = Array.isArray(summaryHeader)
+            ? JSON.parse(summaryHeader[0])
+            : JSON.parse(summaryHeader);
+          for (const key in summary) {
+            span.setAttribute(`ch.${key}`, summary[key]);
+          }
+        } catch (error) {
+          logger.debug(
+            `Failed to parse clickhouse summary header ${summaryHeader}`,
+            error,
+          );
+        }
+      }
+    },
+  );
 }
 
 export async function* queryClickhouseStream<T>(opts: {
@@ -138,7 +144,9 @@ export async function* queryClickhouseStream<T>(opts: {
   tags?: Record<string, string>;
 }): AsyncGenerator<T> {
   const tracer = getTracer("clickhouse-query-stream");
-  const span = tracer.startSpan("clickhouse-query-stream");
+  const span = tracer.startSpan("clickhouse-query-stream", {
+    kind: SpanKind.CLIENT,
+  });
 
   try {
     const res = await context.with(
@@ -146,6 +154,9 @@ export async function* queryClickhouseStream<T>(opts: {
       async () => {
         // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
         span.setAttribute("ch.query.text", opts.query);
+        span.setAttribute("db.system", "clickhouse");
+        span.setAttribute("db.query.text", opts.query);
+        span.setAttribute("db.operation.name", "SELECT");
 
         const res = await clickhouseClient(opts.clickhouseConfigs).query({
           query: opts.query,
@@ -193,51 +204,107 @@ export async function* queryClickhouseStream<T>(opts: {
   }
 }
 
+/**
+ * Determines if an error is retryable (socket hang up, connection reset, etc.)
+ */
+function isRetryableError(error: unknown): boolean {
+  if (!error || typeof error !== "object") return false;
+
+  const errorMessage = (error as Error).message?.toLowerCase() || "";
+
+  // Check for socket hang up and other network-related errors
+  return errorMessage.includes("socket hang up");
+}
+
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
   clickhouseConfigs?: NodeClickHouseClientConfigOptions;
   tags?: Record<string, string>;
 }): Promise<T[]> {
-  return await instrumentAsync({ name: "clickhouse-query" }, async (span) => {
-    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
-    span.setAttribute("ch.query.text", opts.query);
+  return await instrumentAsync(
+    { name: "clickhouse-query", spanKind: SpanKind.CLIENT },
+    async (span) => {
+      // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+      span.setAttribute("ch.query.text", opts.query);
+      span.setAttribute("db.system", "clickhouse");
+      span.setAttribute("db.query.text", opts.query);
+      span.setAttribute("db.operation.name", "SELECT");
 
-    const res = await clickhouseClient(opts.clickhouseConfigs).query({
-      query: opts.query,
-      format: "JSONEachRow",
-      query_params: opts.params,
-      clickhouse_settings: {
-        log_comment: JSON.stringify(opts.tags ?? {}),
-      },
-    });
-    // same logic as for prisma. we want to see queries in development
-    if (env.NODE_ENV === "development") {
-      logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
-    }
+      // Retry logic for socket hang up and other network errors
+      return await backOff(
+        async () => {
+          const res = await clickhouseClient(opts.clickhouseConfigs).query({
+            query: opts.query,
+            format: "JSONEachRow",
+            query_params: opts.params,
+            clickhouse_settings: {
+              log_comment: JSON.stringify(opts.tags ?? {}),
+            },
+          });
 
-    span.setAttribute("ch.queryId", res.query_id);
+          // same logic as for prisma. we want to see queries in development
+          if (env.NODE_ENV === "development") {
+            logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+          }
 
-    // add summary headers to the span. Helps to tune performance
-    const summaryHeader = res.response_headers["x-clickhouse-summary"];
-    if (summaryHeader) {
-      try {
-        const summary = Array.isArray(summaryHeader)
-          ? JSON.parse(summaryHeader[0])
-          : JSON.parse(summaryHeader);
-        for (const key in summary) {
-          span.setAttribute(`ch.${key}`, summary[key]);
-        }
-      } catch (error) {
-        logger.debug(
-          `Failed to parse clickhouse summary header ${summaryHeader}`,
-          error,
-        );
-      }
-    }
+          span.setAttribute("ch.queryId", res.query_id);
 
-    return await res.json<T>();
-  });
+          // add summary headers to the span. Helps to tune performance
+          const summaryHeader = res.response_headers["x-clickhouse-summary"];
+          if (summaryHeader) {
+            try {
+              const summary = Array.isArray(summaryHeader)
+                ? JSON.parse(summaryHeader[0])
+                : JSON.parse(summaryHeader);
+              for (const key in summary) {
+                span.setAttribute(`ch.${key}`, summary[key]);
+              }
+            } catch (error) {
+              logger.debug(
+                `Failed to parse clickhouse summary header ${summaryHeader}`,
+                error,
+              );
+            }
+          }
+
+          return await res.json<T>();
+        },
+        {
+          numOfAttempts: env.LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS,
+          retry: (error: Error, attemptNumber: number) => {
+            const shouldRetry = isRetryableError(error);
+            if (shouldRetry) {
+              logger.warn(
+                `ClickHouse query failed with retryable error (attempt ${attemptNumber}/${env.LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS}): ${error.message}`,
+                {
+                  error: error.message,
+                  attemptNumber,
+                  tags: opts.tags,
+                },
+              );
+              span.addEvent("clickhouse-query-retry", {
+                "retry.attempt": attemptNumber,
+                "retry.error": error.message,
+              });
+            } else {
+              logger.error(
+                `ClickHouse query failed with non-retryable error: ${error.message}`,
+                {
+                  error: error.message,
+                  tags: opts.tags,
+                },
+              );
+            }
+            return shouldRetry;
+          },
+          startingDelay: 100,
+          timeMultiple: 1,
+          maxDelay: 100,
+        },
+      );
+    },
+  );
 }
 
 export async function commandClickhouse(opts: {
@@ -246,41 +313,48 @@ export async function commandClickhouse(opts: {
   clickhouseConfigs?: NodeClickHouseClientConfigOptions;
   tags?: Record<string, string>;
 }): Promise<void> {
-  return await instrumentAsync({ name: "clickhouse-command" }, async (span) => {
-    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
-    span.setAttribute("ch.query.text", opts.query);
-    const res = await clickhouseClient(opts.clickhouseConfigs).command({
-      query: opts.query,
-      query_params: opts.params,
-      clickhouse_settings: {
-        log_comment: JSON.stringify(opts.tags ?? {}),
-      },
-    });
-    // same logic as for prisma. we want to see queries in development
-    if (env.NODE_ENV === "development") {
-      logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
-    }
+  return await instrumentAsync(
+    { name: "clickhouse-command", spanKind: SpanKind.CLIENT },
+    async (span) => {
+      // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+      span.setAttribute("ch.query.text", opts.query);
+      span.setAttribute("db.system", "clickhouse");
+      span.setAttribute("db.query.text", opts.query);
+      span.setAttribute("db.operation.name", "COMMAND");
 
-    span.setAttribute("ch.queryId", res.query_id);
-
-    // add summary headers to the span. Helps to tune performance
-    const summaryHeader = res.response_headers["x-clickhouse-summary"];
-    if (summaryHeader) {
-      try {
-        const summary = Array.isArray(summaryHeader)
-          ? JSON.parse(summaryHeader[0])
-          : JSON.parse(summaryHeader);
-        for (const key in summary) {
-          span.setAttribute(`ch.${key}`, summary[key]);
-        }
-      } catch (error) {
-        logger.debug(
-          `Failed to parse clickhouse summary header ${summaryHeader}`,
-          error,
-        );
+      const res = await clickhouseClient(opts.clickhouseConfigs).command({
+        query: opts.query,
+        query_params: opts.params,
+        clickhouse_settings: {
+          log_comment: JSON.stringify(opts.tags ?? {}),
+        },
+      });
+      // same logic as for prisma. we want to see queries in development
+      if (env.NODE_ENV === "development") {
+        logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
       }
-    }
-  });
+
+      span.setAttribute("ch.queryId", res.query_id);
+
+      // add summary headers to the span. Helps to tune performance
+      const summaryHeader = res.response_headers["x-clickhouse-summary"];
+      if (summaryHeader) {
+        try {
+          const summary = Array.isArray(summaryHeader)
+            ? JSON.parse(summaryHeader[0])
+            : JSON.parse(summaryHeader);
+          for (const key in summary) {
+            span.setAttribute(`ch.${key}`, summary[key]);
+          }
+        } catch (error) {
+          logger.debug(
+            `Failed to parse clickhouse summary header ${summaryHeader}`,
+            error,
+          );
+        }
+      }
+    },
+  );
 }
 
 export function parseClickhouseUTCDateTimeFormat(dateStr: string): Date {

packages/shared/src/server/repositories/definitions.ts

@@ -122,6 +122,43 @@ export const traceRecordInsertSchema = traceRecordBaseSchema.extend({
 });
 export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;
 
+export const traceMtRecordInsertSchema = z.object({
+  // Identifiers
+  project_id: z.string(),
+  id: z.string(),
+  start_time: z.number(),
+  end_time: z.number().nullish(),
+  name: z.string(),
+
+  // Metadata properties
+  metadata: z.record(z.string(), z.string()),
+  user_id: z.string(),
+  session_id: z.string(),
+  environment: z.string(),
+  tags: z.array(z.string()),
+  version: z.string().nullish(),
+  release: z.string().nullish(),
+
+  // UI properties - nullable to prevent absent values being interpreted as overwrites
+  bookmarked: z.boolean().nullish(),
+  public: z.boolean().nullish(),
+
+  // Aggregations
+  observation_ids: z.array(z.string()),
+  score_ids: z.array(z.string()),
+  cost_details: z.record(z.string(), z.number()),
+  usage_details: z.record(z.string(), z.number()),
+
+  // Input/Output
+  input: z.string(),
+  output: z.string(),
+
+  created_at: z.number(),
+  updated_at: z.number(),
+  event_ts: z.number(),
+});
+export type TraceMtRecordInsertType = z.infer<typeof traceMtRecordInsertSchema>;
+
 export const scoreRecordBaseSchema = z.object({
   id: z.string(),
   project_id: z.string(),

packages/shared/src/server/repositories/observations.ts

@@ -687,7 +687,11 @@ const getObservationsTableInternal = async <T>(
   const appliedScoresFilter = scoresFilter.apply();
   const appliedObservationsFilter = observationsFilter.apply();
 
-  const search = clickhouseSearchCondition(opts.searchQuery, opts.searchType);
+  const search = clickhouseSearchCondition(
+    opts.searchQuery,
+    opts.searchType,
+    "o",
+  );
 
   const scoresCte = `WITH scores_agg AS (
     SELECT
@@ -1494,6 +1498,7 @@ export const getGenerationsForPostHog = async function* (
       o.provided_model_name as model,
       o.level as level,
       o.version as version,
+      o.environment as environment,
       t.id as trace_id,
       t.name as trace_name,
       t.session_id as trace_session_id,
@@ -1556,6 +1561,7 @@ export const getGenerationsForPostHog = async function* (
       langfuse_model: record.model,
       langfuse_level: record.level,
       langfuse_tags: record.trace_tags,
+      langfuse_environment: record.environment,
       langfuse_event_version: "1.0.0",
       $session_id: record.posthog_session_id ?? null,
       $set: {

packages/shared/src/server/repositories/scores.ts

@@ -31,6 +31,7 @@ import { _handleGetScoreById, _handleGetScoresByIds } from "./scores-utils";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { recordDistribution } from "../instrumentation";
+import { prisma } from "../../db";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -622,7 +623,7 @@ export const getCategoricalScoresGroupedByName = async (
     : undefined;
 
   const query = `
-    SELECT 
+    SELECT
       name AS label,
       groupArray(DISTINCT string_value) AS values
     FROM scores s
@@ -651,7 +652,58 @@ export const getCategoricalScoresGroupedByName = async (
     },
   });
 
-  return rows;
+  // Get score names from ClickHouse results to query score configs
+  const scoreNames = rows.map((row) => row.label);
+
+  // Query score_configs table for categorical configurations
+  const scoreConfigs =
+    scoreNames.length > 0
+      ? await prisma.scoreConfig.findMany({
+          where: {
+            projectId: projectId,
+            name: {
+              in: scoreNames,
+            },
+            dataType: "CATEGORICAL",
+            isArchived: false,
+          },
+          select: {
+            name: true,
+            categories: true,
+          },
+        })
+      : [];
+
+  // Create a map of score configs for easy lookup
+  const configMap = new Map(
+    scoreConfigs.map((config) => [config.name, config.categories]),
+  );
+
+  // Enhance the results with all possible category values from score configs
+  return rows.map((row) => {
+    const configCategories = configMap.get(row.label);
+
+    if (configCategories && Array.isArray(configCategories)) {
+      // Extract all possible category labels from the score config
+      const allPossibleValues = (
+        configCategories as Array<{ label: string; value: number }>
+      ).map((category) => category.label);
+
+      // Merge actual values from ClickHouse with all possible values from config
+      // Use Set to ensure uniqueness
+      const mergedValues = Array.from(
+        new Set([...row.values, ...allPossibleValues]),
+      );
+
+      return {
+        ...row,
+        values: mergedValues,
+      };
+    }
+
+    // If no config found, return original values
+    return row;
+  });
 };
 
 export const getScoresUiCount = async (props: {
@@ -1284,6 +1336,7 @@ export const getScoresForPostHog = async function* (
       s.name as name,
       s.value as value,
       s.comment as comment,
+      s.environment as environment,
       t.name as trace_name,
       t.session_id as trace_session_id,
       t.user_id as trace_user_id,
@@ -1336,6 +1389,7 @@ export const getScoresForPostHog = async function* (
       langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
       langfuse_release: record.trace_release,
       langfuse_tags: record.trace_tags,
+      langfuse_environment: record.environment,
       langfuse_event_version: "1.0.0",
       $session_id: record.posthog_session_id ?? null,
       $set: {

packages/shared/src/server/repositories/traces.ts

@@ -461,7 +461,7 @@ export const getTracesGroupedByUsers = async (
   );
 
   const tracesFilterRes = tracesFilter.apply();
-  const search = clickhouseSearchCondition(searchQuery);
+  const search = clickhouseSearchCondition(searchQuery, undefined, "t");
 
   // We mainly use queries like this to retrieve filter options.
   // Therefore, we can skip final as some inaccuracy in count is acceptable.
@@ -707,7 +707,7 @@ export const getTotalUserCount = async (
   );
 
   const tracesFilterRes = tracesFilter.apply();
-  const search = clickhouseSearchCondition(searchQuery);
+  const search = clickhouseSearchCondition(searchQuery, undefined, "t");
 
   const query = `
     SELECT COUNT(DISTINCT t.user_id) AS totalCount
@@ -957,6 +957,7 @@ export const getTracesForPostHog = async function* (
       t.release as release,
       t.version as version,
       t.tags as tags,
+      t.environment as environment,
       t.metadata['$posthog_session_id'] as posthog_session_id,
       o.total_cost as total_cost,
       o.latency_milliseconds / 1000 as latency,
@@ -1006,6 +1007,7 @@ export const getTracesForPostHog = async function* (
       langfuse_release: record.release,
       langfuse_version: record.version,
       langfuse_tags: record.tags,
+      langfuse_environment: record.environment,
       langfuse_event_version: "1.0.0",
       $session_id: record.posthog_session_id ?? null,
       $set: {

packages/shared/src/server/services/DashboardService/types.ts

@@ -33,6 +33,10 @@ export const HistogramChartConfig = BaseTotalValueChartConfig.extend({
   bins: z.number().int().min(1).max(100).optional().default(10),
 });
 
+export const PivotTableChartConfig = BaseTotalValueChartConfig.extend({
+  type: z.literal("PIVOT_TABLE"),
+});
+
 // Define dimension schema
 export const DimensionSchema = z.object({
   field: z.string(),
@@ -53,6 +57,7 @@ export const ChartConfigSchema = z.discriminatedUnion("type", [
   PieChartConfig,
   BigNumberChartConfig,
   HistogramChartConfig,
+  PivotTableChartConfig,
 ]);
 
 export const DashboardDefinitionWidgetWidgetSchema = z.object({

packages/shared/src/server/services/DefaultEvaluationModelService/DefaultEvalModelService.ts

@@ -1,6 +1,6 @@
 import z from "zod/v4";
 import { prisma } from "../../../db";
-import { LangfuseNotFoundError } from "../../../errors";
+import { LangfuseNotFoundError, QUEUE_ERROR_MESSAGES } from "../../../errors";
 import { LLMApiKeySchema, ZodModelConfig } from "../../llm/types";
 
 type ValidConfig = {
@@ -156,7 +156,7 @@ export class DefaultEvalModelService {
     if (!selectedModel) {
       return {
         valid: false,
-        error: `No default model or custom model found for project ${projectId}.`,
+        error: `${QUEUE_ERROR_MESSAGES.NO_DEFAULT_MODEL_ERROR} ${projectId}.`,
       };
     }
 
@@ -173,7 +173,7 @@ export class DefaultEvalModelService {
     if (!parsedKey.success) {
       return {
         valid: false,
-        error: `API key for provider "${selectedModel.provider}" not found in project ${projectId}.`,
+        error: `${QUEUE_ERROR_MESSAGES.API_KEY_ERROR} "${selectedModel.provider}" not found in project ${projectId}.`,
       };
     }
 

packages/shared/src/server/services/InMemoryFilterService.ts

@@ -0,0 +1,384 @@
+import { FilterCondition, FilterState } from "../../types";
+import { logger } from "../logger";
+
+export class InMemoryFilterService {
+  /**
+   * Evaluates whether a data object matches the given filter conditions.
+   *
+   * @param data - The data object to evaluate
+   * @param filter - The filter conditions to apply
+   * @param fieldMapper - Function to map filter column names to data object values
+   * @returns true if the data matches all filter conditions, false otherwise
+   */
+  static evaluateFilter<T>(
+    data: T,
+    filter: FilterState,
+    fieldMapper: (data: T, column: string) => unknown, // eslint-disable-line no-unused-vars
+  ): boolean {
+    try {
+      // If no filters, data matches
+      if (!filter || filter.length === 0) {
+        return true;
+      }
+
+      // Evaluate each filter condition
+      for (const condition of filter) {
+        if (!this.evaluateFilterCondition(data, condition, fieldMapper)) {
+          return false;
+        }
+      }
+
+      return true;
+    } catch (error) {
+      logger.error("Error evaluating filter in memory", {
+        error,
+        filterCount: filter?.length || 0,
+      });
+      // On error, return false to be safe (filter doesn't match)
+      return false;
+    }
+  }
+
+  /**
+   * Evaluates a single filter condition against a data object.
+   */
+  private static evaluateFilterCondition<T>(
+    data: T,
+    condition: FilterCondition,
+    fieldMapper: (data: T, column: string) => unknown, // eslint-disable-line no-unused-vars
+  ): boolean {
+    const { column, type, operator } = condition;
+
+    // Get the data field value based on the column
+    const fieldValue = fieldMapper(data, column);
+
+    switch (type) {
+      case "string":
+        return this.evaluateStringFilter(fieldValue, condition.value, operator);
+      case "datetime":
+        return this.evaluateDateTimeFilter(
+          fieldValue,
+          condition.value,
+          operator,
+        );
+      case "stringOptions":
+        return this.evaluateStringOptionsFilter(
+          fieldValue,
+          condition.value,
+          operator,
+        );
+      case "arrayOptions":
+        return this.evaluateArrayOptionsFilter(
+          fieldValue,
+          condition.value,
+          operator,
+        );
+      case "number":
+        return this.evaluateNumberFilter(fieldValue, condition.value, operator);
+      case "boolean":
+        return this.evaluateBooleanFilter(
+          fieldValue,
+          condition.value,
+          operator,
+        );
+      case "categoryOptions":
+        return this.evaluateCategoryOptionsFilter(
+          fieldValue,
+          condition.key,
+          condition.value,
+          operator,
+        );
+      case "stringObject":
+        return this.evaluateStringObjectFilter(
+          fieldValue,
+          condition.key,
+          condition.value,
+          operator,
+        );
+      case "numberObject":
+        return this.evaluateNumberObjectFilter(
+          fieldValue,
+          condition.key,
+          condition.value,
+          operator,
+        );
+      case "null":
+        return this.evaluateNullFilter(fieldValue, operator);
+      default:
+        logger.error("Unsupported filter type for in-memory evaluation", {
+          type,
+          column,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateStringFilter(
+    fieldValue: unknown,
+    filterValue: string,
+    operator: string,
+  ): boolean {
+    const strValue = fieldValue != null ? String(fieldValue) : "";
+
+    switch (operator) {
+      case "=":
+        return strValue === filterValue;
+      case "contains":
+        return strValue.includes(filterValue);
+      case "does not contain":
+        return !strValue.includes(filterValue);
+      case "starts with":
+        return strValue.startsWith(filterValue);
+      case "ends with":
+        return strValue.endsWith(filterValue);
+      default:
+        logger.error("Unsupported string filter operator", {
+          operator,
+          filterValue,
+          fieldValue: strValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateDateTimeFilter(
+    fieldValue: unknown,
+    filterValue: Date,
+    operator: string,
+  ): boolean {
+    if (!(fieldValue instanceof Date)) {
+      return false;
+    }
+
+    const fieldTime = fieldValue.getTime();
+    const filterTime = filterValue.getTime();
+
+    switch (operator) {
+      case ">":
+        return fieldTime > filterTime;
+      case "<":
+        return fieldTime < filterTime;
+      case ">=":
+        return fieldTime >= filterTime;
+      case "<=":
+        return fieldTime <= filterTime;
+      default:
+        logger.error("Unsupported datetime filter operator", {
+          operator,
+          filterValue,
+          fieldValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateNumberFilter(
+    fieldValue: unknown,
+    filterValue: number,
+    operator: string,
+  ): boolean {
+    if (typeof fieldValue !== "number") {
+      return false;
+    }
+
+    switch (operator) {
+      case "=":
+        return fieldValue === filterValue;
+      case ">":
+        return fieldValue > filterValue;
+      case "<":
+        return fieldValue < filterValue;
+      case ">=":
+        return fieldValue >= filterValue;
+      case "<=":
+        return fieldValue <= filterValue;
+      default:
+        logger.error("Unsupported number filter operator", {
+          operator,
+          filterValue,
+          fieldValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateCategoryOptionsFilter(
+    fieldValue: unknown,
+    key: string,
+    filterValues: string[],
+    operator: string,
+  ): boolean {
+    if (!fieldValue || typeof fieldValue !== "object") {
+      return false;
+    }
+
+    // Type assertion is safe here since we've checked typeof fieldValue === "object" above
+    const objectValue = (fieldValue as Record<string, unknown>)[key];
+    const stringValue = objectValue?.toString() || "";
+
+    switch (operator) {
+      case "any of":
+        return filterValues.includes(stringValue);
+      case "none of":
+        return !filterValues.includes(stringValue);
+      default:
+        logger.error("Unsupported categoryOptions filter operator", {
+          operator,
+          filterValues,
+          fieldValue: stringValue,
+          key,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateStringOptionsFilter(
+    fieldValue: unknown,
+    filterValues: string[],
+    operator: string,
+  ): boolean {
+    const strValue = fieldValue ? String(fieldValue) : "";
+
+    switch (operator) {
+      case "any of":
+        return filterValues.includes(strValue);
+      case "none of":
+        return !filterValues.includes(strValue);
+      default:
+        logger.error("Unsupported stringOptions filter operator", {
+          operator,
+          filterValues,
+          fieldValue: strValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateArrayOptionsFilter(
+    fieldValue: unknown,
+    filterValues: string[],
+    operator: string,
+  ): boolean {
+    if (!Array.isArray(fieldValue)) {
+      return false;
+    }
+
+    // Type assertion is safe here since we've checked Array.isArray above
+    const arrayValue = fieldValue as unknown[];
+
+    switch (operator) {
+      case "any of":
+        return arrayValue.some((val) => filterValues.includes(String(val)));
+      case "none of":
+        return !arrayValue.some((val) => filterValues.includes(String(val)));
+      case "all of":
+        return filterValues.every((val) =>
+          arrayValue.map(String).includes(val),
+        );
+      default:
+        logger.error("Unsupported arrayOptions filter operator", {
+          operator,
+          filterValues,
+          fieldValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateBooleanFilter(
+    fieldValue: unknown,
+    filterValue: boolean,
+    operator: string,
+  ): boolean {
+    switch (operator) {
+      case "=":
+        return fieldValue === filterValue;
+      case "<>":
+        return fieldValue !== filterValue;
+      default:
+        logger.error("Unsupported boolean filter operator", {
+          operator,
+          filterValue,
+          fieldValue,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateStringObjectFilter(
+    fieldValue: unknown,
+    key: string,
+    filterValue: string,
+    operator: string,
+  ): boolean {
+    if (!fieldValue || typeof fieldValue !== "object") {
+      return false;
+    }
+
+    // Type assertion is safe here since we've checked typeof fieldValue === "object" above
+    const objectValue = (fieldValue as Record<string, unknown>)[key];
+    const stringValue = objectValue?.toString() || "";
+    return this.evaluateStringFilter(stringValue, filterValue, operator);
+  }
+
+  private static evaluateNumberObjectFilter(
+    fieldValue: unknown,
+    key: string,
+    filterValue: number,
+    operator: string,
+  ): boolean {
+    if (!fieldValue || typeof fieldValue !== "object") {
+      return false;
+    }
+
+    // Type assertion is safe here since we've checked typeof fieldValue === "object" above
+    const objectValue = (fieldValue as Record<string, unknown>)[key];
+    const numValue =
+      typeof objectValue === "number"
+        ? objectValue
+        : parseFloat(String(objectValue));
+
+    if (isNaN(numValue)) {
+      return false;
+    }
+
+    switch (operator) {
+      case "=":
+        return numValue === filterValue;
+      case ">":
+        return numValue > filterValue;
+      case "<":
+        return numValue < filterValue;
+      case ">=":
+        return numValue >= filterValue;
+      case "<=":
+        return numValue <= filterValue;
+      default:
+        logger.error("Unsupported numberObject filter operator", {
+          operator,
+          filterValue,
+          fieldValue: numValue,
+          key,
+        });
+        return false;
+    }
+  }
+
+  private static evaluateNullFilter(
+    fieldValue: unknown,
+    operator: string,
+  ): boolean {
+    switch (operator) {
+      case "is null":
+        return fieldValue === null || fieldValue === undefined;
+      case "is not null":
+        return fieldValue !== null && fieldValue !== undefined;
+      default:
+        logger.error("Unsupported null filter operator", {
+          operator,
+          fieldValue,
+        });
+        return false;
+    }
+  }
+}

packages/shared/src/server/services/PromptService/index.ts

@@ -21,9 +21,12 @@ export class PromptService {
   private ttlSeconds: number;
 
   constructor(
+    // eslint-disable-next-line no-unused-vars
     private prisma: PrismaClient,
     private redis: Redis | Cluster | null,
+    // eslint-disable-next-line no-unused-vars
     private metricIncrementer?: // used for otel metrics
+    // eslint-disable-next-line no-unused-vars
     (name: string, value?: number) => void,
     cacheEnabled?: boolean, // used for testing
   ) {

packages/shared/src/server/services/PromptService/types.ts

@@ -13,8 +13,8 @@ export type PromptParams = {
 );
 
 export enum PromptServiceMetrics {
-  PromptCacheHit = "prompt_cache_hit",
-  PromptCacheMiss = "prompt_cache_miss",
+  PromptCacheHit = "prompt_cache_hit", // eslint-disable-line no-unused-vars
+  PromptCacheMiss = "prompt_cache_miss", // eslint-disable-line no-unused-vars
 }
 
 export type PartialPrompt = Pick<

packages/shared/src/server/services/StorageService.ts

@@ -28,21 +28,21 @@ type UploadFile = {
 };
 
 export interface StorageService {
-  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>;
+  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>; // eslint-disable-line no-unused-vars
 
-  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>;
+  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>; // eslint-disable-line no-unused-vars
 
-  download(path: string): Promise<string>;
+  download(path: string): Promise<string>; // eslint-disable-line no-unused-vars
 
-  listFiles(prefix: string): Promise<{ file: string; createdAt: Date }[]>;
+  listFiles(prefix: string): Promise<{ file: string; createdAt: Date }[]>; // eslint-disable-line no-unused-vars
 
   getSignedUrl(
-    fileName: string,
-    ttlSeconds: number,
-    asAttachment?: boolean,
+    fileName: string, // eslint-disable-line no-unused-vars
+    ttlSeconds: number, // eslint-disable-line no-unused-vars
+    asAttachment?: boolean, // eslint-disable-line no-unused-vars
   ): Promise<string>;
 
-  getSignedUploadUrl(params: {
+  getSignedUploadUrl(params: { // eslint-disable-line no-unused-vars
     path: string;
     ttlSeconds: number;
     sha256Hash: string;
@@ -50,7 +50,7 @@ export interface StorageService {
     contentLength: number;
   }): Promise<string>;
 
-  deleteFiles(paths: string[]): Promise<void>;
+  deleteFiles(paths: string[]): Promise<void>; // eslint-disable-line no-unused-vars
 }
 
 export class StorageServiceFactory {
@@ -213,7 +213,7 @@ class AzureBlobStorageService implements StorageService {
   }
 
   private async streamToString(
-    readableStream: NodeJS.ReadableStream,
+    readableStream: NodeJS.ReadableStream, // eslint-disable-line no-undef
   ): Promise<string> {
     return new Promise((resolve, reject) => {
       const chunks: string[] = [];
@@ -503,6 +503,7 @@ class S3StorageService implements StorageService {
     const listCommand = new ListObjectsV2Command({
       Bucket: this.bucketName,
       Prefix: prefix,
+      MaxKeys: env.LANGFUSE_S3_LIST_MAX_KEYS,
     });
 
     try {

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -329,7 +329,7 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
   const scoresFilterRes = scoresFilter.apply();
   const observationFilterRes = observationsFilter.apply();
 
-  const search = clickhouseSearchCondition(searchQuery, searchType);
+  const search = clickhouseSearchCondition(searchQuery, searchType, "t");
 
   const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
   const orderByCols = [

packages/shared/src/server/utils/DatabaseReadStream.ts

@@ -24,12 +24,12 @@ export class DatabaseReadStream<EntityType> extends Readable {
 
   constructor(
     // the delegate function takes care of querying the database in a paginated manner
-    private queryDelegate: (
-      pageSize: number,
-      offset: number
+    private queryDelegate: ( // eslint-disable-line no-unused-vars
+      pageSize: number, // eslint-disable-line no-unused-vars
+      offset: number // eslint-disable-line no-unused-vars
     ) => Promise<Array<EntityType>>,
-    private pageSize: number,
-    private maxRecords?: number
+    private pageSize: number, // eslint-disable-line no-unused-vars
+    private maxRecords?: number // eslint-disable-line no-unused-vars
   ) {
     super({ objectMode: true }); // Set object mode to true to allow pushing objects to the stream rather than strings or buffers
 

packages/shared/src/server/utils/transforms/transformStreamToCsv.ts

@@ -11,7 +11,7 @@ export function transformStreamToCsv(): Transform {
     objectMode: true,
     transform(
       row: Record<string, any>,
-      encoding: BufferEncoding,
+      encoding: BufferEncoding, // eslint-disable-line no-undef
       callback: TransformCallback,
     ): void {
       if (isFirstChunk) {

packages/shared/src/server/utils/transforms/transformStreamToJson.ts

@@ -9,8 +9,8 @@ export function transformStreamToJson(): Transform {
 
     transform(
       row: any,
-      encoding: BufferEncoding,
-      callback: TransformCallback
+      encoding: BufferEncoding, // eslint-disable-line no-undef, no-unused-vars
+      callback: TransformCallback,
     ): void {
       if (isFirstElement) {
         this.push("["); // Push the opening bracket for the first element

packages/shared/src/server/utils/transforms/transformStreamToJsonl.ts

@@ -7,7 +7,7 @@ export function transformStreamToJsonl(): Transform {
 
     transform(
       row: Record<string, any>,
-      encoding: BufferEncoding,
+      encoding: BufferEncoding, // eslint-disable-line no-undef, no-unused-vars
       callback: TransformCallback,
     ): void {
       this.push(stringify(row) + "\n");

packages/shared/src/utils/prompts.ts

@@ -0,0 +1,24 @@
+/**
+ * Client-safe utility functions for prompt handling
+ */
+
+export interface PromptMessage {
+  type?: string;
+  name?: string;
+  role?: string;
+  content?: string;
+}
+
+/**
+ * Extracts placeholder names from prompt messages.
+ * This is a client-safe version that doesn't depend on server-side types.
+ * @param messages Array of prompt messages
+ * @returns Array of placeholder names
+ */
+export function extractPlaceholderNames(messages: PromptMessage[]): string[] {
+  return messages
+    .filter((msg): msg is PromptMessage & { name: string } =>
+      msg.type === "placeholder" && typeof msg.name === "string"
+    )
+    .map(msg => msg.name);
+}

turbo.json

@@ -1,13 +1,25 @@
 {
   "$schema": "https://turbo.build/schema.json",
-  "globalDotEnv": [".env"],
-  "pipeline": {
+  "globalDependencies": [
+    ".env"
+  ],
+  "envMode": "loose",
+  "tasks": {
     "build": {
-      "dependsOn": ["db:generate", "^build"],
-      "outputs": ["dist/**", ".next/**", "!.next/cache/**"]
+      "dependsOn": [
+        "db:generate",
+        "^build"
+      ],
+      "outputs": [
+        "dist/**",
+        ".next/**",
+        "!.next/cache/**"
+      ]
     },
     "start": {
-      "dependsOn": ["^start"]
+      "dependsOn": [
+        "^start"
+      ]
     },
     "db:migrate": {
       "cache": false
@@ -20,27 +32,41 @@
     "dev": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": [
+        "db:generate",
+        "@langfuse/shared#build"
+      ]
     },
     "dev:worker": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": [
+        "db:generate",
+        "@langfuse/shared#build"
+      ]
     },
     "dev:web": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": [
+        "db:generate",
+        "@langfuse/shared#build"
+      ]
     },
     "db:generate": {
       "cache": false,
-      "dependsOn": ["^db:generate"]
+      "dependsOn": [
+        "^db:generate"
+      ]
     },
     "lint": {
       "cache": false
     },
     "test": {
-      "dependsOn": ["^test", "db:generate"]
+      "dependsOn": [
+        "^test",
+        "db:generate"
+      ]
     }
   }
 }

web/Dockerfile

@@ -5,7 +5,7 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
-RUN npm install turbo@^1.13.4 --global
+RUN npm install turbo@^2.5.4 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.72.1",
+  "version": "3.78.2",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -58,7 +58,7 @@
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/sdk-trace-node": "^1.26.0",
     "@paralleldrive/cuid2": "^2.2.2",
-    "@prisma/instrumentation": "^6.3.0",
+    "@prisma/instrumentation": "^6.10.1",
     "@radix-ui/react-accordion": "^1.2.1",
     "@radix-ui/react-alert-dialog": "^1.1.2",
     "@radix-ui/react-avatar": "^1.1.1",
@@ -128,7 +128,7 @@
     "posthog-js": "^1.176.0",
     "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
-    "prisma": "^6.3.0",
+    "prisma": "^6.10.1",
     "protobufjs": "^7.4.0",
     "rate-limiter-flexible": "^5.0.3",
     "react": "18.2.0",

web/public/generated/api/openapi.yml

@@ -4045,6 +4045,17 @@ paths:
             items:
               type: string
               nullable: true
+        - name: fields
+          in: query
+          description: >-
+            Comma-separated list of fields to include in the response. Available
+            field groups are 'core' (always included), 'io' (input, output,
+            metadata), 'scores', 'observations', 'metrics'. If not provided, all
+            fields are included. Example: 'core,scores,metrics'
+          required: false
+          schema:
+            type: string
+            nullable: true
       responses:
         '200':
           description: ''
@@ -6729,7 +6740,7 @@ components:
         prompt:
           type: array
           items:
-            $ref: '#/components/schemas/ChatMessage'
+            $ref: '#/components/schemas/ChatMessageWithPlaceholders'
         config:
           nullable: true
         labels:
@@ -6843,6 +6854,31 @@ components:
         - config
         - labels
         - tags
+    ChatMessageWithPlaceholders:
+      title: ChatMessageWithPlaceholders
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chatmessage
+            - $ref: '#/components/schemas/ChatMessage'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - placeholder
+            - $ref: '#/components/schemas/PlaceholderMessage'
+          required:
+            - type
     ChatMessage:
       title: ChatMessage
       type: object
@@ -6854,6 +6890,14 @@ components:
       required:
         - role
         - content
+    PlaceholderMessage:
+      title: PlaceholderMessage
+      type: object
+      properties:
+        name:
+          type: string
+      required:
+        - name
     TextPrompt:
       title: TextPrompt
       type: object
@@ -6871,7 +6915,7 @@ components:
         prompt:
           type: array
           items:
-            $ref: '#/components/schemas/ChatMessage'
+            $ref: '#/components/schemas/ChatMessageWithPlaceholders'
       required:
         - prompt
       allOf:

web/public/generated/postman/collection.json

@@ -2151,7 +2151,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ],\n    \"commitMessage\": \"example\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"type\": \"chatmessage\",\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ],\n    \"commitMessage\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -2877,7 +2877,7 @@
           "request": {
             "description": "Get list of traces",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&toTimestamp=&orderBy=&tags=&version=&release=&environment=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&toTimestamp=&orderBy=&tags=&version=&release=&environment=&fields=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -2946,6 +2946,11 @@
                   "key": "environment",
                   "value": "",
                   "description": "Optional filter for traces where the environment is one of the provided values."
+                },
+                {
+                  "key": "fields",
+                  "value": "",
+                  "description": "Comma-separated list of fields to include in the response. Available field groups are 'core' (always included), 'io' (input, output, metadata), 'scores', 'observations', 'metrics'. If not provided, all fields are included. Example: 'core,scores,metrics'"
                 }
               ],
               "variable": []

web/sentry.client.config.ts

@@ -1,5 +1,7 @@
 import * as Sentry from "@sentry/nextjs";
 
+const isEuOrUsRegionNonHipaa = process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ? ["EU", "US"].includes(process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) : false;
+
 Sentry.init({
   dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
   environment: process.env.NEXT_PUBLIC_SENTRY_ENVIRONMENT,
@@ -7,7 +9,10 @@ Sentry.init({
 
   // Replay may only be enabled for the client-side
   integrations: [
-    Sentry.replayIntegration(),
+    Sentry.replayIntegration({
+      maskAllText: !isEuOrUsRegionNonHipaa,
+      blockAllMedia: !isEuOrUsRegionNonHipaa,
+    }),
     Sentry.browserTracingIntegration(),
     Sentry.httpClientIntegration(),
     // Sentry.debugIntegration(),

web/src/__tests__/async/observations-trpc.servertest.ts

@@ -0,0 +1,128 @@
+/** @jest-environment node */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+
+import type { Session } from "next-auth";
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { prisma } from "@langfuse/shared/src/db";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import {
+  createTrace,
+  createTracesCh,
+  createObservation,
+  createObservationsCh,
+  createTraceScore,
+  createScoresCh,
+} from "@langfuse/shared/src/server";
+import { randomUUID } from "crypto";
+
+describe("traces trpc", () => {
+  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+  beforeEach(async () => await pruneDatabase());
+
+  const session: Session = {
+    expires: "1",
+    user: {
+      id: "user-1",
+      canCreateOrganizations: true,
+      name: "Demo User",
+      organizations: [
+        {
+          id: "seed-org-id",
+          name: "Test Organization",
+          role: "OWNER",
+          plan: "cloud:hobby",
+          cloudConfig: undefined,
+          projects: [
+            {
+              id: projectId,
+              role: "ADMIN",
+              retentionDays: 30,
+              deletedAt: null,
+              name: "Test Project",
+            },
+          ],
+        },
+      ],
+      featureFlags: {
+        excludeClickhouseRead: false,
+        templateFlag: true,
+      },
+      admin: true,
+    },
+    environment: {} as any,
+  };
+
+  const ctx = createInnerTRPCContext({ session });
+  const caller = appRouter.createCaller({ ...ctx, prisma });
+
+  describe("generations.all", () => {
+    it("should get all generations with full text search and trace + scores filter", async () => {
+      const traceId = randomUUID();
+      const generationId = randomUUID();
+      const scoreId = randomUUID();
+
+      // Create trace with searchable content
+      const trace = createTrace({
+        id: traceId,
+        project_id: projectId,
+        name: "test-trace-name",
+        user_id: "test-user-123",
+      });
+
+      await createTracesCh([trace]);
+
+      // Create generation with searchable input/output content
+      const generation = createObservation({
+        id: generationId,
+        project_id: projectId,
+        trace_id: traceId,
+        type: "GENERATION",
+        name: "test-generation",
+        input: "Hello world, this is a test input",
+        output: "This is a test response output",
+      });
+
+      await createObservationsCh([generation]);
+
+      // Create score for the trace
+      const score = createTraceScore({
+        id: scoreId,
+        project_id: projectId,
+        trace_id: traceId,
+        name: "quality-score",
+        value: 0.85,
+      });
+
+      await createScoresCh([score]);
+
+      // Test with full-text search, trace filter, and score filter
+      const generations = await caller.generations.all({
+        projectId,
+        searchQuery: "test input", // Full-text search
+        searchType: ["content"], // Search in input/output content
+        filter: [
+          {
+            column: "Trace Name",
+            operator: "contains",
+            value: "test-trace",
+            type: "string",
+          },
+          {
+            column: "Scores (numeric)",
+            key: "test",
+            operator: ">=",
+            value: 5,
+            type: "numberObject",
+          },
+        ],
+        orderBy: null,
+        limit: 50,
+        page: 0,
+      });
+
+      expect(generations.generations).toBeDefined();
+    });
+  });
+});

web/src/__tests__/async/scores-api-v1.servertest.ts

@@ -244,6 +244,67 @@ describe("/api/public/scores API Endpoint", () => {
         "test-key": "test-value-updated",
       });
     });
+
+    it("should post score with score config if in valid range", async () => {
+      const configId = v4();
+      const traceId = v4();
+      const scoreId = v4();
+
+      const { projectId: projectId, auth } = await createOrgProjectAndApiKey();
+
+      const config = await prisma.scoreConfig.create({
+        data: {
+          name: "score-name",
+          id: configId,
+          dataType: "NUMERIC",
+          maxValue: 100,
+          projectId: projectId,
+        },
+      });
+
+      const trace = createTrace({
+        id: traceId,
+        project_id: projectId,
+      });
+      await createTracesCh([trace]);
+
+      const score = createTraceScore({
+        id: scoreId,
+        project_id: projectId,
+        trace_id: traceId,
+        name: "score-name",
+        value: 100,
+        source: "API",
+        comment: "comment",
+        metadata: { "test-key": "test-value" },
+        observation_id: null,
+        environment: "production",
+        config_id: config.id,
+      });
+      await createScoresCh([score]);
+
+      const fetchedScore = await makeZodVerifiedAPICall(
+        GetScoreResponseV1,
+        "GET",
+        `/api/public/scores/${scoreId}`,
+        undefined,
+        auth,
+      );
+
+      expect(fetchedScore.body?.id).toBe(scoreId);
+      expect(fetchedScore.body?.traceId).toBe(traceId);
+      expect(fetchedScore.body?.name).toBe("score-name");
+      expect(fetchedScore.body?.value).toBe(100);
+      expect(fetchedScore.body?.configId).toBe(configId);
+      expect(fetchedScore.body?.observationId).toBeNull();
+      expect(fetchedScore.body?.comment).toBe("comment");
+      expect(fetchedScore.body?.source).toBe("API");
+      expect(fetchedScore.body?.projectId).toBe(projectId);
+      expect(fetchedScore.body?.environment).toBe("production");
+      expect(fetchedScore.body?.metadata).toEqual({
+        "test-key": "test-value",
+      });
+    });
   });
 
   describe("GET /api/public/scores", () => {

web/src/__tests__/async/traces-api.servertest.ts

@@ -601,4 +601,357 @@ describe("/api/public/traces API Endpoint", () => {
       expect(trace2).toBeUndefined();
     }, 40_000);
   }, 60_000);
+
+  describe("Fields Filtering", () => {
+    it("should fetch traces with all fields by default", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-all-fields",
+        user_id: "user-1",
+        project_id: projectId,
+        metadata: { key: "value" },
+        input: JSON.stringify({ prompt: "test" }),
+        output: JSON.stringify({ response: "test response" }),
+        release: "1.0.0",
+        version: "2.0.0",
+      });
+
+      const observation = createObservation({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-observation",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        input: "observation input",
+        output: "observation output",
+      });
+
+      const score = createTraceScore({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-score",
+        value: 0.8,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh([observation]);
+      await createScoresCh([score]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // All fields should be present by default
+      expect(trace.input).toEqual({ prompt: "test" });
+      expect(trace.output).toEqual({ response: "test response" });
+      expect(trace.metadata).toEqual({ key: "value" });
+      expect(trace.observations).toHaveLength(1);
+      expect(trace.scores).toHaveLength(1);
+      expect(trace.totalCost).toBeDefined();
+      expect(trace.latency).toBeCloseTo(1);
+    });
+
+    it("should fetch traces with only core fields when fields=core", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-core-only",
+        user_id: "user-1",
+        project_id: projectId,
+        metadata: { key: "value" },
+        input: JSON.stringify({ prompt: "test" }),
+        output: JSON.stringify({ response: "test response" }),
+        release: "1.0.0",
+        version: "2.0.0",
+      });
+
+      const observation = createObservation({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-observation",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+      });
+
+      const score = createTraceScore({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-score",
+        value: 0.8,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh([observation]);
+      await createScoresCh([score]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // Core fields should be present
+      expect(trace.id).toBe(traceId);
+      expect(trace.name).toBe("trace-core-only");
+      expect(trace.userId).toBe("user-1");
+      expect(trace.projectId).toBe(projectId);
+      expect(trace.release).toBe("1.0.0");
+      expect(trace.version).toBe("2.0.0");
+
+      // Non-core fields should have default values
+      expect(trace.input).toBeNull();
+      expect(trace.output).toBeNull();
+      expect(trace.metadata).toEqual({});
+      expect(trace.observations).toBeUndefined();
+      expect(trace.scores).toBeUndefined();
+      expect(trace.totalCost).toBeUndefined();
+      expect(trace.latency).toBeUndefined();
+    });
+
+    it("should fetch traces with IO fields when fields=core,io", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-io",
+        user_id: "user-1",
+        project_id: projectId,
+        metadata: { key: "value" },
+        input: JSON.stringify({ prompt: "test" }),
+        output: JSON.stringify({ response: "test response" }),
+      });
+
+      await createTracesCh([createdTrace]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core,io",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // Core and IO fields should be present
+      expect(trace.id).toBe(traceId);
+      expect(trace.name).toBe("trace-with-io");
+      expect(trace.input).toEqual({ prompt: "test" });
+      expect(trace.output).toEqual({ response: "test response" });
+      expect(trace.metadata).toEqual({ key: "value" });
+
+      // Other fields should have default values
+      expect(trace.observations).toBeUndefined();
+      expect(trace.scores).toBeUndefined();
+      expect(trace.totalCost).toBeUndefined();
+      expect(trace.latency).toBeUndefined();
+    });
+
+    it("should fetch traces with scores when fields=core,scores", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-scores",
+        project_id: projectId,
+      });
+
+      const score = createTraceScore({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-score",
+        value: 0.8,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createScoresCh([score]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core,scores",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // Core fields and scores should be present
+      expect(trace.id).toBe(traceId);
+      expect(trace.scores).toHaveLength(1);
+
+      // Other fields should have default values
+      expect(trace.input).toBeNull();
+      expect(trace.output).toBeNull();
+      expect(trace.metadata).toEqual({});
+      expect(trace.observations).toBeUndefined();
+      expect(trace.totalCost).toBeUndefined();
+      expect(trace.latency).toBeUndefined();
+    });
+
+    it("should fetch traces with observations when fields=core,observations", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-observations",
+        project_id: projectId,
+      });
+
+      const observation = createObservation({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-observation",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh([observation]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core,observations",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // Core fields and observations should be present
+      expect(trace.id).toBe(traceId);
+      expect(trace.observations).toHaveLength(1);
+
+      // Other fields should have default values
+      expect(trace.input).toBeNull();
+      expect(trace.output).toBeNull();
+      expect(trace.metadata).toEqual({});
+      expect(trace.scores).toBeUndefined();
+      expect(trace.totalCost).toBeUndefined();
+      expect(trace.latency).toBeUndefined();
+    });
+
+    it("should fetch traces with metrics when fields=core,metrics", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-metrics",
+        project_id: projectId,
+      });
+
+      const observation = createObservation({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-observation",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        total_cost: 0.05,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh([observation]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core,metrics",
+      );
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // Core fields and metrics should be present
+      expect(trace.id).toBe(traceId);
+      expect(trace.totalCost).toBe(0.05);
+      expect(trace.latency).toBeCloseTo(1);
+
+      // Other fields should have default values
+      expect(trace.input).toBeNull();
+      expect(trace.output).toBeNull();
+      expect(trace.metadata).toEqual({});
+      expect(trace.observations).toBeUndefined();
+      expect(trace.scores).toBeUndefined();
+    });
+
+    it("should handle invalid field names gracefully", async () => {
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=core,invalid,scores",
+      );
+
+      // Should still work, just ignoring invalid field names
+      expect(traces.status).toBe(200);
+      expect(traces.body.data).toBeDefined();
+    });
+
+    it("should handle empty fields parameter", async () => {
+      const traceId = randomUUID();
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "trace-with-all-fields",
+        user_id: "user-1",
+        project_id: projectId,
+        metadata: { key: "value" },
+        input: JSON.stringify({ prompt: "test" }),
+        output: JSON.stringify({ response: "test response" }),
+        release: "1.0.0",
+        version: "2.0.0",
+      });
+
+      const observation = createObservation({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-observation",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        input: "observation input",
+        output: "observation output",
+      });
+
+      const score = createTraceScore({
+        trace_id: traceId,
+        project_id: projectId,
+        name: "test-score",
+        value: 0.8,
+      });
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh([observation]);
+      await createScoresCh([score]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        "/api/public/traces?fields=",
+      );
+
+      // Should default to all fields when empty
+      expect(traces.status).toBe(200);
+      expect(traces.body.data).toBeDefined();
+
+      const trace = traces.body.data.find((t) => t.id === traceId);
+      expect(trace).toBeTruthy();
+      if (!trace) return;
+
+      // All fields should be present
+      expect(trace.input).toEqual({ prompt: "test" });
+      expect(trace.output).toEqual({ response: "test response" });
+      expect(trace.metadata).toEqual({ key: "value" });
+      expect(trace.observations).toHaveLength(1);
+      expect(trace.scores).toHaveLength(1);
+      expect(trace.totalCost).toBeDefined();
+      expect(trace.latency).toBeDefined();
+    });
+  });
 });

web/src/__tests__/async/traces-trpc.servertest.ts

@@ -6,7 +6,12 @@ import { pruneDatabase } from "@/src/__tests__/test-utils";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
-import { createTrace, createTracesCh } from "@langfuse/shared/src/server";
+import {
+  createTrace,
+  createTracesCh,
+  createTraceScore,
+  createScoresCh,
+} from "@langfuse/shared/src/server";
 import { randomUUID } from "crypto";
 
 describe("traces trpc", () => {
@@ -137,4 +142,56 @@ describe("traces trpc", () => {
       expect(traceRes?.timestamp).toEqual(new Date(trace.timestamp));
     });
   });
+
+  describe("traces.filterOptions", () => {
+    it("should include all possible categorical score values from score configs", async () => {
+      // Create a trace
+      const trace = createTrace({
+        project_id: projectId,
+      });
+      await createTracesCh([trace]);
+
+      // Create a categorical score config with multiple possible values
+      const scoreConfig = await prisma.scoreConfig.create({
+        data: {
+          projectId: projectId,
+          name: "sentiment",
+          dataType: "CATEGORICAL",
+          categories: [
+            { label: "positive", value: 1 },
+            { label: "neutral", value: 0 },
+            { label: "negative", value: -1 },
+          ],
+        },
+      });
+
+      // Create only one actual score (subset of possible values)
+      const score = createTraceScore({
+        project_id: projectId,
+        trace_id: trace.id,
+        name: "sentiment",
+        string_value: "custom",
+        data_type: "CATEGORICAL",
+        config_id: scoreConfig.id,
+      });
+      await createScoresCh([score]);
+
+      // Get filter options
+      const filterOptions = await caller.traces.filterOptions({
+        projectId,
+      });
+
+      // Find the sentiment score in categorical scores
+      const sentimentScore = filterOptions.score_categories.find(
+        (score) => score.label === "sentiment",
+      );
+
+      expect(sentimentScore).toBeDefined();
+      expect(sentimentScore?.values).toEqual(
+        expect.arrayContaining(["custom", "positive", "neutral", "negative"]),
+      );
+      // Should include all possible values from config, not just the actual score value
+      expect(sentimentScore?.values).toHaveLength(4);
+    });
+  });
 });

web/src/__tests__/compileChatMessages.servertest.ts

@@ -0,0 +1,152 @@
+/** @jest-environment node */
+
+import { ChatMessageType, compileChatMessages, extractPlaceholderNames } from "@langfuse/shared";
+
+describe("compileChatMessages", () => {
+  it("should compile message placeholders with provided values", () => {
+    // Simulates how message placeholders would be compiled
+    // during execution (e.g., in playground or experiments)
+
+    const promptTemplate = [
+      { role: "system", content: "You are a helpful assistant." },
+      {
+        type: ChatMessageType.Placeholder,
+        name: "conversation_history"
+      },
+      { role: "user", content: "{{user_question}}" }
+    ];
+
+    const placeholderValues = {
+      conversation_history: [
+        { role: "user", content: "Hello!" },
+        { role: "assistant", content: "Hi there! How can I help you?" },
+        { role: "user", content: "What's the weather like?" }
+      ]
+    };
+
+    const textVariables = {
+      user_question: "Can you continue our conversation?"
+    };
+
+    // Simulate compilation logic that would happen in playground/experiments
+    const compiledMessages = compileChatMessages(
+      promptTemplate,
+      placeholderValues,
+      textVariables
+    );
+
+    expect(compiledMessages).toEqual([
+      { role: "system", content: "You are a helpful assistant." },
+      { role: "user", content: "Hello!" },
+      { role: "assistant", content: "Hi there! How can I help you?" },
+      { role: "user", content: "What's the weather like?" },
+      { role: "user", content: "Can you continue our conversation?" }
+    ]);
+  });
+
+  it("should throw error when placeholder value is missing", () => {
+    const promptTemplate = [
+      { role: "system", content: "You are a helpful assistant." },
+      {
+        type: ChatMessageType.Placeholder,
+        name: "missing_placeholder"
+      },
+      { role: "user", content: "Hello" }
+    ];
+
+    const placeholderValues = {};
+
+    expect(() => {
+      compileChatMessages(promptTemplate, placeholderValues);
+    }).toThrow("Missing value for message placeholder: missing_placeholder");
+  });
+
+  it("should throw error when placeholder messages lack required properties", () => {
+    const promptTemplate = [
+      {
+        type: ChatMessageType.Placeholder,
+        name: "invalid_messages"
+      }
+    ];
+
+    // Test missing role property
+    const placeholderValuesNoRole = {
+      invalid_messages: [
+        { content: "Hello" }
+      ]
+    };
+
+    expect(() => {
+      compileChatMessages(promptTemplate, placeholderValuesNoRole);
+    }).toThrow("Invalid message format in placeholder 'invalid_messages': messages must have 'role' and 'content' properties");
+
+    // Test missing content property
+    const placeholderValuesNoContent = {
+      invalid_messages: [
+        { role: "user" }
+      ]
+    };
+
+    expect(() => {
+      compileChatMessages(promptTemplate, placeholderValuesNoContent);
+    }).toThrow("Invalid message format in placeholder 'invalid_messages': messages must have 'role' and 'content' properties");
+  });
+
+  it("should compile placeholders without applying text substitutions when no variables provided", () => {
+    const promptTemplate = [
+      { role: "system", content: "You are a helpful assistant. {{system_var}}" },
+      {
+        type: ChatMessageType.Placeholder,
+        name: "history"
+      },
+      { role: "user", content: "{{user_var}}" }
+    ];
+
+    const placeholderValues = {
+      history: [
+        { role: "user", content: "Previous message with {{var}}" },
+        { role: "assistant", content: "Response with {{another_var}}" }
+      ]
+    };
+
+    // No text variables provided
+    const compiledMessages = compileChatMessages(promptTemplate, placeholderValues);
+
+    expect(compiledMessages).toEqual([
+      { role: "system", content: "You are a helpful assistant. {{system_var}}" },
+      { role: "user", content: "Previous message with {{var}}" },
+      { role: "assistant", content: "Response with {{another_var}}" },
+      { role: "user", content: "{{user_var}}" }
+    ]);
+  });
+
+  it("should extract all placeholder names from messages", () => {
+    const promptTemplate = [
+      { role: "system", content: "System message" },
+      {
+        type: ChatMessageType.Placeholder,
+        name: "history"
+      },
+      { role: "user", content: "User message" },
+      {
+        type: ChatMessageType.Placeholder,
+        name: "context"
+      }
+    ];
+
+    const placeholderNames = extractPlaceholderNames(promptTemplate);
+
+    expect(placeholderNames).toEqual(["history", "context"]);
+  });
+
+  it("should return empty array when no placeholders exist", () => {
+    const promptTemplate = [
+      { role: "system", content: "System message" },
+      { role: "user", content: "User message" }
+    ];
+
+    const placeholderNames = extractPlaceholderNames(promptTemplate);
+
+    expect(placeholderNames).toEqual([]);
+  });
+});

web/src/__tests__/dashboard-router-pivot-table.servertest.ts

@@ -0,0 +1,679 @@
+/**
+ * @fileoverview Integration tests for Pivot Table widget functionality in dashboard router
+ *
+ * This test suite validates the complete data pipeline for pivot table widgets:
+ * - Query generation and execution through executeQuery function
+ * - SQL generation by QueryBuilder for various pivot table configurations
+ * - Data transformation from raw query results to pivot table structure
+ * - Integration with ClickHouse database and error handling
+ *
+ * Test Coverage:
+ * - Zero dimension pivot tables (grand total only)
+ * - Single dimension pivot tables with subtotals
+ * - Two dimension pivot tables with nested structure
+ * - Row limiting functionality
+ * - Error handling for malformed queries
+ * - Integration with existing dashboard query infrastructure
+ */
+
+import { randomUUID } from "crypto";
+import {
+  createTrace,
+  createTracesCh,
+  createObservation,
+  createObservationsCh,
+} from "@langfuse/shared/src/server";
+import { type QueryType } from "@/src/features/query/types";
+import { executeQuery } from "@/src/features/dashboard/server/dashboard-router";
+import {
+  transformToPivotTable,
+  type DatabaseRow,
+} from "@/src/features/widgets/utils/pivot-table-utils";
+import { QueryBuilder } from "@/src/features/query/server/queryBuilder";
+
+describe("Dashboard Router - Pivot Table Integration", () => {
+  // Single project ID for all tests
+  const projectId = randomUUID();
+
+  // Time references for test data
+  const now = new Date();
+  const oneHourAgo = new Date(now.getTime() - 3600000);
+  const twoHoursAgo = new Date(now.getTime() - 7200000);
+  const threeDaysAgo = new Date(now.getTime() - 3 * 24 * 3600000);
+
+  // Time ranges for queries - ISO format for query builder
+  const defaultFromTime = threeDaysAgo.toISOString();
+  const defaultToTime = new Date(now.getTime() + 3600000).toISOString(); // 1 hour in future
+
+  // Test data statistics for verification
+  const testDataStats = {
+    totalTraces: 0,
+    environmentCounts: {} as Record<string, number>,
+    modelCounts: {} as Record<string, number>,
+    totalObservations: 0,
+  };
+
+  beforeAll(async () => {
+    // Create diverse test data for pivot table testing
+    const traces = [
+      // Production environment traces
+      ...Array(6)
+        .fill(0)
+        .map((_, i) =>
+          createTrace({
+            project_id: projectId,
+            name: "chat-completion",
+            environment: "production",
+            timestamp: now.getTime() - i * 10000,
+            user_id: `user-prod-${i}`,
+          }),
+        ),
+
+      // Development environment traces
+      ...Array(4)
+        .fill(0)
+        .map((_, i) =>
+          createTrace({
+            project_id: projectId,
+            name: "embeddings",
+            environment: "development",
+            timestamp: oneHourAgo.getTime() - i * 15000,
+            user_id: `user-dev-${i}`,
+          }),
+        ),
+
+      // Staging environment traces
+      ...Array(3)
+        .fill(0)
+        .map((_, i) =>
+          createTrace({
+            project_id: projectId,
+            name: "summarize",
+            environment: "staging",
+            timestamp: twoHoursAgo.getTime() - i * 20000,
+            user_id: `user-staging-${i}`,
+          }),
+        ),
+    ];
+
+    // Insert traces into ClickHouse
+    await createTracesCh(traces);
+
+    // Create observations with different models for each trace
+    const observations = [];
+
+    // Production observations - GPT models
+    for (let i = 0; i < 6; i++) {
+      const traceId = traces[i].id;
+      observations.push(
+        createObservation({
+          project_id: projectId,
+          trace_id: traceId,
+          name: "gpt-generation",
+          type: "generation",
+          environment: "production",
+          start_time: now.getTime() - i * 10000,
+          completion_start_time: now.getTime() - i * 10000 + 500,
+          end_time: now.getTime() - i * 10000 + 2000,
+          provided_model_name: i < 3 ? "gpt-4-turbo" : "gpt-3.5-turbo",
+        }),
+      );
+    }
+
+    // Development observations - Claude models
+    for (let i = 0; i < 4; i++) {
+      const traceId = traces[6 + i].id;
+      observations.push(
+        createObservation({
+          project_id: projectId,
+          trace_id: traceId,
+          name: "claude-generation",
+          type: "generation",
+          environment: "development",
+          start_time: oneHourAgo.getTime() - i * 15000,
+          completion_start_time: oneHourAgo.getTime() - i * 15000 + 800,
+          end_time: oneHourAgo.getTime() - i * 15000 + 3000,
+          provided_model_name: i < 2 ? "claude-3-opus" : "claude-3-sonnet",
+        }),
+      );
+    }
+
+    // Staging observations - Mixed models
+    for (let i = 0; i < 3; i++) {
+      const traceId = traces[10 + i].id;
+      observations.push(
+        createObservation({
+          project_id: projectId,
+          trace_id: traceId,
+          name: "mixed-generation",
+          type: "generation",
+          environment: "staging",
+          start_time: twoHoursAgo.getTime() - i * 20000,
+          completion_start_time: twoHoursAgo.getTime() - i * 20000 + 600,
+          end_time: twoHoursAgo.getTime() - i * 20000 + 2500,
+          provided_model_name: "gpt-4-turbo",
+        }),
+      );
+    }
+
+    // Insert observations into ClickHouse
+    await createObservationsCh(observations);
+
+    // Calculate test data statistics for verification
+    testDataStats.totalTraces = traces.length;
+    testDataStats.totalObservations = observations.length;
+
+    // Count by environment
+    traces.forEach((trace) => {
+      testDataStats.environmentCounts[trace.environment] =
+        (testDataStats.environmentCounts[trace.environment] || 0) + 1;
+    });
+
+    // Count by model
+    observations.forEach((obs) => {
+      if (obs.provided_model_name) {
+        testDataStats.modelCounts[obs.provided_model_name] =
+          (testDataStats.modelCounts[obs.provided_model_name] || 0) + 1;
+      }
+    });
+  });
+
+  describe("executeQuery function with pivot table configurations", () => {
+    it("should execute zero-dimension pivot table query (grand total only)", async () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: null,
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const result = await executeQuery(projectId, query);
+
+      // Verify basic query execution
+      expect(result).toBeDefined();
+      expect(Array.isArray(result)).toBe(true);
+
+      // Results should be grouped by time dimension when timeDimension is present
+      expect(result.length).toBeGreaterThan(0);
+
+      // Each row should have time_dimension and aggregated count
+      result.forEach((row) => {
+        expect(row).toHaveProperty("time_dimension");
+        expect(row).toHaveProperty("count_count");
+        expect(typeof row.count_count).toBe("string"); // ClickHouse returns numbers as strings
+      });
+
+      // Sum all counts to verify total
+      const totalCount = result.reduce(
+        (sum, row) => sum + parseInt(row.count_count as string),
+        0,
+      );
+      expect(totalCount).toBe(testDataStats.totalTraces);
+    });
+
+    it("should execute single-dimension pivot table query with environment grouping", async () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [{ field: "environment" }],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [{ field: "environment", direction: "asc" }],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const result = await executeQuery(projectId, query);
+
+      // Verify query execution
+      expect(result).toBeDefined();
+      expect(Array.isArray(result)).toBe(true);
+
+      // Should have one row per environment per time dimension
+      expect(result.length).toBeGreaterThan(0);
+
+      // Verify structure of results
+      result.forEach((row) => {
+        expect(row).toHaveProperty("environment");
+        expect(row).toHaveProperty("time_dimension");
+        expect(row).toHaveProperty("count_count");
+        expect(typeof row.environment).toBe("string");
+        expect(typeof row.count_count).toBe("string"); // ClickHouse returns numbers as strings
+      });
+
+      // Verify data accuracy by checking environment counts
+      const environmentTotals = result.reduce(
+        (acc, row) => {
+          const env = row.environment as string;
+          acc[env] =
+            ((acc[env] as number) || 0) + parseInt(row.count_count as string);
+          return acc;
+        },
+        {} as Record<string, number>,
+      );
+
+      expect(environmentTotals["production"]).toBe(
+        testDataStats.environmentCounts["production"],
+      );
+      expect(environmentTotals["development"]).toBe(
+        testDataStats.environmentCounts["development"],
+      );
+      expect(environmentTotals["staging"]).toBe(
+        testDataStats.environmentCounts["staging"],
+      );
+    });
+
+    it("should execute two-dimension pivot table query with environment and model grouping", async () => {
+      const query: QueryType = {
+        view: "observations",
+        dimensions: [{ field: "environment" }, { field: "providedModelName" }],
+        metrics: [
+          { measure: "count", aggregation: "count" },
+          { measure: "totalTokens", aggregation: "sum" },
+        ],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [
+          { field: "environment", direction: "asc" },
+          { field: "providedModelName", direction: "asc" },
+        ],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const result = await executeQuery(projectId, query);
+
+      // Verify query execution
+      expect(result).toBeDefined();
+      expect(Array.isArray(result)).toBe(true);
+      expect(result.length).toBeGreaterThan(0);
+
+      // Should have combinations of environment and model
+      result.forEach((row) => {
+        expect(row).toHaveProperty("environment");
+        expect(row).toHaveProperty("providedModelName");
+        expect(row).toHaveProperty("time_dimension");
+        expect(row).toHaveProperty("count_count");
+        expect(row).toHaveProperty("sum_totalTokens");
+
+        expect(typeof row.environment).toBe("string");
+        expect(typeof row.providedModelName).toBe("string");
+        expect(typeof row.count_count).toBe("string");
+        expect(typeof row.sum_totalTokens).toBe("string");
+      });
+
+      // Verify total observation count matches test data
+      const totalObservations = result.reduce(
+        (sum, row) => sum + parseInt(row.count_count as string),
+        0,
+      );
+      expect(totalObservations).toBe(testDataStats.totalObservations);
+    });
+
+    it("should handle empty results gracefully", async () => {
+      const futureTime = new Date(now.getTime() + 86400000).toISOString(); // 1 day in future
+      const farFutureTime = new Date(now.getTime() + 172800000).toISOString(); // 2 days in future
+
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [{ field: "environment" }],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: futureTime,
+        toTimestamp: farFutureTime,
+        orderBy: null,
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const result = await executeQuery(projectId, query);
+
+      // Should handle empty results without errors
+      expect(result).toBeDefined();
+      expect(Array.isArray(result)).toBe(true);
+      // Time dimension queries can return rows with 0 counts for time filling
+      expect(result.length).toBeGreaterThanOrEqual(0);
+    });
+  });
+
+  describe("Query Builder integration with pivot table configurations", () => {
+    it("should generate correct SQL for zero-dimension pivot table", () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: null,
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const queryBuilder = new QueryBuilder(query.chartConfig);
+      const { query: sql, parameters } = queryBuilder.build(query, projectId);
+
+      // Verify SQL generation
+      expect(sql).toBeDefined();
+      expect(typeof sql).toBe("string");
+      expect(parameters).toBeDefined();
+      expect(typeof parameters).toBe("object");
+
+      // SQL should contain GROUP BY for time dimension when timeDimension is present
+      expect(sql.toLowerCase()).toContain("group by");
+
+      // Should contain aggregation
+      expect(sql.toLowerCase()).toContain("count(");
+    });
+
+    it("should generate correct SQL for single-dimension pivot table", () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [{ field: "environment" }],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [{ field: "environment", direction: "asc" }],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const queryBuilder = new QueryBuilder(query.chartConfig);
+      const { query: sql, parameters } = queryBuilder.build(query, projectId);
+
+      // Verify SQL generation
+      expect(sql).toBeDefined();
+      expect(typeof sql).toBe("string");
+      expect(parameters).toBeDefined();
+
+      // SQL should contain GROUP BY for dimension
+      expect(sql.toLowerCase()).toContain("group by");
+      expect(sql.toLowerCase()).toContain("environment");
+
+      // Should contain ORDER BY
+      expect(sql.toLowerCase()).toContain("order by");
+    });
+
+    it("should generate correct SQL for two-dimension pivot table", () => {
+      const query: QueryType = {
+        view: "observations",
+        dimensions: [{ field: "environment" }, { field: "providedModelName" }],
+        metrics: [
+          { measure: "count", aggregation: "count" },
+          { measure: "totalTokens", aggregation: "sum" },
+        ],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [
+          { field: "environment", direction: "asc" },
+          { field: "providedModelName", direction: "asc" },
+        ],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const queryBuilder = new QueryBuilder(query.chartConfig);
+      const { query: sql, parameters } = queryBuilder.build(query, projectId);
+
+      // Verify SQL generation
+      expect(sql).toBeDefined();
+      expect(typeof sql).toBe("string");
+      expect(parameters).toBeDefined();
+
+      // SQL should contain GROUP BY for both dimensions
+      expect(sql.toLowerCase()).toContain("group by");
+      expect(sql.toLowerCase()).toContain("environment");
+      expect(sql.toLowerCase()).toContain("providedmodelname");
+
+      // Should contain multiple aggregations
+      expect(sql.toLowerCase()).toContain("count(");
+      expect(sql.toLowerCase()).toContain("sum(");
+
+      // Should contain ORDER BY for both dimensions
+      expect(sql.toLowerCase()).toContain("order by");
+    });
+  });
+
+  describe("End-to-end pivot table data transformation", () => {
+    it("should transform query results to pivot table structure for zero dimensions", async () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: null, // No time dimension for simpler test
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: null,
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      // Execute query to get raw data
+      const rawData = await executeQuery(projectId, query);
+
+      // Transform to pivot table structure
+      const pivotTableData = transformToPivotTable(rawData as DatabaseRow[], {
+        dimensions: [],
+        metrics: ["count_count"],
+        rowLimit: 20,
+      });
+
+      // Verify transformation
+      expect(pivotTableData).toBeDefined();
+      expect(Array.isArray(pivotTableData)).toBe(true);
+      expect(pivotTableData.length).toBe(1); // Should have only grand total
+
+      const totalRow = pivotTableData[0];
+      expect(totalRow.type).toBe("total");
+      expect(totalRow.level).toBe(0);
+      expect(totalRow.label).toBe("Total");
+      expect(totalRow.isTotal).toBe(true);
+      expect(totalRow.values).toHaveProperty("count_count");
+    });
+
+    it("should transform query results to pivot table structure for single dimension", async () => {
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [{ field: "environment" }],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: null, // No time dimension for simpler test
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [{ field: "environment", direction: "asc" }],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      // Execute query to get raw data
+      const rawData = await executeQuery(projectId, query);
+
+      // Transform to pivot table structure
+      const pivotTableData = transformToPivotTable(rawData as DatabaseRow[], {
+        dimensions: ["environment"],
+        metrics: ["count_count"],
+        rowLimit: 20,
+      });
+
+      // Verify transformation
+      expect(pivotTableData).toBeDefined();
+      expect(Array.isArray(pivotTableData)).toBe(true);
+      expect(pivotTableData.length).toBeGreaterThan(1); // Should have data rows + total
+
+      // Should have data rows for each environment
+      const dataRows = pivotTableData.filter((row) => row.type === "data");
+      const totalRow = pivotTableData.find((row) => row.type === "total");
+
+      expect(dataRows.length).toBeGreaterThan(0);
+      expect(totalRow).toBeDefined();
+      expect(totalRow!.isTotal).toBe(true);
+
+      // Verify data row structure
+      dataRows.forEach((row) => {
+        expect(row.type).toBe("data");
+        expect(row.level).toBe(0);
+        expect(row.values).toHaveProperty("count_count");
+        expect(typeof row.values.count_count).toBe("number");
+      });
+    });
+
+    it("should transform query results to pivot table structure for two dimensions", async () => {
+      const query: QueryType = {
+        view: "observations",
+        dimensions: [{ field: "environment" }, { field: "providedModelName" }],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: null, // No time dimension for simpler test
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: [
+          { field: "environment", direction: "asc" },
+          { field: "providedModelName", direction: "asc" },
+        ],
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      // Execute query to get raw data
+      const rawData = await executeQuery(projectId, query);
+
+      // Transform to pivot table structure
+      const pivotTableData = transformToPivotTable(rawData as DatabaseRow[], {
+        dimensions: ["environment", "providedModelName"],
+        metrics: ["count_count"],
+        rowLimit: 20,
+      });
+
+      // Verify transformation
+      expect(pivotTableData).toBeDefined();
+      expect(Array.isArray(pivotTableData)).toBe(true);
+      expect(pivotTableData.length).toBeGreaterThan(1);
+
+      // Should have nested structure with data rows, subtotals, and grand total
+      const dataRows = pivotTableData.filter((row) => row.type === "data");
+      const subtotalRows = pivotTableData.filter(
+        (row) => row.type === "subtotal",
+      );
+      const totalRow = pivotTableData.find((row) => row.type === "total");
+
+      expect(dataRows.length).toBeGreaterThan(0);
+      expect(totalRow).toBeDefined();
+
+      // Verify indentation levels
+      dataRows.forEach((row) => {
+        expect(row.level).toBe(1); // Second level for two dimensions
+        expect(row.values).toHaveProperty("count_count");
+      });
+
+      if (subtotalRows.length > 0) {
+        subtotalRows.forEach((row) => {
+          expect(row.level).toBe(0); // First level subtotals
+          expect(row.isSubtotal).toBe(true);
+        });
+      }
+
+      expect(totalRow!.level).toBe(0);
+      expect(totalRow!.isTotal).toBe(true);
+    });
+  });
+
+  describe("Error handling and edge cases", () => {
+    it("should handle empty results gracefully in transformation", async () => {
+      // Transform empty data to pivot table structure
+      const pivotTableData = transformToPivotTable([], {
+        dimensions: ["environment"],
+        metrics: ["count_count"],
+        rowLimit: 20,
+      });
+
+      // Should handle empty data gracefully
+      expect(pivotTableData).toBeDefined();
+      expect(Array.isArray(pivotTableData)).toBe(true);
+
+      // Should still have total row with zero values
+      expect(pivotTableData.length).toBe(1);
+      const totalRow = pivotTableData[0];
+      expect(totalRow.type).toBe("total");
+      expect(totalRow.values.count_count).toBe(0);
+    });
+
+    it("should handle missing project data gracefully", async () => {
+      const nonExistentProjectId = randomUUID();
+
+      const query: QueryType = {
+        view: "traces",
+        dimensions: [],
+        metrics: [{ measure: "count", aggregation: "count" }],
+        filters: [],
+        timeDimension: {
+          granularity: "day",
+        },
+        fromTimestamp: defaultFromTime,
+        toTimestamp: defaultToTime,
+        orderBy: null,
+        chartConfig: {
+          type: "PIVOT_TABLE",
+          row_limit: 20,
+        },
+      };
+
+      const result = await executeQuery(nonExistentProjectId, query);
+
+      // Should return results even for non-existent project (time fill creates rows)
+      expect(result).toBeDefined();
+      expect(Array.isArray(result)).toBe(true);
+      expect(result.length).toBeGreaterThanOrEqual(0);
+    });
+  });
+});

web/src/__tests__/pivot-table.clienttest.tsx

@@ -0,0 +1,383 @@
+/**
+ * @fileoverview Unit Tests for PivotTable React Component
+ *
+ * Comprehensive test suite for the PivotTable component functionality including:
+ * - Component rendering with various data scenarios
+ * - Proper styling and CSS class application
+ * - Indentation behavior for nested dimensions
+ * - Empty data and error state handling
+ * - Metric value formatting and display
+ * - Column header formatting
+ *
+ * Uses Jest and React Testing Library for component testing.
+ * This test focuses on component behavior rather than data transformation logic.
+ */
+
+import React from "react";
+import { render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+
+import {
+  PivotTable,
+  type PivotTableProps,
+} from "@/src/features/widgets/chart-library/PivotTable";
+import { type DataPoint } from "@/src/features/widgets/chart-library/chart-props";
+
+describe("PivotTable Component", () => {
+  describe("Basic Rendering", () => {
+    test("renders table with simple data", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "gpt-4",
+          metric: 100,
+        },
+        {
+          time_dimension: "2024-01-01",
+          dimension: "gpt-3.5",
+          metric: 75,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      // Check table structure exists
+      expect(screen.getByRole("table")).toBeInTheDocument();
+      // Should have dimension column and metric column
+      expect(screen.getAllByRole("columnheader")).toHaveLength(2);
+    });
+
+    test("displays data when no configuration provided", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 50,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+      };
+
+      render(<PivotTable {...props} />);
+
+      // Should still render a table
+      expect(screen.getByRole("table")).toBeInTheDocument();
+    });
+  });
+
+  describe("Column Header Formatting", () => {
+    test("formats single dimension header correctly", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model_name"],
+          metrics: ["request_count"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(
+        screen.getByRole("columnheader", { name: "Model Name" }),
+      ).toBeInTheDocument();
+      expect(
+        screen.getByRole("columnheader", { name: "Request Count" }),
+      ).toBeInTheDocument();
+    });
+
+    test("formats multiple dimension headers correctly", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model_name", "time_period"],
+          metrics: ["request_count", "avg_duration"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(
+        screen.getByRole("columnheader", { name: "Model Name / Time Period" }),
+      ).toBeInTheDocument();
+      expect(
+        screen.getByRole("columnheader", { name: "Request Count" }),
+      ).toBeInTheDocument();
+      expect(
+        screen.getByRole("columnheader", { name: "Avg Duration" }),
+      ).toBeInTheDocument();
+    });
+
+    test("defaults to 'Dimension' when no dimensions configured", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: [],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(
+        screen.getByRole("columnheader", { name: "Dimension" }),
+      ).toBeInTheDocument();
+    });
+  });
+
+  describe("Empty Data Handling", () => {
+    test("displays 'No data available' for empty data array", () => {
+      const props: PivotTableProps = {
+        data: [],
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(screen.getByText("No data available")).toBeInTheDocument();
+      expect(screen.queryByRole("table")).not.toBeInTheDocument();
+    });
+
+    test("displays 'No data available' for null data", () => {
+      const props: PivotTableProps = {
+        data: null as any,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(screen.getByText("No data available")).toBeInTheDocument();
+      expect(screen.queryByRole("table")).not.toBeInTheDocument();
+    });
+
+    test("displays 'No data available' for undefined data", () => {
+      const props: PivotTableProps = {
+        data: undefined as any,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      expect(screen.getByText("No data available")).toBeInTheDocument();
+      expect(screen.queryByRole("table")).not.toBeInTheDocument();
+    });
+  });
+
+  describe("Responsive Design", () => {
+    test("includes overflow-auto class for responsive scrolling", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      const { container } = render(<PivotTable {...props} />);
+
+      const pivotTableContainer = container.firstChild as HTMLElement;
+      expect(pivotTableContainer).toHaveClass("h-full", "overflow-auto");
+    });
+  });
+
+  describe("Accessibility", () => {
+    test("provides proper table structure for screen readers", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "gpt-4",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      // Check table has proper semantic structure
+      expect(screen.getByRole("table")).toBeInTheDocument();
+      expect(screen.getAllByRole("columnheader")).toHaveLength(2);
+      // Should have at least header row
+      expect(screen.getAllByRole("row").length).toBeGreaterThanOrEqual(1);
+    });
+
+    test("provides proper cell associations", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      render(<PivotTable {...props} />);
+
+      // Header cells should be properly associated
+      const headers = screen.getAllByRole("columnheader");
+      expect(headers.length).toBeGreaterThan(0);
+
+      // Each header should be part of a row
+      headers.forEach((header) => {
+        expect(header.closest("tr")).toBeInTheDocument();
+      });
+    });
+  });
+
+  describe("Component Props", () => {
+    test("handles missing config gracefully", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        // No config provided
+      };
+
+      render(<PivotTable {...props} />);
+
+      // Should still render a table
+      expect(screen.getByRole("table")).toBeInTheDocument();
+    });
+
+    test("passes accessibilityLayer prop correctly", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test",
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        accessibilityLayer: true,
+      };
+
+      // Should render without error
+      expect(() => render(<PivotTable {...props} />)).not.toThrow();
+    });
+  });
+
+  describe("Data Processing Integration", () => {
+    test("handles various metric data types", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test1",
+          metric: 100, // number
+        },
+        {
+          time_dimension: "2024-01-01",
+          dimension: "test2",
+          metric: [
+            [10, 20],
+            [30, 40],
+          ], // nested array
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      // Should render without error
+      expect(() => render(<PivotTable {...props} />)).not.toThrow();
+      expect(screen.getByRole("table")).toBeInTheDocument();
+    });
+
+    test("handles missing dimension data", () => {
+      const data: DataPoint[] = [
+        {
+          time_dimension: "2024-01-01",
+          dimension: undefined, // missing dimension
+          metric: 100,
+        },
+      ];
+
+      const props: PivotTableProps = {
+        data,
+        config: {
+          dimensions: ["model"],
+          metrics: ["metric"],
+        },
+      };
+
+      // Should render without error
+      expect(() => render(<PivotTable {...props} />)).not.toThrow();
+      expect(screen.getByRole("table")).toBeInTheDocument();
+    });
+  });
+});

web/src/__tests__/prompts.v1.servertest.ts

@@ -8,7 +8,7 @@ import {
   LegacyPromptSchema,
   PromptType,
   type LegacyValidatedPrompt,
-} from "@/src/features/prompts/server/utils/validation";
+} from "@langfuse/shared";
 import { getObservationById } from "@langfuse/shared/src/server";
 
 describe("/api/public/prompts API Endpoint", () => {

web/src/__tests__/prompts.v2.servertest.ts

@@ -3,20 +3,22 @@
 import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
-import { type Prompt } from "@langfuse/shared";
 import {
   PromptSchema,
   PromptType,
   type ValidatedPrompt,
-} from "@/src/features/prompts/server/utils/validation";
+  type ChatMessage,
+  type Prompt,
+} from "@langfuse/shared";
 import { parsePromptDependencyTags } from "@langfuse/shared";
-import { nanoid } from "ai";
+import { generateId, nanoid } from "ai";
 
 import { type PromptsMetaResponse } from "@/src/features/prompts/server/actions/getPromptsMeta";
 import {
   createOrgProjectAndApiKey,
   getObservationById,
   MAX_PROMPT_NESTING_DEPTH,
+  ChatMessageType,
 } from "@langfuse/shared/src/server";
 import { randomUUID } from "node:crypto";
 
@@ -452,6 +454,52 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(validatedPrompt.commitMessage).toBe("chore: setup initial prompt");
     });
 
+    it("should create and fetch a chat prompt with message placeholders", async () => {
+      const promptName = `prompt-name-message-placeholders${generateId()}`;
+      const commitMessage = "feat: add message placeholders support";
+      const chatMessages = [
+        { role: "system", content: "You are a helpful assistant with conversation context." },
+        {
+          type: ChatMessageType.Placeholder,
+          name: "conversation_history"
+        },
+        { role: "user", content: "{{user_question}}" }
+      ];
+
+      const response = await makeAPICall("POST", baseURI, {
+        name: promptName,
+        prompt: chatMessages,
+        type: "chat",
+        labels: ["production"],
+        commitMessage: commitMessage
+      });
+
+      expect(response.status).toBe(201);
+
+      const { body: fetchedPrompt } = await makeAPICall(
+        "GET",
+        `${baseURI}/${promptName}`,
+        undefined,
+      );
+
+      const validatedPrompt = validatePrompt(fetchedPrompt);
+
+      expect(validatedPrompt.name).toBe(promptName);
+      expect(validatedPrompt.prompt).toEqual(chatMessages);
+      expect(validatedPrompt.type).toBe("chat");
+      expect(validatedPrompt.version).toBe(1);
+      expect(validatedPrompt.labels).toEqual(["production", "latest"]);
+      expect(validatedPrompt.createdBy).toBe("API");
+      expect(validatedPrompt.config).toEqual({});
+      expect(validatedPrompt.commitMessage).toBe(commitMessage);
+
+      // Verify the placeholder message structure is preserved
+      const messages = validatedPrompt.prompt as ChatMessage[];
+      const placeholderMessage = messages[1] as { type: ChatMessageType.Placeholder; name: string };
+      expect(placeholderMessage.type).toBe(ChatMessageType.Placeholder);
+      expect(placeholderMessage.name).toBe("conversation_history");
+    });
+
     it("should fail if chat prompt has string prompt", async () => {
       const promptName = "prompt-name";
       const response = await makeAPICall("POST", baseURI, {
@@ -705,7 +753,9 @@ describe("/api/public/v2/prompts API Endpoint", () => {
         });
         expect(response.status).toBe(400);
         expect(response.body.message).toBe("Invalid request data");
-        const hasExpectedMessage = JSON.stringify(response.body.error).includes(`"message":"${expectedError}"`);
+        const hasExpectedMessage = JSON.stringify(response.body.error).includes(
+          `"message":"${expectedError}"`,
+        );
         expect(hasExpectedMessage).toBe(true);
       };
 
@@ -723,10 +773,19 @@ describe("/api/public/v2/prompts API Endpoint", () => {
 
       it("should reject invalid prompt names", async () => {
         // Test invalid patterns
-        await testInvalidName("/invalid-name", "Name cannot start with a slash");
+        await testInvalidName(
+          "/invalid-name",
+          "Name cannot start with a slash",
+        );
         await testInvalidName("invalid-name/", "Name cannot end with a slash");
-        await testInvalidName("invalid//name", "Name cannot contain consecutive slashes");
-        await testInvalidName("invalid|name", "Prompt name cannot contain '|' character");
+        await testInvalidName(
+          "invalid//name",
+          "Name cannot contain consecutive slashes",
+        );
+        await testInvalidName(
+          "invalid|name",
+          "Prompt name cannot contain '|' character",
+        );
         await testInvalidName("new", "Prompt name cannot be 'new'");
         await testInvalidName("", "Enter a name");
       });
@@ -848,7 +907,68 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       // Verify the name with slashes is preserved
       expect(validatedPrompt.name).toBe(promptName);
       expect(validatedPrompt.name).toContain("/");
-      expect(validatedPrompt.prompt).toBe("This is a prompt in a folder structure");
+      expect(validatedPrompt.prompt).toBe(
+        "This is a prompt in a folder structure",
+      );
+    });
+
+    it("should prevent creating a prompt with both a variable and placeholder with the same name", async () => {
+      const promptName = "prompt-same-name-conflict-" + nanoid();
+
+      // Try to create a prompt where the same name is used as both a variable and a placeholder
+      const response = await makeAPICall("POST", baseURI, {
+        name: promptName,
+        prompt: [
+          { role: "system", content: "Hello {{userName}}" },
+          { type: ChatMessageType.Placeholder, name: "userName" },
+          { role: "user", content: "How are you?" }
+        ],
+        type: "chat",
+      });
+
+      // This should fail with a 400 error
+      expect(response.status).toBe(400);
+      expect(response.body).toHaveProperty("error");
+      expect(response.body).toHaveProperty("message");
+      // @ts-expect-error
+      expect(response.body.message).toContain("variables and placeholders must be unique");
+      // @ts-expect-error
+      expect(response.body.message).toContain("userName");
+    });
+
+    it("should allow creating a new version of a prompt with placeholder names that conflict a variable name in a previous version", async () => {
+      const promptName = "prompt-with-variable-conflict-" + nanoid();
+
+      // First, create a chat prompt with a message variable
+      const v1Response = await makeAPICall("POST", baseURI, {
+        name: promptName,
+        prompt: [
+          { role: "system", content: "You are a helpful {{conversationHistory}}" },
+          { role: "user", content: "Continue our conversation" }
+        ],
+        type: "chat",
+        labels: ["production"],
+      });
+
+      expect(v1Response.status).toBe(201);
+
+      // Try to create a new version with a text variable that has the same name as the placeholder
+      const v2Response = await makeAPICall("POST", baseURI, {
+        name: promptName,
+        prompt: [
+          { role: "system", content: "You are a helpful assistant with context: {{newHistory}}" },
+          { type: "placeholder", name: "conversationHistory" },
+          { role: "user", content: "Continue our conversation" }
+        ],
+        type: "chat"
+      });
+
+      // This should succeed, we allow cross-version name reuse
+      expect(v2Response.status).toBe(201);
+      expect(v2Response.body).toHaveProperty("id");
+      expect(v2Response.body).toHaveProperty("version");
+      // @ts-expect-error - Response body type is flexible for testing
+      expect(v2Response.body.version).toBe(2);
     });
   });