chore: release v3.78.2

* fix(scores): update score validation logic to accept inclusive ranges for score config validation

* chore: typo

.cursor/Dockerfile

@@ -0,0 +1,54 @@
+FROM node:20
+
+# ---------- System packages --------------------------------------------------
+# The buildpack-deps base already ships git, build-essential, python, etc.
+# Add a few extra tools handy during Langfuse development.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      openssl \
+      wget \
+      curl \
+      ca-certificates \
+      postgresql-client \
+      redis-tools \
+      less nano \
+      sudo \
+    && rm -rf /var/lib/apt/lists/*
+
+# ---------- Docker -----------------------------------------------------------
+# Install Docker for background agents that need container capabilities
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
+    sh get-docker.sh && \
+    rm get-docker.sh
+
+# ---------- pnpm -------------------------------------------------------------
+# Langfuse monorepo relies on pnpm 9.5.0 (see CONTRIBUTING.md)
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable && \
+    corepack prepare pnpm@9.5.0 --activate
+
+# ---------- golang-migrate ----------------------------------------------------
+# CLI used for database migrations during development.
+ENV MIGRATE_VERSION=4.18.2
+RUN wget -qO- "https://github.com/golang-migrate/migrate/releases/download/v${MIGRATE_VERSION}/migrate.linux-amd64.tar.gz" \
+    | tar -xz -C /usr/local/bin && \
+    chmod +x /usr/local/bin/migrate
+
+# ---------- Non-root user -----------------------------------------------------
+# Create non-root user with sudo privileges and docker group access
+RUN useradd -ms /bin/bash ubuntu && \
+    usermod -aG sudo ubuntu && \
+    usermod -aG docker ubuntu && \
+    echo "ubuntu ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Pre-create pnpm store and set correct ownership to avoid first-run cost & permission issues
+RUN pnpm store path > /dev/null && \
+    chown -R ubuntu:ubuntu /pnpm
+
+USER ubuntu
+WORKDIR /home/ubuntu
+ENV HOME=/home/ubuntu
+
+# Container starts with a bash shell ready for hacking.
+CMD ["bash"]

.cursor/environment.json

@@ -0,0 +1,14 @@
+{
+  "install": "cp .env.dev.example .env && pnpm i",
+  "build": {
+    "context": ".",
+    "dockerfile": "Dockerfile"
+  },
+  "start": "sudo service docker start",
+  "terminals": [
+    {
+      "name": "dev server",
+      "command": "pnpm run dx-f"
+    }
+  ]
+}

.cursor/rules/frontend-features.mdc

@@ -11,7 +11,7 @@ alwaysApply: true
 ## File Structure
 
 - We generally put all code related to a net-new feature into a folder within web/src/features.
-- Checkout other features to learn about the common structure.
+- Check out other features to learn about the common structure.
 
 ## API for frontend features
 

.cursor/rules/general-info.mdc

@@ -0,0 +1,9 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+# General rules
+
+- Linting in this repo only works if the development server is running
+- Always run the full mono-repo via `pnpm run dx` (use `pnpm dx-f` when you run this in a background agent). Thereby the database will also be seeded.

.devcontainer.json

@@ -0,0 +1,6 @@
+{
+    "name": "langfuse-development",
+    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+    "onCreateCommand": "npm install -g pnpm@9.5.0",
+    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+}

.env.dev-redis-cluster.example

@@ -0,0 +1,84 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="bitnami"
+REDIS_CLUSTER_ENABLED="true"
+REDIS_CLUSTER_NODES="127.0.0.1:6370,127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373,127.0.0.1:6374,127.0.0.1:6375"
+LANGFUSE_INGESTION_QUEUE_SHARD_COUNT=8
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.dev.example

@@ -75,7 +75,11 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 # openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+# For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
+# LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10

.env.prod.example

@@ -175,6 +175,10 @@ OTEL_SERVICE_NAME="langfuse"
 # REDIS_CONNECTION_STRING=
 # REDIS_ENABLE_AUTO_PIPELINING=
 
+# Redis Cluster configuration (optional)
+# REDIS_CLUSTER_ENABLED=false
+# REDIS_CLUSTER_NODES=redis-node1:6379,redis-node2:6379,redis-node3:6379
+
 # Cache configuration
 # LANGFUSE_CACHE_API_KEY_ENABLED=
 # LANGFUSE_CACHE_API_KEY_TTL_SECONDS=
@@ -251,6 +255,8 @@ OTEL_SERVICE_NAME="langfuse"
 # Plain Chat
 # NEXT_PUBLIC_PLAIN_APP_ID=
 # PLAIN_AUTHENTICATION_SECRET=
+# PLAIN_API_KEY=
+# PLAIN_CARDS_API_TOKEN=
 
 # Admin API
 # ADMIN_API_KEY=

.github/workflows/pipeline.yml

@@ -171,12 +171,12 @@ jobs:
     needs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
-    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.deploy-mode }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
-        blob-provider: ["", "-azure"]
+        deploy-mode: ["", "-azure", "-redis-cluster"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -208,8 +208,8 @@ jobs:
           pnpm install
       - name: Load default env
         run: |
-          cp .env.dev${{ matrix.blob-provider }}.example .env
-          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          cp .env.dev${{ matrix.deploy-mode }}.example .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.deploy-mode }}.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
           echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100" >> .env
@@ -217,7 +217,7 @@ jobs:
           echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
       - name: Run dev containers
         run: |
-          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.deploy-mode }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
@@ -250,12 +250,12 @@ jobs:
     needs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.deploy-mode }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
-        blob-provider: ["", "-azure"]
+        deploy-mode: ["", "-azure", "-redis-cluster"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -287,12 +287,12 @@ jobs:
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev${{ matrix.blob-provider }}.example .env
-          cp .env.dev${{ matrix.blob-provider }}.example web/.env
-          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
+          cp .env.dev${{ matrix.deploy-mode }}.example .env
+          cp .env.dev${{ matrix.deploy-mode }}.example web/.env
+          cp .env.dev${{ matrix.deploy-mode }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.deploy-mode }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status

.gitignore

@@ -4,6 +4,7 @@
 /node_modules
 /.pnp
 .pnp.js
+migrate
 
 # testing
 /coverage
@@ -39,6 +40,7 @@ yarn-error.log*
 .env*
 !.env.dev.example
 !.env.dev-azure.example
+!.env.dev-redis-cluster.example
 !.env.prod.example
 
 # vercel

CLAUDE.md

@@ -0,0 +1,188 @@
+# CLAUDE.md
+
+## Project Overview
+
+Langfuse is an open-source LLM engineering platform that helps teams collaboratively develop, monitor, evaluate, and debug AI applications.
+The main feature areas are tracing, evals and prompt management. Langfuse consists of the web application (this repo), documentation, python SDK and javascript/typescript SDK.
+This repo contains the web application, worker, and supporting packages but notably not the JS nor Python client SDKs.
+
+## Repository Structure
+High level structure. There are more folders (eg for hooks etc).
+```
+langfuse/
+├── web/                     # Next.js 14 frontend/backend application
+│   ├── src/
+│   │   ├── components/     # Reusable UI components (shadcn/ui)
+│   │   ├── features/       # Feature-specific code organized by domain
+│   │   ├── pages/          # Next.js pages (Pages Router)
+│   │   └── server/         # tRPC API routes and server logic
+│   └── public/             # Static assets
+├── worker/                  # Express.js background job processor
+│   └── src/
+│       ├── queues/         # BullMQ job queues
+│       └── services/       # Background processing services
+├── packages/
+│   ├── shared/             # Shared types, schemas, and utilities
+│   │   ├── prisma/         # Database schema and migrations
+│   │   └── src/            # Shared TypeScript code
+│   ├── config-eslint/      # ESLint configuration
+│   └── config-typescript/  # TypeScript configuration
+├── ee/                     # Enterprise Edition features
+├── fern/                   # API documentation and OpenAPI specs
+├── generated/              # Auto-generated client code
+└── scripts/                # Development and deployment scripts
+```
+
+## Repository Architecture
+This is a **pnpm + Turbo monorepo** with the following key packages:
+
+### Core Applications
+- **`/web/`** - Next.js 14 application (Pages Router) providing both frontend UI and backend APIs
+- **`/worker/`** - Express.js background job processing server
+- **`/packages/shared/`** - Shared database schema, types, and utilities
+
+### Supporting Packages
+- **`/ee/`** - Enterprise Edition features (separate licensing)
+- **`/packages/config-eslint/`** - Shared ESLint configuration
+- **`/packages/config-typescript/`** - Shared TypeScript configuration
+
+## Development Commands
+
+### Development
+```sh
+pnpm i               # Install dependencies
+pnpm run dev         # Start all services (web + worker)
+pnpm run dev:web     # Web app only (localhost:3000) - **used in most cases!**
+pnpm run dev:worker  # Worker only
+pnpm run dx          # Full initial setup: install deps, reset DBs, resets node modules, seed data, start dev. USE SPARINGLY AS IT WIPES THE DATABASE & node_modules
+```
+
+### Database Management
+database commands are to be run in the `packages/shared/` folder.
+```sh
+pnpm run db:generate       # Build prisma models
+pnpm run db:migrate        # Run Prisma migrations
+pnpm run db:reset          # Reset and reseed databases
+pnpm run db:seed           # Seed with example data
+```
+
+### Infrastructure
+```sh
+pnpm run infra:dev:up      # Start Docker services (PostgreSQL, ClickHouse, Redis, MinIO)
+pnpm run infra:dev:down    # Stop Docker services
+```
+
+### Building
+```sh
+pnpm --filter=PACKAGE_NAME run build  # Runs the build command, will show real typescript errors etc.
+```
+
+### Testing in Web Package
+The web package uses JEST for unit tests.
+Depending on the file location (sync, async)
+`web` related tests must go into the `web/src/__tests__/` folder.
+```sh
+pnpm test-sync --testPathPattern="$FILE_LOCATION_PATTERN" --testNamePattern="$TEST_NAME_PATTERN"
+pnpm test-async --testPathPattern="$FILE_LOCATION_PATTERN" --testNamePattern="$TEST_NAME_PATTERN"
+```
+
+### Testing in the Worker Package
+The worker uses `vitest` for unit tests.
+```sh
+pnpm run test --filter=worker -- $TEST_FILE_NAME -t "$TEST_NAME"
+```
+
+### Utilities
+```bash
+pnpm run nuke              # Remove all node_modules, build files, wipe database, docker containers. **USE WITH CAUTION**
+```
+
+## Technology Stack
+
+### Web Application (`/web/`)
+- **Framework**: Next.js 14 (Pages Router)
+- **APIs**: tRPC (type-safe client-server communication) + REST APIs for public access
+- **Authentication**: NextAuth.js/Auth.js
+- **Database**: Prisma ORM with PostgreSQL
+- **Analytics Database**: ClickHouse (high-volume trace data)
+- **Validation**: Zod schemas, we use zodv4 (always import from `zod/v4`)
+- **Styling**: Tailwind CSS with CSS variables for theming
+- **Components**: shadcn/ui (Radix UI primitives)
+- **State Management**: TanStack Query (React Query) + tRPC
+- **Charts**: Tremor, Recharts
+
+### Worker Application (`/worker/`)
+- **Framework**: Express.js
+- **Queue System**: BullMQ with Redis
+- **Purpose**: Async processing (data ingestion, evaluations, exports, integrations)
+
+### Infrastructure
+- **Primary Database**: PostgreSQL (via Prisma ORM)
+- **Analytics Database**: ClickHouse
+- **Cache/Queues**: Redis
+- **Blob Storage**: MinIO/S3
+
+## Development Guidelines
+
+### Frontend Features
+- All new features go in `/web/src/features/[feature-name]/`
+- Use tRPC for full-stack features (entry point: `web/src/server/api/root.ts`)
+- Follow existing feature structure for consistency
+- Use shadcn/ui components from `@/src/components/ui`
+- Custom reusable components go in `@/src/components`
+
+### Public API Development
+- All public API routes in `/web/src/pages/api/public`
+- Use `withMiddlewares.ts` wrapper
+- Define types in `/web/src/features/public-api/types` with strict Zod v4 objects
+- Add end-to-end tests (see `datasets-api.servertest.ts`)
+- Manually update Fern API specs in `/fern/`, then regenerate OpenAPI spec via Fern CLI
+
+### Authorization & RBAC
+- Check `/web/src/features/rbac/README.md` for authorization patterns
+- Implement proper entitlements checking (see `/web/src/features/entitlements/README.md`)
+
+### Database
+- **Dual database system**: PostgreSQL (primary) + ClickHouse (analytics)
+- Use `golang-migrate` CLI for database migrations
+- All database operations go through Prisma ORM for PostgreSQL
+- Foreign key relationships may not be enforced in schema to allow unordered ingestion
+
+### Testing
+- Jest for API tests, Playwright for E2E tests
+- For backend/API changes, tests must pass before pushes
+- Add tests for new API endpoints and features
+
+### Code Conventions
+- **Pages Router** (not App Router)
+- Follow conventional commits on main branch
+- Use CSS variables for theming (supports auto dark/light mode)
+- TypeScript throughout
+- Zod v4 for all input validation
+
+## Environment Setup
+
+- **Node.js**: Version 20 (specified in `.nvmrc`)
+- **Package Manager**: pnpm v9.5.0
+- **Database Dependencies**: Docker for local PostgreSQL, ClickHouse, Redis, MinIO
+- **Environment**: Copy `.env.dev.example` to `.env`
+
+## Login for Development
+
+When running locally with seed data:
+- Username: `demo@langfuse.com`
+- Password: `password`
+- Demo project URL: `http://localhost:3000/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a`
+
+## Linear MCP
+To get a project, use the `get_project` capability with the full project name as it is in the title.
+- bad: message-placeholder-in-chat-messages-2beb6f02ec48
+- good: Message placeholder in chat messages
+
+## Front-end Tips
+
+### Window Location Handling
+- Whenever you want to use or do use window.location..., ensure that you also add proper handling for a custom basePath
+
+## TypeScript Best Practices
+- In TypeScript, if possible, don't use the `any` type

CONTRIBUTING.md

@@ -42,7 +42,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
   - NextAuth.js / Auth.js
   - tRPC: Frontend APIs
   - Prisma ORM
-  - Zod
+  - Zod v4
   - Tailwind CSS
   - shadcn/ui tailwind components (using Radix and tanstack)
   - Fern: generate OpenAPI spec and Pydantic models
@@ -58,16 +58,30 @@ See this [diagram](https://langfuse.com/self-hosting#architecture) for an overvi
 ### Network Overview
 
 ```mermaid
-flowchart LR
-   Browser ---|Web UI & TRPC API| App
-   Integrations/SDKs ---|Public HTTP API| App
-   subgraph i1["Application Network"]
-      App["Langfuse Application"]
-   end
-   subgraph i2["Database Network"]
-      DB["Postgres Database"]
-   end
-   App --- DB
+flowchart TB
+    User["UI, API, SDKs"]
+    subgraph vpc["VPC"]
+        Web["Web Server<br/>(langfuse/langfuse)"]
+        Worker["Async Worker<br/>(langfuse/worker)"]
+        Postgres["Postgres - OLTP<br/>(Transactional Data)"]
+        Cache["Redis/Valkey<br/>(Cache, Queue)"]
+        Clickhouse["Clickhouse - OLAP<br/>(Observability Data)"]
+        S3["S3 / Blob Storage<br/>(Raw events, multi-modal attachments)"]
+    end
+    LLM["LLM API/Gateway<br/>(optional)"]
+
+    User --> Web
+    Web --> S3
+    Web --> Postgres
+    Web --> Cache
+    Web --> Clickhouse
+    Web -.->|"optional for playground"| LLM
+
+    Cache --> Worker
+    Worker --> Clickhouse
+    Worker --> Postgres
+    Worker --> S3
+    Worker -.->|"optional for evals"| LLM
 ```
 
 ### Database Overview
@@ -116,10 +130,11 @@ Requirements
     cp .env.dev.example .env
    ```
 
-4. Run the entire infrastructure in dev mode
+4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it.
 
    ```bash
-   pnpm run dx
+   pnpm run dx # first run only (resets db, node_modules, ...)
+   pnpm run dev # any subsequent runs
    ```
 
    You will be asked whether you want to reset Postgres and ClickHouse. Confirm both with 'Y' and press enter.
@@ -134,6 +149,12 @@ Requirements
    - Username: `demo@langfuse.com`
    - Password: `password`
 
+
+To get comprehensive example data, you can use the `seed` command:
+```sh
+pnpm run db:seed:examples
+```
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -183,23 +204,41 @@ Requirements
 
 On the main branch, we adhere to the best practices of [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/). All pull requests and branches are squash-merged to maintain a clean and readable history. This approach ensures the addition of a conventional commit message when merging contributions.
 
-## Test the public API
+## Running Unit Tests
 
-The API is tested using Jest. With the development server running, you can run the tests with:
+All tests run in the CI and must pass before merging.
+All tests run against a running langfuse instance and **write/delete real data from the database**.
 
-Run all
+### Tests in the `web` package (public API)
+We're using Jest with in the `web` package. Therefore, if you want to provide an argument to the test runner, do it directly without an intermittent ` -- `.
 
-```bash
-npm run test
+There are three types of unit tests:
+- `test-sync`
+- `test-async`
+- `test-client`
+
+To run a specific test, for example the test: `"should handle special characters in prompt names"` in `prompts.v2.servertest.ts`, run:
+```sh
+cd web  # or with --filter=web
+pnpm test-sync --testPathPattern="prompts\.v2\.servertest" --testNamePattern="should handle special characters in prompt names"
 ```
 
-Run interactively in watch mode
-
-```bash
-npm run test:watch
+To run all tests:
+```sh
+pnpm run test
 ```
 
-These tests are also run in CI.
+Run interactively in watch mode (not recommended!)
+```sh
+pnpm run test:watch
+```
+
+### Tests in the `worker` package
+For the `worker` package, we're using `vitest` to run unit tests.
+
+```sh
+pnpm run test --filter=worker -- FILE_YOU_WANT_TO_TEST.ts -t "test name"
+```
 
 ## CI/CD
 
@@ -327,6 +366,20 @@ Please note that
 
 Until the V3 release, both the JSON record must be updated **and** a migration must be created to continue supporting self-hosted users. Note that the migration must updated both the `models` as well as the `prices` table accordingly.
 
+## Updating the OpenAPI Specs & fern SDKs
+
+We maintain the API specifications manually to guarantee a high degree of understandability. If you made changes to the API, please update the respective `.yml` files in `fern/apis/...`.
+
+To generate the respective `openapi.yml` files which power the online API reference & SDKs, run:
+
+```sh
+npx fern-api generate --api server  # for the server API
+npx fern-api generate --api client  # for the client API
+npx fern-api generate --api organizations  # for the organizations API
+```
+
+**Note:** You need a signed in fern account to run those commands.
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

README.cn.md

@@ -26,7 +26,7 @@
     <a href="https://langfuse.com/roadmap"><strong>路线图</strong></a> ·
   </div>
   <br/>
-  <span>Langfuse 使用 <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a> 作为支持和功能请求的平台。</span>
+  <span>Langfuse 使用 <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a> 作为支持和功能请求的平台。</span>
   <br/>
   <span><b>我们正在招聘。</b> <a href="https://langfuse.com/careers"><strong>加入我们</strong></a>，从事产品工程和技术市场职位。</span>
   <br/>
@@ -213,8 +213,8 @@ LANGFUSE_HOST="https://cloud.langfuse.com" # 🇪🇺 欧盟区域
 
 创建示例代码（文件名：**main.py**）：
 
-````python:main.py
-from langfuse.decorators import observe
+```python:main.py
+from langfuse import observe
 from langfuse.openai import openai  # OpenAI 集成
 
 @observe()

README.ja.md

@@ -26,7 +26,7 @@
       <a href="https://langfuse.com/roadmap"><strong>ロードマップ</strong></a> ·
    </div>
    <br/>
-   <span>Langfuseは、サポートと機能リクエストのために <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a> を利用しています。</span>
+   <span>Langfuseは、サポートと機能リクエストのために <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a> を利用しています。</span>
    <br/>
    <span><b>We're hiring.</b> <a href="https://langfuse.com/careers"><strong>チームに加わる</strong></a> （製品エンジニアリングおよびテクニカルGTMのポジション）への応募をお待ちしています。</span>
    <br/>
@@ -219,7 +219,7 @@ LANGFUSE_HOST="https://cloud.langfuse.com" # 🇪🇺 EUリージョン
 ```
 
 ```python:/@observe()/ /from langfuse.openai import openai/ filename="main.py"
-from langfuse.decorators import observe
+from langfuse import observe
 from langfuse.openai import openai  # OpenAI統合
 
 @observe()

README.kr.md

@@ -201,7 +201,7 @@ LANGFUSE_HOST="https://cloud.langfuse.com" # 🇪🇺 EU region
 ```
 
 ```python:main.py
-from langfuse.decorators import observe
+from langfuse import observe
 from langfuse.openai import openai # OpenAI integration
 
 @observe()

README.md

@@ -3,6 +3,9 @@
 <div align="center">
    <div>
       <h3>
+        <a href="https://langfuse.com/blog/2025-06-04-open-sourcing-langfuse-product">
+            <strong>Langfuse Is Doubling Down On Open Source</strong>
+         </a> <br> <br>
          <a href="https://cloud.langfuse.com">
             <strong>Langfuse Cloud</strong>
          </a> · 
@@ -23,7 +26,7 @@
       <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
    </div>
    <br/>
-   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>GitHub Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
    <span><b>We're hiring.</b> <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in product engineering and technical go-to-market roles.</span>
    <br/>
@@ -93,7 +96,7 @@ Langfuse is an **open source LLM engineering** platform. It helps teams collabor
 
 ### Langfuse Cloud
 
-Managed deployment by the Langfuse team, generous free-tier (hobby plan), no credit card required.
+Managed deployment by the Langfuse team, generous free-tier, no credit card required.
 
 <div align="center">
     <a href="https://cloud.langfuse.com" target="_blank">
@@ -115,12 +118,11 @@ Run Langfuse on your own infrastructure:
   # Run the langfuse docker compose
   docker compose up
   ```
-
-- [Kubernetes (Helm)](https://langfuse.com/self-hosting/kubernetes-helm): Run Langfuse on a Kubernetes cluster using Helm. This is the preferred production deployment.
 - [VM](https://langfuse.com/self-hosting/docker-compose): Run Langfuse on a single Virtual Machine using Docker Compose.
-- Planned: Cloud-specific deployment guides, please upvote and comment on the following threads: [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- [Kubernetes (Helm)](https://langfuse.com/self-hosting/kubernetes-helm): Run Langfuse on a Kubernetes cluster using Helm. This is the preferred production deployment.
+- Terraform Templates: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
-See [self-hosting documentation](https://langfuse.com/self-hosting) to learn more about the architecture and configuration options.
+See [self-hosting documentation](https://langfuse.com/self-hosting) to learn more about architecture and configuration options.
 
 ## 🔌 Integrations
 
@@ -191,7 +193,7 @@ LANGFUSE_HOST="https://cloud.langfuse.com" # 🇪🇺 EU region
 ```
 
 ```python /@observe()/ /from langfuse.openai import openai/ filename="main.py"
-from langfuse.decorators import observe
+from langfuse import observe
 from langfuse.openai import openai # OpenAI integration
 
 @observe()

docker-compose.dev-redis-cluster.yml

@@ -0,0 +1,146 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.3
+    user: "101:101"
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - 127.0.0.1:8123:8123
+      - 127.0.0.1:9000:9000
+    depends_on:
+      - postgres
+
+  minio:
+    image: minio/minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: miniosecret
+    ports:
+      - 127.0.0.1:9090:9000
+      - 127.0.0.1:9091:9001
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 127.0.0.1:5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+  # Redis Cluster. Requires a Unix host to work for network_mode: host. I.e. tests will fail on windows and mac with this setup.
+  redis-node-0:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-0:/bitnami/redis/data
+    environment:
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6370
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+
+  redis-node-1:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-1:/bitnami/redis/data
+    environment:
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6371
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+
+  redis-node-2:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-2:/bitnami/redis/data
+    environment:
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6372
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+
+  redis-node-3:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-3:/bitnami/redis/data
+    environment:
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6373
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+
+  redis-node-4:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-4:/bitnami/redis/data
+    environment:
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6374
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+
+  redis-node-5:
+    image: docker.io/bitnami/redis-cluster:8.0
+    network_mode: host
+    volumes:
+      - redis-cluster_data-5:/bitnami/redis/data
+    depends_on:
+      - redis-node-0
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+      - redis-node-4
+    environment:
+      REDISCLI_AUTH: bitnami
+      REDIS_CLUSTER_REPLICAS: 1
+      REDIS_PASSWORD: bitnami
+      REDIS_PORT_NUMBER: 6375
+      REDIS_NODES: 127.0.0.1:6370 127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375
+      REDIS_CLUSTER_CREATOR: yes
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
+    driver: local
+  redis-cluster_data-0:
+    driver: local
+  redis-cluster_data-1:
+    driver: local
+  redis-cluster_data-2:
+    driver: local
+  redis-cluster_data-3:
+    driver: local
+  redis-cluster_data-4:
+    driver: local
+  redis-cluster_data-5:
+    driver: local

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
       CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
       CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
       LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}

ee/package.json

@@ -28,9 +28,9 @@
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
-    "next": "^14.2.26",
+    "next": "^14.2.30",
     "next-auth": "^4.24.11",
-    "zod": "^3.24.4"
+    "zod": "^3.25.62"
   },
   "devDependencies": {
     "@repo/eslint-config": "workspace:*",

ee/src/env.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({

fern/apis/server/definition/commons.yml

@@ -457,13 +457,13 @@ types:
         docs: Unit used by this model.
         type: optional<ModelUsageUnit>
       inputPrice:
-        docs: Price (USD) per input unit
+        docs: Deprecated. See 'prices' instead. Price (USD) per input unit
         type: optional<double>
       outputPrice:
-        docs: Price (USD) per output unit
+        docs: Deprecated. See 'prices' instead. Price (USD) per output unit
         type: optional<double>
       totalPrice:
-        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        docs: Deprecated. See 'prices' instead. Price (USD) per total unit. Cannot be set if input or output price is set.
         type: optional<double>
       tokenizerId:
         docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
@@ -473,6 +473,13 @@ types:
         type: optional<unknown>
       isLangfuseManaged:
         type: boolean
+      prices:
+        docs: Price (USD) by usage type
+        type: map<string, ModelPrice>
+
+  ModelPrice:
+    properties:
+      price: double
 
   # Utilities
   ModelUsageUnit:

fern/apis/server/definition/metrics.yml

@@ -28,7 +28,7 @@ service:
                 "metrics": [              // Required. At least one metric must be provided
                   {
                     "measure": string,    // What to measure, e.g. "count", "latency", "value"
-                    "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95"
+                    "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95", "histogram"
                   }
                 ],
                 "filters": [              // Optional. Default: []
@@ -50,7 +50,11 @@ service:
                     "field": string,      // Field to order by
                     "direction": string   // "asc" or "desc"
                   }
-                ]
+                ],
+                "config": {               // Optional. Query-specific configuration
+                  "bins": number,         // Optional. Number of bins for histogram (1-100), default: 10
+                  "row_limit": number     // Optional. Row limit for results (1-1000)
+                }
               }
               ```
       response: MetricsResponse
@@ -62,3 +66,4 @@ types:
         docs: |
           The metrics data. Each item in the list contains the metric values and dimensions requested in the query.
           Format varies based on the query parameters.
+          Histograms will return an array with [lower, upper, height] tuples.

fern/apis/server/definition/observations.yml

@@ -39,7 +39,7 @@ service:
             docs: Optional filter for observations where the environment is one of the provided values.
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time or or after this datetime (ISO 8601).
+            docs: Retrieve only observations with a start_time on or after this datetime (ISO 8601).
           toStartTime:
             type: optional<datetime>
             docs: Retrieve only observations with a start_time before this datetime (ISO 8601).

fern/apis/server/definition/organizations.yml

@@ -37,6 +37,12 @@ service:
       request: MembershipRequest
       response: MembershipResponse
 
+    getOrganizationProjects:
+      docs: Get all projects for the organization associated with the API key (requires organization-scoped API key)
+      method: GET
+      path: /organizations/projects
+      response: OrganizationProjectsResponse
+
 types:
   MembershipRole:
     enum:
@@ -59,4 +65,16 @@ types:
 
   MembershipsResponse:
     properties:
-      memberships: list<MembershipResponse>
+      memberships: list<MembershipResponse>
+
+  OrganizationProject:
+    properties:
+      id: string
+      name: string
+      metadata: optional<map<string, unknown>>
+      createdAt: datetime
+      updatedAt: datetime
+
+  OrganizationProjectsResponse:
+    properties:
+      projects: list<OrganizationProject>

fern/apis/server/definition/projects.yml

@@ -11,7 +11,7 @@ service:
       method: GET
       path: /projects
       response: Projects
-    
+
     create:
       docs: Create a new project (requires organization-scoped API key)
       method: POST
@@ -28,7 +28,7 @@ service:
               type: integer
               docs: Number of days to retain data. Must be 0 or at least 3 days. Requires data-retention entitlement for non-zero values. Optional.
       response: Project
-    
+
     update:
       docs: Update a project by ID (requires organization-scoped API key).
       method: PUT
@@ -47,7 +47,7 @@ service:
               type: integer
               docs: Number of days to retain data. Must be 0 or at least 3 days. Requires data-retention entitlement for non-zero values. Optional.
       response: Project
-    
+
     delete:
       docs: Delete a project by ID (requires organization-scoped API key). Project deletion is processed asynchronously.
       method: DELETE
@@ -57,7 +57,7 @@ service:
       response:
         status-code: 202
         type: ProjectDeletionResponse
-        
+
     # API Key endpoints
     getApiKeys:
       docs: Get all API keys for a project (requires organization-scoped API key)
@@ -104,14 +104,14 @@ types:
         type: map<string, unknown>
         docs: Metadata for the project
       retentionDays:
-        type: integer
+        type: optional<integer>
         docs: Number of days to retain data. Null or 0 means no retention. Omitted if no retention is configured.
 
   ProjectDeletionResponse:
     properties:
       success: boolean
       message: string
-      
+
   ApiKeyList:
     docs: List of API keys for a project
     properties:
@@ -137,7 +137,7 @@ types:
       secretKey: string
       displaySecretKey: string
       note: optional<string>
-      
+
   ApiKeyDeletionResponse:
     docs: Response for API key deletion
     properties:

fern/apis/server/definition/prompts.yml

@@ -82,7 +82,7 @@ types:
   CreateChatPromptRequest:
     properties:
       name: string
-      prompt: list<ChatMessage>
+      prompt: list<ChatMessageWithPlaceholders>
       config: optional<unknown>
       labels:
         type: optional<list<string>>
@@ -132,6 +132,11 @@ types:
         type: optional<map<string, unknown>>
         docs: The dependency resolution graph for the current prompt. Null if prompt has no dependencies.
 
+  ChatMessageWithPlaceholders:
+    union:
+      chatmessage: ChatMessage
+      placeholder: PlaceholderMessage
+
   ChatMessage:
     properties:
       role:
@@ -139,6 +144,11 @@ types:
       content:
         type: string
 
+  PlaceholderMessage:
+    properties:
+      name:
+        type: string
+
   TextPrompt:
     extends: BasePrompt
     properties:
@@ -147,4 +157,4 @@ types:
   ChatPrompt:
     extends: BasePrompt
     properties:
-      prompt: list<ChatMessage>
+      prompt: list<ChatMessageWithPlaceholders>

fern/apis/server/definition/trace.yml

@@ -63,6 +63,9 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Optional filter for traces where the environment is one of the provided values.
+          fields:
+            type: optional<string>
+            docs: "Comma-separated list of fields to include in the response. Available field groups are 'core' (always included), 'io' (input, output, metadata), 'scores', 'observations', 'metrics'. If not provided, all fields are included. Example: 'core,scores,metrics'"
       response: Traces
     deleteMultiple:
       docs: Delete multiple traces

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.62.0",
+  "version": "3.78.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -18,26 +18,27 @@
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
     "dx": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset -f && SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "dev:worker": "turbo run dev --filter=worker",
     "dev:web": "turbo run dev --filter=web",
+    "dev:web-turbo": "turbo run dev --filter=web -- --turbo",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "release": "dotenv -e ../.env -- release-it",
     "prepare": "husky"
   },
   "devDependencies": {
-    "@release-it/bumper": "^7.0.1",
+    "@release-it/bumper": "^7.0.5",
     "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
     "husky": "^9.0.11",
     "prettier": "^3.3.3",
-    "release-it": "^18.1.2",
-    "turbo": "^1.13.4"
+    "release-it": "^19.0.3",
+    "turbo": "^2.5.4"
   },
   "release-it": {
     "git": {
@@ -86,7 +87,9 @@
   "pnpm": {
     "overrides": {
       "nanoid": "^3.3.8",
-      "katex": "^0.16.21"
+      "katex": "^0.16.21",
+      "tar-fs": "^2.1.2",
+      "rollup@^4.0.0": "^4.22.4"
     },
     "patchedDependencies": {
       "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.4",
+    "eslint-config-turbo": "^2.5.4",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/clustered/0019_analytics_traces.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_traces ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0019_analytics_traces.up.sql

@@ -0,0 +1,15 @@
+CREATE VIEW analytics_traces ON CLUSTER default AS
+SELECT
+    project_id,
+    toStartOfHour(timestamp) AS hour,
+    uniq(id) AS countTraces,
+    max(user_id IS NOT NULL) AS hasUsers,
+    max(session_id IS NOT NULL) AS hasSessions,
+    max(if(environment != 'default', 1, 0)) AS hasEnvironments,
+    max(length(tags) > 0) AS hasTags
+FROM
+    traces
+WHERE toStartOfHour(timestamp) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    hour;

packages/shared/clickhouse/migrations/clustered/0020_analytics_observations.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_observations ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0020_analytics_observations.up.sql

@@ -0,0 +1,18 @@
+CREATE VIEW analytics_observations ON CLUSTER default AS
+SELECT
+    project_id,
+    type,
+    toStartOfHour(start_time) AS hour,
+    uniq(id) AS countObservations,
+    max(level != 'DEFAULT') AS hasLevel,
+    max(provided_model_name IS NOT NULL) AS hasProvidedModelName,
+    max(length(provided_usage_details) > 0) AS hasProvidedUsageDetails,
+    max(length(provided_cost_details) > 0) AS hasProvidedCostDetails,
+    max(prompt_name IS NOT NULL) AS hasPromptName
+FROM
+    observations
+WHERE toStartOfHour(start_time) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    type,
+    hour;

packages/shared/clickhouse/migrations/clustered/0021_analytics_scores.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_scores ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0021_analytics_scores.up.sql

@@ -0,0 +1,21 @@
+CREATE VIEW analytics_scores ON CLUSTER default AS
+SELECT
+    project_id,
+    toStartOfHour(timestamp) AS hour,
+    uniq(id) AS countScores,
+    max(source = 'ANNOTATION') AS hasAnnotation,
+    max(source = 'API') AS hasApi,
+    max(source = 'EVAL') AS hasEval,
+    max(observation_id IS NOT NULL) AS hasObservationScore,
+    max(session_id IS NOT NULL) AS hasSessionScore,
+    max(dataset_run_id IS NOT NULL) AS hasDatasetRunScore,
+    max(data_type = 'BOOLEAN') AS hasBoolScore,
+    max(data_type = 'NUMERIC') AS hasNumericScore,
+    max(data_type = 'CATEGORICAL') AS hasCategoricalScore,
+    max(comment IS NOT NULL) AS hasComment
+FROM
+    scores
+WHERE toStartOfHour(timestamp) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    hour;

packages/shared/clickhouse/migrations/unclustered/0019_analytics_traces.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_traces;

packages/shared/clickhouse/migrations/unclustered/0019_analytics_traces.up.sql

@@ -0,0 +1,15 @@
+CREATE VIEW analytics_traces AS
+SELECT
+    project_id,
+    toStartOfHour(timestamp) AS hour,
+    uniq(id) AS countTraces,
+    max(user_id IS NOT NULL) AS hasUsers,
+    max(session_id IS NOT NULL) AS hasSessions,
+    max(if(environment != 'default', 1, 0)) AS hasEnvironments,
+    max(length(tags) > 0) AS hasTags
+FROM
+    traces
+WHERE toStartOfHour(timestamp) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    hour;

packages/shared/clickhouse/migrations/unclustered/0020_analytics_observations.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_observations;

packages/shared/clickhouse/migrations/unclustered/0020_analytics_observations.up.sql

@@ -0,0 +1,18 @@
+CREATE VIEW analytics_observations AS
+SELECT
+    project_id,
+    type,
+    toStartOfHour(start_time) AS hour,
+    uniq(id) AS countObservations,
+    max(level != 'DEFAULT') AS hasLevel,
+    max(provided_model_name IS NOT NULL) AS hasProvidedModelName,
+    max(length(provided_usage_details) > 0) AS hasProvidedUsageDetails,
+    max(length(provided_cost_details) > 0) AS hasProvidedCostDetails,
+    max(prompt_name IS NOT NULL) AS hasPromptName
+FROM
+    observations
+WHERE toStartOfHour(start_time) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    type,
+    hour;

packages/shared/clickhouse/migrations/unclustered/0021_analytics_scores.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS analytics_scores;

packages/shared/clickhouse/migrations/unclustered/0021_analytics_scores.up.sql

@@ -0,0 +1,21 @@
+CREATE VIEW analytics_scores AS
+SELECT
+    project_id,
+    toStartOfHour(timestamp) AS hour,
+    uniq(id) AS countScores,
+    max(source = 'ANNOTATION') AS hasAnnotation,
+    max(source = 'API') AS hasApi,
+    max(source = 'EVAL') AS hasEval,
+    max(observation_id IS NOT NULL) AS hasObservationScore,
+    max(session_id IS NOT NULL) AS hasSessionScore,
+    max(dataset_run_id IS NOT NULL) AS hasDatasetRunScore,
+    max(data_type = 'BOOLEAN') AS hasBoolScore,
+    max(data_type = 'NUMERIC') AS hasNumericScore,
+    max(data_type = 'CATEGORICAL') AS hasCategoricalScore,
+    max(comment IS NOT NULL) AS hasComment
+FROM
+    scores
+WHERE toStartOfHour(timestamp) <= toStartOfHour(subtractHours(now(), 1))
+GROUP BY
+    project_id,
+    hour;

packages/shared/clickhouse/scripts/down.sh

@@ -36,8 +36,12 @@ if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+  # If SKIP_CONFIRM is set, automatically answer the confirmation prompt. Otherwise run interactively.
+  if [ "$SKIP_CONFIRM" = "1" ] || [ "$SKIP_CONFIRM" = "true" ]; then
+    printf 'y\n' | migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+  else
+    migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+  fi
 else
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
@@ -45,6 +49,10 @@ else
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
   fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
+  # If SKIP_CONFIRM is set, automatically answer the confirmation prompt. Otherwise run interactively.
+  if [ "$SKIP_CONFIRM" = "1" ] || [ "$SKIP_CONFIRM" = "true" ]; then
+    printf 'y\n' | migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
+  else
+    migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
+  fi
 fi

packages/shared/clickhouse/scripts/seed.ts

@@ -1,70 +0,0 @@
-import { prisma } from "../../src/db";
-import { ObservationRecordReadType, redis } from "../../src/server";
-import { prepareClickhouse } from "../../scripts/prepareClickhouse";
-import { createDatasets } from "../../prisma/seed";
-import { queryClickhouse } from "../../src/server/repositories/clickhouse";
-import { convertObservation } from "../../src/server/repositories/observations_converters";
-
-async function main() {
-  try {
-    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
-    if (
-      await prisma.project.findFirst({
-        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
-      })
-    ) {
-      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
-    }
-    await prepareClickhouse(projectIds, {
-      numberOfDays: 3,
-      totalObservations: 10000,
-    });
-
-    const project1 = await prisma.project.findFirst({
-      where: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-    });
-
-    const project2 =
-      projectIds.length > 1
-        ? await prisma.project.findFirst({
-            where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
-          })
-        : await prisma.project.findFirst();
-
-    const query = `
-        SELECT *
-        FROM observations o
-        WHERE o.project_id IN ({projectIds: Array(String)})
-        LIMIT 2000;
-      `;
-
-    const res = await queryClickhouse<ObservationRecordReadType>({
-      query,
-      params: {
-        projectIds,
-      },
-    });
-
-    await createDatasets(
-      project1!,
-      project2!,
-      (await Promise.all(res.map(convertObservation))).map((o) => ({
-        ...o,
-        metadata: {},
-        modelParameters: {},
-        input: {},
-        output: {},
-      })),
-    );
-
-    console.log("Clickhouse preparation completed successfully.");
-  } catch (error) {
-    console.error("Error during Clickhouse preparation:", error);
-  } finally {
-    await prisma.$disconnect();
-    redis?.disconnect();
-    console.log("Disconnected from Clickhouse.");
-  }
-}
-
-main();

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 65",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -47,11 +47,11 @@
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
     "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed",
-    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' clickhouse/scripts/seed.ts",
-    "load:setup": "dotenv -e ../../.env -- tsx scripts/load-seed.ts"
+    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' scripts/seeder/seed-clickhouse.ts",
+    "load:setup": "dotenv -e ../../.env -- tsx scripts/seeder/load-seed-clickhouse.ts"
   },
   "prisma": {
-    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
+    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} scripts/seeder/seed-postgres.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
@@ -60,18 +60,18 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.4.0",
+    "@clickhouse/client": "^1.11.2",
     "@google-cloud/storage": "^7.15.2",
-    "@langchain/anthropic": "^0.3.12",
-    "@langchain/aws": "^0.1.3",
-    "@langchain/core": "^0.3.37",
-    "@langchain/google-genai": "^0.1.9",
-    "@langchain/google-vertexai": "^0.1.8",
-    "@langchain/openai": "^0.3.17",
+    "@langchain/anthropic": "^0.3.22",
+    "@langchain/aws": "^0.1.11",
+    "@langchain/core": "^0.3.58",
+    "@langchain/google-genai": "^0.2.12",
+    "@langchain/google-vertexai": "^0.2.12",
+    "@langchain/openai": "^0.5.13",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^6.3.0",
-    "@react-email/components": "^0.0.19",
-    "@react-email/render": "^0.0.15",
+    "@prisma/client": "^6.10.1",
+    "@react-email/components": "^0.1.0",
+    "@react-email/render": "^1.1.2",
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",
@@ -81,17 +81,18 @@
     "exponential-backoff": "^3.1.1",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
+    "jsonpath-plus": "10.3.0",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.15",
-    "langfuse-langchain": "3.30.3",
+    "langchain": "^0.3.28",
+    "langfuse-langchain": "3.38.1",
     "lodash": "^4.17.21",
-    "lossless-json": "^4.0.2",
+    "lossless-json": "^4.1.1",
     "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
     "winston": "^3.15.0",
-    "zod": "^3.24.4",
+    "zod": "^3.25.62",
     "zod-to-json-schema": "^3.23.5"
   },
   "devDependencies": {
@@ -101,6 +102,7 @@
     "@types/node": "^20.11.29",
     "@types/nodemailer": "^6.4.16",
     "@types/pg": "^8.11.10",
+    "@types/react": "18.2.79",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
@@ -110,7 +112,7 @@
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
     "prettier": "^3.3.3",
-    "prisma": "^6.3.0",
+    "prisma": "^6.10.1",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",

packages/shared/prisma/generated/types.ts

@@ -110,7 +110,9 @@ export const DashboardWidgetChartType = {
     HORIZONTAL_BAR: "HORIZONTAL_BAR",
     VERTICAL_BAR: "VERTICAL_BAR",
     PIE: "PIE",
-    NUMBER: "NUMBER"
+    NUMBER: "NUMBER",
+    HISTOGRAM: "HISTOGRAM",
+    PIVOT_TABLE: "PIVOT_TABLE"
 } as const;
 export type DashboardWidgetChartType = (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export type Account = {
@@ -228,8 +230,8 @@ export type BlobStorageIntegration = {
     type: BlobStorageIntegrationType;
     bucket_name: string;
     prefix: string;
-    access_key_id: string;
-    secret_access_key: string;
+    access_key_id: string | null;
+    secret_access_key: string | null;
     region: string;
     endpoint: string | null;
     force_path_style: boolean;
@@ -326,17 +328,29 @@ export type DatasetRuns = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
-export type EvalTemplate = {
+export type DefaultLlmModel = {
     id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     project_id: string;
+    llm_api_key_id: string;
+    provider: string;
+    adapter: string;
+    model: string;
+    model_params: unknown | null;
+};
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string | null;
     name: string;
     version: number;
     prompt: string;
-    model: string;
-    provider: string;
-    model_params: unknown;
+    partner: string | null;
+    model: string | null;
+    provider: string | null;
+    model_params: unknown | null;
     vars: Generated<string[]>;
     output_schema: unknown;
 };
@@ -362,6 +376,7 @@ export type JobExecution = {
     updated_at: Generated<Timestamp>;
     project_id: string;
     job_configuration_id: string;
+    job_template_id: string | null;
     status: JobExecutionStatus;
     start_time: Timestamp | null;
     end_time: Timestamp | null;
@@ -703,6 +718,7 @@ export type DB = {
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
     datasets: Dataset;
+    default_llm_models: DefaultLlmModel;
     eval_templates: EvalTemplate;
     job_configurations: JobConfiguration;
     job_executions: JobExecution;

packages/shared/prisma/migrations/20250523100511_add_default_eval_model_table/migration.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "default_llm_models" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "llm_api_key_id" TEXT NOT NULL,
+    "provider" TEXT NOT NULL,
+    "adapter" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB,
+
+    CONSTRAINT "default_llm_models_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "default_llm_models" ADD CONSTRAINT "default_llm_models_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "default_llm_models" ADD CONSTRAINT "default_llm_models_llm_api_key_id_fkey" FOREIGN KEY ("llm_api_key_id") REFERENCES "llm_api_keys"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddUniqueConstraint
+ALTER TABLE "default_llm_models" ADD CONSTRAINT "default_llm_models_project_id_key" UNIQUE ("project_id");

packages/shared/prisma/migrations/20250523110540_modify_nullable_cols_eval_templates/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "eval_templates" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "eval_templates" ALTER COLUMN "model" DROP NOT NULL;
+ALTER TABLE "eval_templates" ALTER COLUMN "provider" DROP NOT NULL;
+ALTER TABLE "eval_templates" ALTER COLUMN "model_params" DROP NOT NULL;
+ALTER TABLE "eval_templates" ADD COLUMN "partner" TEXT;

packages/shared/prisma/migrations/20250523120545_add_nullable_job_template_id/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "job_executions" ADD COLUMN "job_template_id" TEXT;
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_template_id_fkey" FOREIGN KEY ("job_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL;

packages/shared/prisma/migrations/20250529071241_make_blobstorage_integration_credentials_optional/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "blob_storage_integrations" ALTER COLUMN "access_key_id" DROP NOT NULL,
+ALTER COLUMN "secret_access_key" DROP NOT NULL;

packages/shared/prisma/migrations/20250604085536_add_histogram_chart_type/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DashboardWidgetChartType" ADD VALUE 'HISTOGRAM';

packages/shared/prisma/migrations/20250625_add_pivot_table_charttype/migration.sql

@@ -0,0 +1,9 @@
+-- Database migration to add PIVOT_TABLE chart type to DashboardWidgetChartType enum
+-- This enables the creation of pivot table widgets in the dashboard system
+-- 
+-- This migration adds support for tabular data visualization with configurable
+-- row dimensions and metrics, extending the existing widget types (line charts,
+-- bar charts, pie charts, etc.) to include pivot table functionality.
+
+-- AlterEnum
+ALTER TYPE "DashboardWidgetChartType" ADD VALUE 'PIVOT_TABLE'; 

packages/shared/prisma/schema.prisma

@@ -157,6 +157,7 @@ model Project {
   Dashboard              Dashboard[]
   DashboardWidget        DashboardWidget[]
   TableViewPreset        TableViewPreset[]
+  DefaultLlmModel        DefaultLlmModel[]
 
   @@index([orgId])
   @@map("projects")
@@ -222,8 +223,9 @@ model LlmApiKeys {
   extraHeaderKeys   String[] @default([]) @map("extra_header_keys")
   config            Json?
 
-  projectId String  @map("project_id")
-  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  projectId       String            @map("project_id")
+  project         Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  DefaultLlmModel DefaultLlmModel[] @relation("LlmApiKeyId")
 
   @@unique([projectId, provider])
   @@map("llm_api_keys")
@@ -793,18 +795,22 @@ model EvalTemplate {
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
 
-  projectId String  @map("project_id")
-  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-  name             String
-  version          Int
-  prompt           String
-  model            String
-  provider         String
-  modelParams      Json               @map("model_params")
+  name    String
+  version Int
+  prompt  String
+
+  partner String? // e.g. "ragas", describes the partner that created the template
+
+  model            String?
+  provider         String?
+  modelParams      Json?              @map("model_params")
   vars             String[]           @default([])
   outputSchema     Json               @map("output_schema")
   JobConfiguration JobConfiguration[]
+  JobExecution     JobExecution[]
 
   @@unique([projectId, name, version])
   @@index([projectId, id])
@@ -864,6 +870,9 @@ model JobExecution {
   jobConfigurationId String           @map("job_configuration_id")
   jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
 
+  jobTemplateId String?       @map("job_template_id")
+  jobTemplate   EvalTemplate? @relation(fields: [jobTemplateId], references: [id], onDelete: SetNull, onUpdate: NoAction)
+
   status    JobExecutionStatus
   startTime DateTime?          @map("start_time")
   endTime   DateTime?          @map("end_time")
@@ -887,6 +896,26 @@ model JobExecution {
   @@map("job_executions")
 }
 
+model DefaultLlmModel {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  Project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  llmApiKeyId String     @map("llm_api_key_id")
+  LlmApiKey   LlmApiKeys @relation("LlmApiKeyId", fields: [llmApiKeyId], references: [id], onDelete: Cascade)
+
+  provider    String
+  adapter     String // This controls the interface that is used to connect with the LLM, e.g. 'openai' or 'anthropic'
+  model       String
+  modelParams Json?  @map("model_params")
+
+  @@unique([projectId])
+  @@map("default_llm_models")
+}
+
 // Single Sign-On configuration for a domain
 // This feature is part of the Enterprise Edition
 model SsoConfig {
@@ -922,8 +951,8 @@ model BlobStorageIntegration {
   type            BlobStorageIntegrationType @map("type")
   bucketName      String                     @map("bucket_name")
   prefix          String                     @map("prefix")
-  accessKeyId     String                     @map("access_key_id")
-  secretAccessKey String                     @map("secret_access_key")
+  accessKeyId     String?                    @map("access_key_id")
+  secretAccessKey String?                    @map("secret_access_key")
   region          String                     @map("region")
   endpoint        String?                    @map("endpoint")
   forcePathStyle  Boolean                    @map("force_path_style")
@@ -1127,6 +1156,8 @@ enum DashboardWidgetChartType {
   VERTICAL_BAR
   PIE
   NUMBER
+  HISTOGRAM
+  PIVOT_TABLE
 }
 
 model DashboardWidget {

packages/shared/scripts/prepareClickhouse.ts

@@ -1,314 +0,0 @@
-import { SEED_PROMPTS } from "../prisma/seed";
-import { prisma } from "../src/db";
-import { clickhouseClient, logger } from "../src/server";
-
-function randn_bm(min: number, max: number, skew: number) {
-  let u = 0,
-    v = 0;
-  while (u === 0) u = Math.random(); //Converting [0,1) to (0,1)
-  while (v === 0) v = Math.random();
-  let num = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
-
-  num = num / 10.0 + 0.5; // Translate to 0 -> 1
-  if (num > 1 || num < 0)
-    num = randn_bm(min, max, skew); // resample between 0 and 1 if out of range
-  else {
-    num = Math.pow(num, skew); // Skew
-    num *= max - min; // Stretch to fill range
-    num += min; // offset to min
-  }
-  return num;
-}
-
-export const prepareClickhouse = async (
-  projectIds: string[],
-  opts: {
-    numberOfDays: number;
-    totalObservations: number;
-  },
-) => {
-  logger.info(
-    `Preparing Clickhouse for ${projectIds.length} projects and ${opts.numberOfDays} days.`,
-  );
-
-  const projectData = projectIds.map((projectId) => {
-    const observationsPerProject = Math.ceil(
-      randn_bm(0, opts.totalObservations, 2),
-    ); // Skew the number of observations
-
-    const tracesPerProject = Math.floor(observationsPerProject / 6); // On average, one trace should have 6 observations
-    const scoresPerProject = tracesPerProject * 10; // On average, one trace should have 10 scores
-    return {
-      projectId,
-      observationsPerProject,
-      tracesPerProject,
-      scoresPerProject,
-    };
-  });
-
-  for (const data of projectData) {
-    const {
-      projectId,
-      tracesPerProject,
-      observationsPerProject,
-      scoresPerProject,
-    } = data;
-    logger.info(
-      `Preparing Clickhouse for ${projectId}: Traces: ${tracesPerProject}, Scores: ${scoresPerProject}, Observations: ${observationsPerProject}`,
-    );
-
-    const tracesQuery = `
-    INSERT INTO traces
-    SELECT toString(number) AS id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-      concat('name_', toString(rand() % 100)) AS name,
-      concat('user_id_', toInt64(randExponential(1 / 100))) AS user_id,
-      map('prototype', 'test') AS metadata,
-      concat('release_', toString(randUniform(0, 100))) AS release,
-      concat('version_', toString(randUniform(0, 100))) AS version,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      if(rand() < 0.8, true, false) as public,
-      if(rand() < 0.8, true, false) as bookmarked,
-      array('tag1', 'tag2') as tags,
-      repeat('input', toInt64(randExponential(1 / 100))) AS input,
-      repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      if(randUniform(0, 1) < 0.2, NULL, concat('session_', toString(rand() % 1000))) AS session_id,
-      timestamp AS created_at,
-      timestamp AS updated_at,
-      timestamp AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${tracesPerProject});
-  `;
-
-    const observationsQuery = `
-    INSERT INTO observations
-    SELECT toString(number) AS id,
-      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
-      toString(rand()) AS parent_observation_id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
-      addSeconds(start_time, if(rand() < 0.6, floor(randUniform(0, 20)), floor(randUniform(0, 3600)))) AS end_time,
-      concat('name', toString(rand() % 100)) AS name,
-      map('prototype', 'test') AS metadata,
-      if(randUniform(0, 1) < 0.9, 'DEFAULT', if(randUniform(0, 1) < 0.5, 'ERROR', if(randUniform(0, 1) < 0.5, 'DEBUG', 'WARNING'))) AS level,
-      'status_message' AS status_message,
-      'version' AS version,
-      repeat('input', toInt64(randExponential(1 / 100))) AS input,
-      repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      case
-        when number % 2 = 0 then 'claude-3-haiku-20230407'
-        else 'gpt-4'
-      end as provided_model_name,
-      case
-        when number % 2 = 0 then 'cltra4wbs0000k1407g0ya3'
-        else '1cmtk9y0000y3y79x9jgxj'
-      end as internal_model_id,
-      if("type" = 'GENERATION',
-        '{"temperature": 0.7, "max_tokens": 150}',
-        '{}') AS model_parameters,
-      if("type" = 'GENERATION',
-        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
-        map()) AS provided_usage_details,
-      if("type" = 'GENERATION',
-        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
-        map()) AS usage_details,
-      if("type" = 'GENERATION',
-        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
-        map()) AS provided_cost_details,
-      if("type" = 'GENERATION',
-        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
-        map()) AS cost_details,
-      if("type" = 'GENERATION',
-        toDecimal64(randUniform(0, 2000), 12),
-        NULL) AS total_cost,
-      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
-      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,
-      array(${SEED_PROMPTS.map((p) => `'${p.name}'`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_name,
-      array(${SEED_PROMPTS.map((p) => `'${p.version}'`).join(
-        ",",
-      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_version,
-      start_time AS created_at,
-      start_time AS updated_at,
-      start_time AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${observationsPerProject});
-  `;
-
-    const scoresQuery = `
-    INSERT INTO scores
-    SELECT toString(number) AS id,
-      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-      '${projectId}' AS project_id,
-      'default' AS environment,
-      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
-      NULL AS session_id,
-      NULL AS dataset_run_id,
-      if(
-        rand() > 0.9,
-        toString(floor(randUniform(0, ${observationsPerProject}))),
-        NULL
-      ) AS observation_id,
-      concat('name_', toString(rand() % 10)) AS name,
-      randUniform(0, 100) as value,
-      'API' as source,
-      'comment' as comment,
-      map('prototype', 'test') AS metadata,
-      toString(rand() % 100) as author_user_id,
-      toString(rand() % 100) as config_id,
-      if (rand() < 0.33, 'NUMERIC', if (rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) as data_type,
-      toString(rand() % 100) as string_value,
-      NULL as queue_id,
-      timestamp AS created_at,
-      timestamp AS updated_at,
-      timestamp AS event_ts,
-      0 AS is_deleted
-    FROM numbers(${scoresPerProject});
-  `;
-
-    const queries = [tracesQuery, scoresQuery, observationsQuery];
-
-    for (const query of queries) {
-      logger.info(`Executing query: ${query}`);
-      await clickhouseClient().command({
-        query,
-        clickhouse_settings: {
-          wait_end_of_query: 1,
-        },
-      });
-    }
-    // we also need to upsert trace sessions in postgres
-
-    const sessionQuery = `
-      SELECT session_id, project_id
-      FROM traces 
-      WHERE session_id IS NOT NULL;
-    `;
-    const sessionResult = await clickhouseClient().query({
-      query: sessionQuery,
-      format: "JSONEachRow",
-    });
-
-    const sessionData = await sessionResult.json<{
-      session_id: string;
-      project_id: string;
-    }>();
-
-    const sessionsToScore = sessionData
-      .filter(() => Math.random() < 0.5)
-      .slice(0, Math.min(500, sessionData.length));
-
-    if (sessionsToScore.length > 0) {
-      // Generate session scores query with specific session IDs
-      const sessionScoresQuery = `
-        INSERT INTO scores
-        SELECT
-          concat('session-', toString(number)) AS id,
-          toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
-          '${projectId}' AS project_id,
-          'default' AS environment,
-          NULL AS trace_id,
-          arrayElement(['${sessionsToScore.map((s) => s.session_id).join("','")}'], 1 + (number % ${sessionsToScore.length})) AS session_id,
-          NULL AS dataset_run_id,
-          NULL AS observation_id,
-          concat('session_quality_', toString(rand() % 10)) AS name,
-          randUniform(0, 100) AS value,
-          'API' AS source,
-          'Session-level assessment score' AS comment,
-          map('key', 'value') AS metadata,
-          toString(rand() % 100) AS author_user_id,
-          toString(rand() % 100) AS config_id,
-          if(rand() < 0.33, 'NUMERIC', if(rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) AS data_type,
-          toString(rand() % 100) AS string_value,
-          NULL AS queue_id,
-          timestamp AS created_at,
-          timestamp AS updated_at,
-          timestamp AS event_ts,
-          0 AS is_deleted
-        FROM numbers(${sessionsToScore.length})
-      `;
-
-      await clickhouseClient().command({
-        query: sessionScoresQuery,
-        clickhouse_settings: {
-          wait_end_of_query: 1,
-        },
-      });
-    }
-
-    const idProjectIdCombinations = sessionData.map((session) => ({
-      id: session.session_id,
-      projectId: session.project_id,
-      public: Math.random() < 0.1,
-      bookmarked: Math.random() < 0.1,
-    }));
-
-    await prisma.traceSession.createMany({
-      data: idProjectIdCombinations,
-      skipDuplicates: true,
-    });
-  }
-
-  const tables = ["traces", "scores", "observations"];
-  for (const table of tables) {
-    const query = `
-            SELECT
-            project_id,
-            count() AS per_project_count,
-            bar(per_project_count, 0, (
-                SELECT count(*)
-                FROM ${table}
-            ), 50) AS bar_representation
-          FROM ${table}
-          GROUP BY project_id
-          ORDER BY count() desc
-          `;
-
-    const result = await clickhouseClient().query({
-      query,
-      format: "TabSeparated",
-    });
-
-    logger.info(
-      `${table.charAt(0).toUpperCase() + table.slice(1)} per Project: \n` +
-        (await result.text()),
-    );
-  }
-
-  const tablesWithDateColumns = [
-    { name: "traces", dateColumn: "timestamp" },
-    { name: "scores", dateColumn: "timestamp" },
-    { name: "observations", dateColumn: "start_time" },
-  ];
-
-  for (const { name: table, dateColumn } of tablesWithDateColumns) {
-    const query = `
-            SELECT
-            toDate(${dateColumn}) AS event_date,
-            count() AS per_date_count,
-            bar(per_date_count, 0, (
-                SELECT count(*)
-                FROM ${table}
-            ), 50) AS bar_representation
-          FROM ${table}
-          GROUP BY event_date
-          ORDER BY event_date desc
-          `;
-
-    const result = await clickhouseClient().query({
-      query,
-      format: "TabSeparated",
-    });
-
-    logger.info(
-      `${table.charAt(0).toUpperCase() + table.slice(1)} per Date: \n` +
-        (await result.text()),
-    );
-  }
-};

packages/shared/scripts/seeder/load-seed-clickhouse.ts

@@ -1,8 +1,8 @@
 import { randomUUID } from "crypto";
-import { prisma } from "../src/db";
-import { getDisplaySecretKey, hashSecretKey, logger } from "../src/server";
-import { prepareClickhouse } from "./prepareClickhouse";
-import { redis } from "../src/server";
+import { prisma } from "../../src/db";
+import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
+import { prepareClickhouse } from "./prepare-clickhouse";
+import { redis } from "../../src/server";
 
 const createRandomProjectId = () => randomUUID().toString();
 
@@ -117,6 +117,7 @@ async function main() {
     await prepareClickhouse(createdProjectIds, {
       numberOfDays,
       totalObservations: totalObservations ?? 1000,
+      numberOfRuns: 3,
     });
 
     logger.info("Clickhouse preparation completed successfully.");

packages/shared/scripts/seeder/prepare-clickhouse.ts

@@ -0,0 +1,35 @@
+import { SeederOrchestrator } from "./utils/seeder-orchestrator";
+import { SeederOptions } from "./utils/types";
+import { logger } from "../../src/server";
+
+/**
+ * ClickHouse data preparation using the seeder abstraction.
+ */
+export const prepareClickhouse = async (
+  projectIds: string[],
+  opts: {
+    numberOfDays: number;
+    totalObservations: number;
+    numberOfRuns?: number;
+  },
+) => {
+  logger.info(
+    `Preparing ClickHouse for ${projectIds.length} projects and ${opts.numberOfDays} days.`,
+  );
+
+  const formattedOpts: SeederOptions = {
+    numberOfDays: opts.numberOfDays,
+    totalObservations: opts.totalObservations,
+    numberOfRuns: opts.numberOfRuns || 1,
+  };
+
+  const orchestrator = new SeederOrchestrator();
+
+  try {
+    await orchestrator.executeFullSeed(projectIds, formattedOpts);
+    logger.info("ClickHouse preparation completed successfully");
+  } catch (error) {
+    logger.error("ClickHouse preparation failed:", error);
+    throw error;
+  }
+};

packages/shared/scripts/seeder/seed-clickhouse.ts

@@ -0,0 +1,31 @@
+import { prisma } from "../../src/db";
+import { redis } from "../../src/server";
+import { prepareClickhouse } from "./prepare-clickhouse";
+
+async function main() {
+  try {
+    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
+    if (
+      await prisma.project.findFirst({
+        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      })
+    ) {
+      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
+    }
+    await prepareClickhouse(projectIds, {
+      numberOfDays: 3,
+      totalObservations: 1000,
+      numberOfRuns: 3,
+    });
+
+    console.log("Clickhouse preparation completed successfully.");
+  } catch (error) {
+    console.error("Error during Clickhouse preparation:", error);
+  } finally {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    console.log("Disconnected from Clickhouse.");
+  }
+}
+
+main();

packages/shared/scripts/seeder/seed-postgres.ts

@@ -0,0 +1,924 @@
+import { randomUUID } from "node:crypto";
+import { parseArgs } from "node:util";
+import { hash } from "bcryptjs";
+import { v4 } from "uuid";
+import { encrypt } from "../../src/encryption";
+import {
+	type JobConfiguration,
+	JobExecutionStatus,
+	PrismaClient,
+	type Project,
+	ScoreDataType,
+} from "../../src/index";
+import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
+import { redis } from "../../src/server/redis/redis";
+import {EVAL_TRACE_COUNT,
+  FAILED_EVAL_TRACE_INTERVAL,
+  SEED_CHAT_ML_PROMPTS,
+  SEED_DATASETS,
+  SEED_EVALUATOR_CONFIGS,
+  SEED_EVALUATOR_TEMPLATES,
+  SEED_PROMPT_VERSIONS,
+  SEED_TEXT_PROMPTS,
+} from "./utils/postgres-seed-constants";
+import {
+  generateDatasetRunTraceId,
+  generateEvalObservationId,
+  generateEvalScoreId,
+  generateEvalTraceId,
+} from "./utils/seed-helpers";
+
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
+const options = {
+  environment: { type: "string" },
+} as const;
+
+const prisma = new PrismaClient();
+
+async function main() {
+  const environment = parseArgs({
+    options,
+  }).values.environment;
+
+  const seedOrgId = "seed-org-id";
+  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  const seedUserId1 = "user-1"; // Owner of org
+  const seedUserId2 = "user-2"; // Member of org, admin of project
+
+  const user = await prisma.user.upsert({
+    where: { id: seedUserId1 },
+    update: {
+      name: "Demo User",
+      email: "demo@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId1,
+      name: "Demo User",
+      email: "demo@langfuse.com",
+      password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
+    },
+  });
+  const user2 = await prisma.user.upsert({
+    where: { id: seedUserId2 },
+    update: {
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId2,
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+  });
+
+  await prisma.organization.upsert({
+    where: { id: seedOrgId },
+    update: {
+      name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
+    },
+    create: {
+      id: seedOrgId,
+      name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
+    },
+  });
+
+  const project1 = await prisma.project.upsert({
+    where: { id: seedProjectId },
+    update: {
+      name: "llm-app",
+      orgId: seedOrgId,
+    },
+    create: {
+      id: seedProjectId,
+      name: "llm-app",
+      orgId: seedOrgId,
+    },
+  });
+
+  await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user.id,
+      orgId: seedOrgId,
+      role: "OWNER",
+    },
+    update: {},
+  });
+
+  const orgMembership2 = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user2.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user2.id,
+      orgId: seedOrgId,
+      role: "MEMBER",
+    },
+    update: {},
+  });
+
+  await prisma.projectMembership.upsert({
+    where: {
+      projectId_userId: {
+        projectId: project1.id,
+        userId: user2.id,
+      },
+    },
+    create: {
+      userId: user2.id,
+      projectId: project1.id,
+      role: "ADMIN",
+      orgMembershipId: orgMembership2.id,
+    },
+    update: {
+      orgMembershipId: orgMembership2.id,
+    },
+  });
+
+  await prisma.prompt.upsert({
+    where: {
+      projectId_name_version: {
+        projectId: seedProjectId,
+        name: "summary-prompt",
+        version: 1,
+      },
+    },
+    create: {
+      name: "summary-prompt",
+      project: { connect: { id: seedProjectId } },
+      prompt: "prompt {{variable}} {{anotherVariable}}",
+      labels: ["production", "latest"],
+      version: 1,
+      createdBy: "user-1",
+    },
+    update: {},
+  });
+
+  const seedApiKey = {
+    id: "seed-api-key",
+    secret: process.env.SEED_SECRET_KEY ?? "sk-lf-1234567890", // eslint-disable-line turbo/no-undeclared-env-vars
+    public: "pk-lf-1234567890",
+    note: "seeded key",
+  };
+
+  if (!(await prisma.apiKey.findUnique({ where: { id: seedApiKey.id } }))) {
+    await prisma.apiKey.create({
+      data: {
+        note: seedApiKey.note,
+        id: seedApiKey.id,
+        publicKey: seedApiKey.public,
+        hashedSecretKey: await hashSecretKey(seedApiKey.secret),
+        displaySecretKey: getDisplaySecretKey(seedApiKey.secret),
+        scope: "PROJECT",
+        project: {
+          connect: {
+            id: project1.id,
+          },
+        },
+      },
+    });
+  }
+
+  // Do not run the following for local docker compose setup
+  if (environment === "examples" || environment === "load") {
+    const seedOrgIdOrg2 = "demo-org-id";
+    const project2Id = "239ad00f-562f-411d-af14-831c75ddd875";
+    const org2 = await prisma.organization.upsert({
+      where: { id: seedOrgIdOrg2 },
+      update: {
+        name: "Langfuse Demo",
+      },
+      create: {
+        id: seedOrgIdOrg2,
+        name: "Langfuse Demo",
+      },
+    });
+    const project2 = await prisma.project.upsert({
+      where: { id: project2Id },
+      create: {
+        id: project2Id,
+        name: "demo-app",
+        orgId: org2.id,
+      },
+      update: { orgId: seedOrgIdOrg2 },
+    });
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: {
+          userId: user.id,
+          orgId: seedOrgIdOrg2,
+        },
+      },
+      create: {
+        userId: user.id,
+        orgId: seedOrgIdOrg2,
+        role: "VIEWER",
+      },
+      update: {},
+    });
+
+    const secondKey = {
+      id: "seed-api-key-2",
+      secret: process.env.SEED_SECRET_KEY ?? "sk-lf-asdfghjkl", // eslint-disable-line turbo/no-undeclared-env-vars
+      public: "pk-lf-asdfghjkl",
+      note: "seeded key 2",
+    };
+    if (!(await prisma.apiKey.findUnique({ where: { id: secondKey.id } }))) {
+      await prisma.apiKey.create({
+        data: {
+          note: secondKey.note,
+          id: secondKey.id,
+          publicKey: secondKey.public,
+          hashedSecretKey: await hashSecretKey(secondKey.secret),
+          displaySecretKey: getDisplaySecretKey(secondKey.secret),
+          scope: "PROJECT",
+          project: {
+            connect: {
+              id: project2.id,
+            },
+          },
+        },
+      });
+    }
+
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
+    await generateQueuesForProject([project1, project2], configIdsAndNames);
+    await generatePromptsForProject([project1, project2]);
+    await createDatasets(project1, project2);
+    await createTraceSessions(project1, project2);
+
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY; // eslint-disable-line turbo/no-undeclared-env-vars
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+          adapter: "openai",
+        },
+      });
+    } else {
+      logger.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key.",
+      );
+    }
+
+    // add eval objects
+    for (const evalTemplate of SEED_EVALUATOR_TEMPLATES) {
+      await prisma.evalTemplate.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project1.id,
+            name: evalTemplate.name,
+            version: 1,
+          },
+        },
+        create: {
+          id: evalTemplate.id,
+          projectId: project1.id,
+          name: evalTemplate.name,
+          version: evalTemplate.version,
+          prompt: evalTemplate.prompt,
+          model: evalTemplate.model,
+          vars: evalTemplate.vars,
+          provider: evalTemplate.provider,
+          outputSchema: evalTemplate.outputSchema,
+          modelParams: evalTemplate.modelParams,
+        },
+        update: {},
+      });
+    }
+
+    for (const evalConfig of SEED_EVALUATOR_CONFIGS) {
+      await prisma.jobConfiguration.upsert({
+        where: {
+          id: evalConfig.id,
+        },
+        create: {
+          id: evalConfig.id,
+          evalTemplateId: evalConfig.evalTemplateId,
+          projectId: project1.id,
+          jobType: evalConfig.jobType as any,
+          status: evalConfig.status as any,
+          scoreName: evalConfig.scoreName,
+          filter: evalConfig.filter,
+          variableMapping: evalConfig.variableMapping,
+          targetObject: evalConfig.targetObject,
+          sampling: evalConfig.sampling,
+          delay: evalConfig.delay,
+        },
+        update: {},
+      });
+    }
+
+    await generateEvalJobExecutions(
+      [project1, project2],
+      SEED_EVALUATOR_CONFIGS as unknown as Partial<JobConfiguration>[],
+    );
+
+    await createDashboardsAndWidgets([project1, project2]);
+
+    await prisma.llmSchema.createMany({
+      data: [
+        {
+          projectId: project1.id,
+          name: "get_weather",
+          description: "Fetches weather in Celsius for a given location",
+          schema: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+              unit: {
+                type: "string",
+                enum: ["celsius", "fahrenheit"],
+              },
+            },
+            required: ["location", "unit"],
+          },
+        },
+        {
+          projectId: project1.id,
+          name: "calculator",
+          description: "Performs basic arithmetic calculations",
+          schema: {
+            type: "object",
+            properties: {
+              expression: {
+                type: "string",
+                description:
+                  "The mathematical expression to evaluate, e.g. '2 + 2'",
+              },
+            },
+            required: ["expression"],
+          },
+        },
+      ],
+    });
+  }
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+  })
+  .catch(async (e) => {
+    logger.error(e);
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+    process.exit(1);
+  });
+
+async function createDashboardsAndWidgets(projects: Project[]) {
+  logger.info("Creating dashboards and widgets");
+
+  // Process each project
+  for (const project of projects) {
+    const widget = await prisma.dashboardWidget.upsert({
+      where: { id: "cabc" },
+      create: {
+        id: "cabc",
+        projectId: project.id,
+        name: "Trace Counts",
+        description: "Trace Counts by Name Over Time",
+        view: "TRACES",
+        dimensions: [{ field: "name" }],
+        metrics: [{ measure: "count", agg: "count" }],
+        filters: [],
+        chartType: "BAR_TIME_SERIES",
+        chartConfig: {
+          type: "BAR_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    const widget2 = await prisma.dashboardWidget.upsert({
+      where: { id: "cdef" },
+      create: {
+        id: "cdef",
+        projectId: project.id,
+        name: "Observation Latencies by Model",
+        description: "p95 Observation Latencies by Model Name",
+        view: "OBSERVATIONS",
+        dimensions: [{ field: "providedModelName" }],
+        metrics: [{ measure: "count", agg: "sum" }],
+        filters: [],
+        chartType: "LINE_TIME_SERIES",
+        chartConfig: {
+          type: "LINE_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    // Create a dashboard with multiple widgets
+    await prisma.dashboard.upsert({
+      where: { id: "seed-dashboard" },
+      create: {
+        id: "seed-dashboard",
+        projectId: project.id,
+        name: "Performance Overview",
+        description: "Dashboard with various performance metrics",
+        definition: {
+          widgets: [
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget.id,
+              x: 0,
+              y: 0,
+              x_size: 6,
+              y_size: 6,
+            },
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget2.id,
+              x: 6,
+              y: 0,
+              x_size: 6,
+              y_size: 6,
+            },
+          ],
+        },
+      },
+      update: {},
+    });
+  }
+}
+
+export async function createDatasets(
+  project1: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  project2: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+) {
+  for (const data of SEED_DATASETS) {
+    for (const projectId of [project1.id, project2.id]) {
+      const datasetName = data.name;
+
+      // check if ds already exists
+      const dataset =
+        (await prisma.dataset.findFirst({
+          where: {
+            projectId,
+            name: datasetName,
+          },
+        })) ??
+        (await prisma.dataset.create({
+          data: {
+            name: datasetName,
+            description: data.description,
+            projectId,
+            metadata: data.metadata,
+          },
+        }));
+
+      const datasetItemIds: string[] = [];
+      for (let index = 0; index < data.items.length; index++) {
+        const item = data.items[index];
+        const sourceTraceId =
+          Math.random() > 0.3
+            ? `${Math.floor(Math.random() * 100)}`
+            : undefined;
+
+        // Use upsert to prevent duplicates
+        const datasetItem = await prisma.datasetItem.upsert({
+          where: {
+            id_projectId: {
+              id: `${dataset.id}-${index}`,
+              projectId,
+            },
+          },
+          create: {
+            projectId,
+            id: `${dataset.id}-${index}`,
+            datasetId: dataset.id,
+            sourceTraceId: sourceTraceId ?? null,
+            sourceObservationId: null,
+            input: item.input,
+            expectedOutput: item.output,
+            metadata: Math.random() > 0.5 ? { key: "value" } : undefined,
+          },
+          update: {}, // Don't update if it exists
+        });
+        datasetItemIds.push(datasetItem.id);
+      }
+
+      for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
+        const datasetRun = await prisma.datasetRuns.upsert({
+          where: {
+            datasetId_projectId_name: {
+              datasetId: dataset.id,
+              projectId,
+              name: `demo-dataset-run-${datasetRunNumber}`,
+            },
+          },
+          create: {
+            projectId,
+            name: `demo-dataset-run-${datasetRunNumber}`,
+            description: Math.random() > 0.5 ? "Dataset run description" : "",
+            datasetId: dataset.id,
+            metadata: [
+              undefined,
+              "string",
+              100,
+              { key: "value" },
+              ["tag1", "tag2"],
+            ][datasetRunNumber % 5],
+          },
+          update: {},
+        });
+
+        for (let index = 0; index < datasetItemIds.length; index++) {
+          await prisma.datasetRunItems.upsert({
+            where: {
+              id_projectId: {
+                id: `${dataset.id}-${index}-${datasetRunNumber}`,
+                projectId,
+              },
+            },
+            create: {
+              id: `${dataset.id}-${index}-${datasetRunNumber}`,
+              projectId,
+              datasetItemId: datasetItemIds[index],
+              traceId: `${generateDatasetRunTraceId(datasetName, index, projectId, datasetRunNumber)}`,
+              datasetRunId: datasetRun.id,
+            },
+            update: {},
+          });
+        }
+      }
+    }
+  }
+}
+
+async function generateEvalJobExecutions(
+  projects: Project[],
+  evalJobConfigurations: Partial<JobConfiguration>[],
+) {
+  for (const project of projects) {
+    for (let i = 0; i < EVAL_TRACE_COUNT; i++) {
+      const jobConfiguration =
+        evalJobConfigurations[i % evalJobConfigurations.length];
+
+      const isFailed = i % FAILED_EVAL_TRACE_INTERVAL === 0;
+      await prisma.jobExecution.create({
+        data: {
+          projectId: project.id,
+          jobTemplateId: jobConfiguration.evalTemplateId,
+          jobInputTraceId: generateEvalTraceId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+          jobConfigurationId: jobConfiguration.id!,
+          status: isFailed
+            ? JobExecutionStatus.ERROR
+            : JobExecutionStatus.COMPLETED,
+          error: isFailed ? "Error message" : undefined,
+          jobOutputScoreId: generateEvalScoreId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+          jobInputObservationId: generateEvalObservationId(
+            jobConfiguration.evalTemplateId!,
+            i,
+            project.id,
+          ),
+        },
+      });
+    }
+  }
+}
+
+async function generatePromptsForProject(projects: Project[]) {
+  const promptIds = new Map<string, string[]>();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const promptIdsForProject = await generatePrompts(project);
+      promptIds.set(project.id, promptIdsForProject);
+    }),
+  );
+  return promptIds;
+}
+
+export const PROMPT_IDS: string[] = [];
+
+async function generatePrompts(project: Project) {
+  const promptIds = [];
+  for (const prompt of SEED_TEXT_PROMPTS) {
+    const versions = Math.floor(Math.random() * 20) + 1;
+    for (let i = 1; i <= versions; i++) {
+      const promptId = `prompt-${v4()}`;
+      await prisma.prompt.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project.id,
+            name: prompt.name,
+            version: i,
+          },
+          id: promptId,
+        },
+        create: {
+          id: promptId,
+          projectId: project.id,
+          createdBy: prompt.createdBy,
+          prompt: `${prompt.prompt} version ${i} content`,
+          name: prompt.name,
+          version: i,
+          labels: i === versions ? prompt.labels : [],
+        },
+        update: {
+          id: promptId,
+        },
+      });
+      promptIds.push(promptId);
+    }
+  }
+
+  for (const prompt of SEED_CHAT_ML_PROMPTS) {
+    const promptId = `prompt-${v4()}`;
+    const versions = Math.floor(Math.random() * 20) + 1;
+    for (let i = 1; i <= versions; i++) {
+      const versionAddition = [
+        {
+          role: "user",
+          content: "This is content for version " + i,
+        },
+      ];
+
+      await prisma.prompt.upsert({
+        where: {
+          projectId_name_version: {
+            projectId: project.id,
+            name: prompt.name,
+            version: prompt.version,
+          },
+          id: promptId,
+        },
+        create: {
+          id: promptId,
+          projectId: project.id,
+          createdBy: prompt.createdBy,
+          prompt: [...prompt.prompt, ...versionAddition],
+          name: prompt.name,
+          version: i,
+          type: "chat",
+          labels: prompt.labels,
+          tags: prompt.tags,
+        },
+        update: {
+          id: promptId,
+        },
+      });
+      promptIds.push(promptId);
+    }
+  }
+
+  for (const version of SEED_PROMPT_VERSIONS) {
+    const id = `prompt-${v4()}`;
+    await prisma.prompt.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project.id,
+          name: version.name,
+          version: version.version,
+        },
+        id: id,
+      },
+      create: {
+        id: id,
+        projectId: project.id,
+        createdBy: version.createdBy,
+        prompt: version.prompt,
+        name: version.name,
+        config: version.config,
+        version: version.version,
+        labels: version.labels,
+      },
+      update: {
+        id: id,
+      },
+    });
+    promptIds.push(id);
+  }
+
+  return promptIds;
+}
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    }),
+  );
+  return projectIdsToConfigs;
+}
+
+async function createTraceSessions(project1: Project, project2: Project) {
+  for (const project of [project1, project2]) {
+    for (let i = 0; i < 100; i++) {
+      await prisma.traceSession.create({
+        data: {
+          projectId: project.id,
+          id: `session_${i}`,
+          createdAt: new Date(),
+        },
+      });
+    }
+  }
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
+  }
+
+  return configNameAndId;
+}
+
+async function generateQueuesForProject(
+  projects: Project[],
+  configIdsAndNames: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >,
+) {
+  const projectIdsToQueues: Map<string, string[]> = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const queueIds = await generateQueues(
+        project,
+        configIdsAndNames.get(project.id) ?? [],
+      );
+      projectIdsToQueues.set(project.id, queueIds);
+    }),
+  );
+  return projectIdsToQueues;
+}
+
+async function generateQueues(
+  project: Project,
+  configIdsAndNames: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[],
+) {
+  const queue = {
+    id: `queue-${v4()}`,
+    name: "Default",
+    description: "Default queue",
+    scoreConfigIds: configIdsAndNames.map((config) => config.id),
+    projectId: project.id,
+  };
+
+  await prisma.annotationQueue.upsert({
+    where: {
+      projectId_name: {
+        projectId: queue.projectId,
+        name: queue.name,
+      },
+    },
+    create: {
+      ...queue,
+    },
+    update: {
+      id: queue.id,
+    },
+  });
+
+  return [queue.id];
+}

packages/shared/scripts/seeder/utils/README.md

@@ -0,0 +1,160 @@
+# Langfuse Seeder System
+
+System for generating test data in ClickHouse and PostgreSQL for Langfuse development and testing.
+
+## Architecture Overview
+
+```
+seeder/
+├── types.ts                 # Core interfaces and types
+├── data-generators.ts       # Data generation logic
+├── clickhouse-builder.ts    # ClickHouse query building
+├── seeder-orchestrator.ts   # Main orchestration logic
+├── postgres-seed-constants.ts  # PostgreSQL data constants
+├── clickhouse-seed-constants.ts  # ClickHouse data constants
+└── seed-helpers.ts          # Utility functions
+```
+
+## Quick Start
+
+```typescript
+import { SeederOrchestrator } from "./seeder/seeder-orchestrator";
+
+const orchestrator = new SeederOrchestrator();
+
+// Full seed (datasets + evaluation + synthetic data)
+await orchestrator.executeFullSeed(projectIds, {
+  numberOfDays: 30,
+  totalObservations: 10000,
+  numberOfRuns: 3,
+});
+
+// Individual data types
+await orchestrator.createDatasetExperimentData(projectIds, config);
+await orchestrator.createEvaluationData(projectIds);
+await orchestrator.createSyntheticData(projectIds, config);
+```
+
+## Generated Data
+
+### 1. Dataset Experiment Data
+
+- **Purpose**: Realistic experiment traces based on actual datasets
+- **Environment**: `langfuse-prompt-experiments`
+- **Structure**: Each dataset item links to a trace with a single generation observation
+- **ID Pattern**: `trace-dataset-{datasetName}-{itemIndex}-{projectId}-{runNumber}`
+
+### 2. Evaluation Data
+
+- **Purpose**: Evaluation metrics and scoring data - to be linked to evaluation logs
+- **Environment**: `langfuse-evaluation`
+- **Structure**: Traces with multiple observations and comprehensive scoring
+- **ID Pattern**: `trace-eval-{index}-{projectId}`
+
+### 3. Synthetic Data
+
+- **Purpose**: Large-scale realistic tracing data
+- **Environment**: `default`
+- **Structure**: Hierarchical traces with multiple observations and scores
+- **ID Pattern**: `trace-synthetic-{index}-{projectId}`
+
+## Abstraction Architecture
+
+### DataGenerator
+
+Generates realistic data for all three types. If you need to change any clickhouse data, you should modify this class. Key methods:
+
+- `generateDatasetTrace()` - Creates traces from dataset items
+- `generateSyntheticTraces()` - Creates realistic synthetic traces
+- `generateEvaluationTraces()` - Creates evaluation-focused traces
+
+### ClickHouseQueryBuilder
+
+Builds optimized ClickHouse insert queries. No need to edit this file. Handles proper escaping and type handling.
+
+### SeederOrchestrator
+
+Main coordination class that:
+
+- Loads file content for realistic inputs/outputs
+- Coordinates data generation and insertion
+- Handles batching and error recovery
+- Provides logging and statistics
+
+## Making Changes
+
+### Configuration Options
+
+```typescript
+interface SeederConfig {
+  numberOfDays: number; // How far back to generate timestamps
+  numberOfRuns?: number; // How many experiment runs per dataset
+  totalObservations?: number; // Total observations for synthetic data
+}
+```
+
+### Extending the System
+
+#### Adding New Data Types
+
+1. Add interface to `types.ts`
+2. Add generator method to `DataGenerator`
+3. Add query builder method to `ClickHouseQueryBuilder`
+4. Add orchestration method to `SeederOrchestrator`
+5. Update interdependency documentation
+
+#### Adding New File Sources
+
+1. Add file path to `SeederOrchestrator.loadFileContent()`
+2. Add processing logic to `DataGenerator`
+3. Update `FileContent` interface if needed
+
+#### Changing Data Distribution
+
+1. Modify generator methods in `DataGenerator`
+2. Update constants in `clickhouse-seed-constants.ts`
+3. Test with small datasets first
+
+#### Changing ID Generation
+
+1. **Check**: All places that query ClickHouse by ID
+2. **Check**: PostgreSQL foreign key references
+3. **Check**: Dataset run item and evaluation trace creation logic
+4. **Action**: Update `seed-helpers.ts` functions consistently
+
+#### Changing Environment Names
+
+1. **Check**: All ClickHouse queries that filter by environment
+2. **Check**: PostgreSQL dataset and prompt environment fields
+3. **Check**: UI environment filtering logic
+4. **Action**: Update constants in both systems
+
+#### Changing Data Structure
+
+1. **Check**: ClickHouse table schema compatibility
+2. **Check**: PostgreSQL table relationships
+3. **Check**: API response serialization
+4. **Action**: Update both schemas before changing data generation
+
+#### Adding New Data Types
+
+1. **Check**: Whether PostgreSQL needs corresponding tables
+2. **Check**: Whether new foreign key relationships are needed
+3. **Check**: Whether UI needs to handle new data types
+4. **Action**: Plan database migrations carefully
+
+## File Dependencies
+
+### Required Files
+
+```
+packages/shared/clickhouse/
+├── nested_json.json      # Large JSON for realistic inputs
+├── markdown.txt          # Markdown content for document analysis
+└── chat_ml_json.json     # Chat ML format examples
+```
+
+### Constants Files
+
+- `postgres-seed-constants.ts` - Datasets, prompts, and PostgreSQL data
+- `clickhouse-seed-constants.ts` - ClickHouse-specific constants (models, names)

packages/shared/scripts/seeder/utils/chat_ml_json.json

@@ -0,0 +1,135 @@
+{
+  "messages": [
+    {
+      "content": "what is the weather in sf",
+      "additional_kwargs": {},
+      "response_metadata": {},
+      "type": "human",
+      "name": null,
+      "id": "2166b887-b9fb-4282-a9f7-828d4a6cca54",
+      "example": false
+    },
+    {
+      "content": "",
+      "additional_kwargs": {
+        "tool_calls": [
+          {
+            "id": "call_pNLR4DoZiptb5xGlb299QsoN",
+            "function": {
+              "arguments": "{\"query\":\"current weather in San Francisco\"}",
+              "name": "search"
+            },
+            "type": "function"
+          }
+        ],
+        "refusal": null
+      },
+      "response_metadata": {
+        "token_usage": {
+          "completion_tokens": 17,
+          "prompt_tokens": 48,
+          "total_tokens": 65,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          }
+        },
+        "model_name": "gpt-4o-mini-2024-07-18",
+        "system_fingerprint": "fp_34a54ae93c",
+        "finish_reason": "tool_calls",
+        "logprobs": null
+      },
+      "type": "ai",
+      "name": null,
+      "id": "run-14c4801c-ae47-4c4a-8d49-29ba08c5679d-0",
+      "example": false,
+      "tool_calls": [
+        {
+          "name": "search",
+          "args": {
+            "query": "current weather in San Francisco"
+          },
+          "id": "call_pNLR4DoZiptb5xGlb299QsoN",
+          "type": "tool_call"
+        }
+      ],
+      "invalid_tool_calls": [],
+      "usage_metadata": {
+        "input_tokens": 48,
+        "output_tokens": 17,
+        "total_tokens": 65,
+        "input_token_details": {
+          "audio": 0,
+          "cache_read": 0
+        },
+        "output_token_details": {
+          "audio": 0,
+          "reasoning": 0
+        }
+      }
+    },
+    {
+      "content": "It's 60 degrees and foggy.",
+      "additional_kwargs": {},
+      "response_metadata": {},
+      "type": "tool",
+      "name": "search",
+      "id": "220ba511-6816-4d1a-8acf-8e5791c2d88e",
+      "tool_call_id": "call_pNLR4DoZiptb5xGlb299QsoN",
+      "artifact": null,
+      "status": "success"
+    },
+    {
+      "content": "The current weather in San Francisco is 60 degrees and foggy.",
+      "additional_kwargs": {
+        "refusal": null
+      },
+      "response_metadata": {
+        "token_usage": {
+          "completion_tokens": 15,
+          "prompt_tokens": 80,
+          "total_tokens": 95,
+          "completion_tokens_details": {
+            "accepted_prediction_tokens": 0,
+            "audio_tokens": 0,
+            "reasoning_tokens": 0,
+            "rejected_prediction_tokens": 0
+          },
+          "prompt_tokens_details": {
+            "audio_tokens": 0,
+            "cached_tokens": 0
+          }
+        },
+        "model_name": "gpt-4o-mini-2024-07-18",
+        "system_fingerprint": "fp_34a54ae93c",
+        "finish_reason": "stop",
+        "logprobs": null
+      },
+      "type": "ai",
+      "name": null,
+      "id": "run-f051be3b-2656-4b19-8704-a2e492306131-0",
+      "example": false,
+      "tool_calls": [],
+      "invalid_tool_calls": [],
+      "usage_metadata": {
+        "input_tokens": 80,
+        "output_tokens": 15,
+        "total_tokens": 95,
+        "input_token_details": {
+          "audio": 0,
+          "cache_read": 0
+        },
+        "output_token_details": {
+          "audio": 0,
+          "reasoning": 0
+        }
+      }
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -0,0 +1,242 @@
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../../../src/server";
+import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
+import {
+  createTracesCh,
+  createObservationsCh,
+  createScoresCh,
+} from "../../../src/server";
+import { InsertResult } from "@clickhouse/client";
+
+/**
+ * Builds or executes ClickHouse SQL INSERT queries for seeding test data.
+ *
+ * Use executeXxxInsert() for custom curated data with detailed control.
+ * Use buildBulkXxxInsert() for large datasets (>1000 items) for random distribution of data.
+ */
+export class ClickHouseQueryBuilder {
+  private escapeString(str: string): string {
+    return str.replace(/'/g, "''");
+  }
+
+  /**
+   * Creates INSERT query for trace data using VALUES syntax.
+   * Use for: Small datasets, detailed trace objects with all fields populated.
+   */
+  async executeTracesInsert(
+    traces: TraceRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createTracesCh(traces);
+  }
+
+  /**
+   * Creates INSERT query for observation data using VALUES syntax.
+   * Use for: Small datasets, observations that link to postgres data (e.g. dataset runs)
+   */
+  async executeObservationsInsert(
+    observations: ObservationRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createObservationsCh(observations);
+  }
+
+  /**
+   * Creates INSERT query for score data using VALUES syntax.
+   * Use for: Small datasets, scores with custom values and metadata.
+   */
+  async executeScoresInsert(
+    scores: ScoreRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createScoresCh(scores);
+  }
+
+  /**
+   * Creates INSERT using ClickHouse numbers() function.
+   * Use for: Large datasets (>1000 traces), realistic timestamps, bulk generation.
+   */
+  buildBulkTracesInsert(
+    projectId: string,
+    count: number,
+    environment: string = "default",
+    fileContent?: { heavyMarkdown: string; nestedJson: any; chatMlJson: any },
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    // Escape file content if provided
+    const escapedHeavyMarkdown = fileContent
+      ? this.escapeString(fileContent.heavyMarkdown)
+      : "Sample heavy markdown content";
+    const escapedNestedJson = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.nestedJson))
+      : '{"sample": "nested json"}';
+    const escapedChatMl = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.chatMlJson))
+      : '{"messages": []}';
+
+    return `
+      INSERT INTO traces
+      SELECT 
+        concat('trace-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+        concat('trace-', toString(number % 10)) AS name,
+        if(randUniform(0, 1) < 0.3, concat('user_', toString(rand() % 1000)), NULL) AS user_id,
+        map('generated', 'bulk') AS metadata,
+        NULL AS release,
+        NULL AS version,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        if(rand() < 0.8, true, false) AS public,
+        if(rand() < 0.1, true, false) AS bookmarked,
+        array() AS tags,
+        if(randUniform(0, 1) < 0.3, '${escapedHeavyMarkdown}',
+          '${escapedChatMl}'
+        ) AS input,
+        if(randUniform(0, 1) < 0.2, '${escapedNestedJson}',
+          '${escapedChatMl}'
+        ) AS output,
+        if(randUniform(0, 1) < 0.3, concat('session_', toString(rand() % 100)), NULL) AS session_id,
+        now() AS created_at,
+        now() AS updated_at,
+        now() AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${count});
+    `;
+  }
+
+  /**
+   * Creates observations with automatic prompt linking (10% rate).
+   * Use for: Large datasets, hierarchical observations, cost/latency variation.
+   */
+  buildBulkObservationsInsert(
+    projectId: string,
+    tracesCount: number,
+    observationsPerTrace: number = 5,
+    environment: string = "default",
+    fileContent?: { heavyMarkdown: string; nestedJson: any; chatMlJson: any },
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    const totalObservations = tracesCount * observationsPerTrace;
+
+    // Escape file content if provided
+    const escapedHeavyMarkdown = fileContent
+      ? this.escapeString(fileContent.heavyMarkdown)
+      : "Sample heavy markdown content";
+    const escapedNestedJson = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.nestedJson))
+      : '{"sample": "nested json"}';
+    const escapedChatMl = fileContent
+      ? this.escapeString(JSON.stringify(fileContent.chatMlJson))
+      : '{"messages": []}';
+
+    return `
+      INSERT INTO observations
+      SELECT 
+        concat('obs-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        concat('trace-bulk-', toString(number % ${tracesCount}), '-${projectId.slice(-8)}') AS trace_id,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
+        if(number % 6 = 0, NULL, toString(number - 1)) AS parent_observation_id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
+        addMilliseconds(start_time, 
+          case 
+            when type = 'GENERATION' then floor(randUniform(5, 30))
+            when type = 'SPAN' then floor(randUniform(1, 50))
+            else floor(randUniform(1, 10))
+          end) AS end_time,
+        case
+          when type = 'GENERATION' then concat('generation-', toString(number % 10))
+          when type = 'SPAN' then concat('span-', toString(number % 10))
+          else concat('event-', toString(number % 10))
+        end AS name,
+        map('key', 'value') AS metadata,
+        if(randUniform(0, 1) < 0.85, 'DEFAULT', if(randUniform(0, 1) < 0.7, 'DEBUG', if(randUniform(0, 1) < 0.3, 'ERROR', 'WARNING'))) AS level,
+        NULL AS status_message,
+        NULL AS version,
+        if(type = 'GENERATION', 
+          if(randUniform(0, 1) < 0.4, '${escapedHeavyMarkdown}', '${escapedChatMl}'),
+          NULL) AS input,
+        if(type = 'GENERATION', 
+          if(randUniform(0, 1) < 0.3, '${escapedNestedJson}', '${escapedChatMl}'),
+          NULL) AS output,
+        if(type = 'GENERATION', 'gpt-4', NULL) AS provided_model_name,
+        if(type = 'GENERATION', concat('model_', toString(rand() % 1000)), NULL) AS internal_model_id,
+        if(type = 'GENERATION', '{"temperature": 0.7}', '{}') AS model_parameters,
+        if(type = 'GENERATION', map('input', toUInt64(randUniform(20, 200)), 'output', toUInt64(randUniform(10, 100)), 'total', toUInt64(randUniform(30, 300))), map()) AS provided_usage_details,
+        if(type = 'GENERATION', map('input', toUInt64(randUniform(20, 200)), 'output', toUInt64(randUniform(10, 100)), 'total', toUInt64(randUniform(30, 300))), map()) AS usage_details,
+        if(type = 'GENERATION', map('input', toDecimal64(randUniform(0.00001, 0.001), 8), 'output', toDecimal64(randUniform(0.00001, 0.002), 8), 'total', toDecimal64(randUniform(0.00002, 0.003), 8)), map()) AS provided_cost_details,
+        if(type = 'GENERATION', map('input', toDecimal64(randUniform(0.00001, 0.001), 8), 'output', toDecimal64(randUniform(0.00001, 0.002), 8), 'total', toDecimal64(randUniform(0.00002, 0.003), 8)), map()) AS cost_details,
+        if(type = 'GENERATION', toDecimal64(randUniform(0.00002, 0.003), 8), NULL) AS total_cost,
+        if(type = 'GENERATION', addMilliseconds(start_time, floor(randUniform(100, 500))), NULL) AS completion_start_time,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.id).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_id,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.name).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_name,
+        if("type" = 'GENERATION' AND number % 10 = 0,
+        arrayElement(['${SEED_TEXT_PROMPTS.map((p) => p.version).join("','")}'], 1 + (number % ${SEED_TEXT_PROMPTS.length})),
+        NULL) AS prompt_version,
+        start_time AS created_at,
+        start_time AS updated_at,
+        start_time AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${totalObservations});
+    `;
+  }
+
+  /**
+   * Creates scores with mixed data types (NUMERIC/BOOLEAN/CATEGORICAL).
+   * Use for: Large datasets, varied score distributions, synthetic metrics.
+   */
+  buildBulkScoresInsert(
+    projectId: string,
+    tracesCount: number,
+    scoresPerTrace: number = 2,
+    environment: string = "default",
+    opts: { numberOfDays: number } = { numberOfDays: 1 },
+  ): string {
+    const totalScores = tracesCount * scoresPerTrace;
+
+    return `
+      INSERT INTO scores
+      SELECT 
+        concat('score-bulk-', toString(number), '-${projectId.slice(-8)}') AS id,
+        toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+        '${projectId}' AS project_id,
+        '${environment}' AS environment,
+        concat('trace-bulk-', toString(number % ${tracesCount}), '-${projectId.slice(-8)}') AS trace_id,
+        if(randUniform(0, 1) < 0.3, concat('session_', toString(rand() % 100)), NULL) AS session_id,
+        NULL AS dataset_run_id,
+        if(randUniform(0, 1) < 0.1, concat('obs-bulk-', toString(rand() % (${tracesCount} * 5)), '-${projectId.slice(-8)}'), NULL) AS observation_id,
+        concat('metric_', toString((number % ${scoresPerTrace * 5}) + 1)) AS name,
+        case 
+          when (number % 3) = 0 then toDecimal64(randUniform(0, 100), 8)
+          when (number % 3) = 1 then if(randUniform(0, 1) < 0.5, 1, 0)
+          else NULL
+        end AS value,
+        'API' AS source,
+        'Generated synthetic score' AS comment,
+        map() AS metadata,
+        NULL AS author_user_id,
+        NULL AS config_id,
+        case 
+          when (number % 3) = 0 then 'NUMERIC'
+          when (number % 3) = 1 then 'BOOLEAN'
+          else 'CATEGORICAL'
+        end AS data_type,
+        case 
+          when (number % 3) = 1 then if(value = 1, 'true', 'false')
+          when (number % 3) = 2 then concat('category_', toString((rand() % 5) + 1))
+          else NULL
+        end AS string_value,
+        NULL AS queue_id,
+        timestamp AS created_at,
+        timestamp AS updated_at,
+        timestamp AS event_ts,
+        0 AS is_deleted
+      FROM numbers(${totalScores});
+    `;
+  }
+}

packages/shared/scripts/seeder/utils/clickhouse-seed-constants.ts

@@ -0,0 +1,97 @@
+export const REALISTIC_TRACE_NAMES = [
+  "LangGraph",
+  "ChatCompletion",
+  "DocumentAnalysis",
+  "CodeGeneration",
+  "DataProcessing",
+  "QueryExecution",
+  "ModelInference",
+  "WebScraping",
+  "TextSummarization",
+  "ImageClassification",
+];
+
+export const REALISTIC_SPAN_NAMES = [
+  "agent",
+  "tools",
+  "search",
+  "retrieval",
+  "preprocessing",
+  "validation",
+  "transformation",
+  "classification",
+  "extraction",
+  "postprocessing",
+];
+
+export const REALISTIC_GENERATION_NAMES = [
+  "ChatOpenAI",
+  "GPT-4",
+  "Claude-3",
+  "Gemini",
+  "Llama-2",
+  "PaLM",
+  "CodeLlama",
+  "Mistral",
+  "Falcon",
+  "Vicuna",
+];
+
+export const REALISTIC_MODELS = [
+  "gpt-4o-mini-2024-07-18",
+  "gpt-4-turbo-2024-04-09",
+  "claude-3-haiku-20240307",
+  "claude-3-sonnet-20240229",
+  "claude-3-opus-20240229",
+  "gemini-pro",
+  "llama-2-70b-chat",
+  "mistral-7b-instruct",
+  "codellama-34b-instruct",
+];
+
+export const REALISTIC_USER_INPUTS = [
+  "What is the weather in San Francisco?",
+  "Summarize this document for me",
+  "Generate a Python function to sort a list",
+  "Explain quantum computing in simple terms",
+  "Analyze this data and provide insights",
+  "Translate this text to Spanish",
+  "Create a marketing email for our product",
+  "Debug this code and fix the errors",
+  "What are the latest trends in AI?",
+  "Help me plan a trip to Europe",
+];
+
+export const REALISTIC_AI_RESPONSES = [
+  "The current weather in San Francisco is 60 degrees and foggy.",
+  "Here's a summary of the key points from the document...",
+  "Here's a Python function that sorts a list efficiently...",
+  "Quantum computing uses quantum mechanics principles...",
+  "Based on the data analysis, I found the following insights...",
+  "Here's the Spanish translation of your text...",
+  "I've created a compelling marketing email for your product...",
+  "I found several issues in your code and here are the fixes...",
+  "The latest AI trends include large language models...",
+  "Here's a detailed 10-day European itinerary for you...",
+];
+
+export const REALISTIC_METADATA_EXAMPLES = [
+  { thread_id: 42, session_type: "interactive" },
+  { user_id: "user_123", conversation_id: "conv_456" },
+  { model_version: "v2.1", temperature: 0.7 },
+  { request_id: "req_789", timestamp: "2024-05-23T15:42:11.996Z" },
+  { environment: "production", region: "us-west-2" },
+  {
+    document_type: "state_of_union",
+    file_size: "142KB",
+    processing_time: "2.3s",
+  },
+  { data_source: "product_catalog", record_count: 30, format: "nested_json" },
+  { analysis_type: "sentiment", language: "en", confidence: 0.92 },
+  {
+    extraction_task: "policy_analysis",
+    domain: "politics",
+    entities_found: 15,
+  },
+  { file_type: "JSON", validation: "passed", schema_version: "v1.2" },
+];

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -0,0 +1,594 @@
+import { FileContent, DatasetItemInput } from "./types";
+import {
+  REALISTIC_TRACE_NAMES,
+  REALISTIC_SPAN_NAMES,
+  REALISTIC_GENERATION_NAMES,
+  REALISTIC_MODELS,
+} from "./clickhouse-seed-constants";
+import {
+  generateDatasetRunTraceId,
+  generateEvalObservationId,
+  generateEvalScoreId,
+  generateEvalTraceId,
+} from "./seed-helpers";
+import { v4 as uuidv4 } from "uuid";
+import {
+  FAILED_EVAL_TRACE_INTERVAL,
+  SEED_EVALUATOR_CONFIGS,
+} from "./postgres-seed-constants";
+import {
+  createTrace,
+  createObservation,
+  createTraceScore,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+  TraceRecordInsertType,
+} from "../../../src/server";
+
+/**
+ * Generates realistic test data for traces, observations, and scores.
+ *
+ * Use generateXxxTraces() for creating different data types:
+ * - generateDatasetTrace(): For dataset experiment runs (langfuse-prompt-experiments env)
+ * - generateEvaluationTraces(): For evaluation data (langfuse-evaluation env)
+ * - generateSyntheticTraces(): For large-scale synthetic data (default env)
+ */
+export class DataGenerator {
+  private static instance: DataGenerator;
+  private fileContent: FileContent | null = null;
+
+  static getInstance(): DataGenerator {
+    if (!DataGenerator.instance) {
+      DataGenerator.instance = new DataGenerator();
+    }
+    return DataGenerator.instance;
+  }
+
+  setFileContent(content: FileContent) {
+    this.fileContent = content;
+  }
+
+  private randomElement<T>(array: T[]): T {
+    return array[Math.floor(Math.random() * array.length)];
+  }
+
+  private randomBoolean(probability: number = 0.5): boolean {
+    return Math.random() < probability;
+  }
+
+  private randomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
+
+  /**
+   * Creates traces from dataset items for experiment runs.
+   * Use for: Dataset experiments scenarios.
+   */
+  generateDatasetTrace(
+    input: DatasetItemInput,
+    projectId: string,
+  ): TraceRecordInsertType {
+    const traceId = generateDatasetRunTraceId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    let traceInput: string;
+    let traceOutput: string;
+
+    // Transform dataset item based on type
+    if (input.datasetName === "demo-countries-dataset") {
+      const data = input.item as { input: { country: string }; output: string };
+      traceInput = `What is the capital of ${data.input.country}?`;
+      traceOutput = `The capital of ${data.input.country} is ${data.output}.`;
+    } else if (input.datasetName === "demo-english-transcription-dataset") {
+      const data = input.item as { input: { word: string }; output: string };
+      traceInput = `What is the IPA transcription of the word "${data.input.word}"?`;
+      traceOutput = `The IPA transcription of "${data.input.word}" is ${data.output}.`;
+    } else {
+      traceInput = JSON.stringify(input.item.input);
+      traceOutput = JSON.stringify(input.item.output);
+    }
+
+    return createTrace({
+      id: traceId,
+      project_id: projectId,
+      name: `dataset-run-item-${uuidv4()}`,
+      input: traceInput,
+      output: traceOutput,
+      environment: "langfuse-prompt-experiments",
+      metadata: { experimentType: "langfuse-prompt-experiments" },
+      public: false,
+      bookmarked: false,
+      session_id: null,
+      tags: [],
+    });
+  }
+
+  /**
+   * Creates observations for dataset experiment traces with variable costs/latency.
+   * Use for: Dataset experiments requiring detailed observation tracking.
+   */
+  generateDatasetObservation(
+    trace: TraceRecordInsertType,
+    input: DatasetItemInput,
+    projectId: string,
+  ): ObservationRecordInsertType {
+    const observationId = `observation-dataset-${input.datasetName}-${input.itemIndex}-${input.runNumber}-${projectId.slice(-8)}`;
+
+    // Generate variable usage and cost for each observation
+    const inputTokens = this.randomInt(30, 150);
+    const outputTokens = this.randomInt(10, 80);
+    const totalTokens = inputTokens + outputTokens;
+
+    // Cost should be fraction of cents (0.0001-0.01 range)
+    const inputCost = (inputTokens * this.randomInt(1, 5)) / 1000000; // $0.000001-0.000005 per token
+    const outputCost = (outputTokens * this.randomInt(2, 10)) / 1000000; // $0.000002-0.00001 per token
+    const totalCost = inputCost + outputCost;
+
+    return createObservation({
+      id: observationId,
+      trace_id: trace.id,
+      project_id: projectId,
+      type: "GENERATION",
+      name: `dataset-generation-${input.itemIndex}-run-${input.runNumber}`,
+      input: trace.input,
+      output: trace.output,
+      provided_model_name: "gpt-3.5-turbo",
+      model_parameters: JSON.stringify({ temperature: 0.7 }),
+      usage_details: {
+        input: inputTokens,
+        output: outputTokens,
+        total: totalTokens,
+      },
+      provided_usage_details: {
+        input: inputTokens,
+        output: outputTokens,
+        total: totalTokens,
+      },
+      cost_details: {
+        input: Math.round(inputCost * 100000) / 100000, // Round to 5 decimal places
+        output: Math.round(outputCost * 100000) / 100000,
+        total: Math.round(totalCost * 100000) / 100000,
+      },
+      provided_cost_details: {
+        input: Math.round(inputCost * 100000) / 100000,
+        output: Math.round(outputCost * 100000) / 100000,
+        total: Math.round(totalCost * 100000) / 100000,
+      },
+      total_cost: Math.round(totalCost * 100000) / 100000,
+      environment: "langfuse-prompt-experiments",
+    });
+  }
+
+  /**
+   * Creates large-scale synthetic traces for performance testing.
+   * Use for: Load testing, dashboard demos, realistic usage simulation.
+   */
+  generateSyntheticTraces(
+    projectId: string,
+    count: number,
+  ): TraceRecordInsertType[] {
+    const traces: TraceRecordInsertType[] = [];
+
+    for (let i = 0; i < count; i++) {
+      const trace = createTrace({
+        id: `trace-synthetic-${i}-${projectId.slice(-8)}`,
+        project_id: projectId,
+        name: this.randomElement(REALISTIC_TRACE_NAMES),
+        input: this.generateTraceInput(),
+        output: this.generateTraceOutput(),
+        user_id: this.randomBoolean(0.3)
+          ? `user_${this.randomInt(1, 1000)}`
+          : null,
+        session_id: this.randomBoolean(0.3)
+          ? `session_${this.randomInt(1, 100)}`
+          : undefined,
+        environment: "default",
+        metadata: { generated: "synthetic" },
+        tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
+        public: this.randomBoolean(0.8),
+        bookmarked: this.randomBoolean(0.1),
+        release: this.randomBoolean(0.4)
+          ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
+          : undefined,
+        version: this.randomBoolean(0.4)
+          ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
+          : undefined,
+      });
+
+      traces.push(trace);
+    }
+
+    return traces;
+  }
+
+  generateEvaluationObservations(
+    traces: TraceRecordInsertType[],
+    observationsPerTrace: number = 5,
+    projectId: string,
+  ): ObservationRecordInsertType[] {
+    const observations: ObservationRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      traces.forEach((trace, traceIndex) => {
+        for (let i = 0; i < observationsPerTrace; i++) {
+          const obsType = this.randomBoolean(0.47)
+            ? "GENERATION"
+            : this.randomBoolean(0.94)
+              ? "SPAN"
+              : "EVENT";
+
+          const observation: ObservationRecordInsertType = createObservation({
+            id: generateEvalObservationId(
+              evalJobConfiguration.evalTemplateId,
+              traceIndex,
+              projectId,
+            ),
+            trace_id: trace.id,
+            project_id: projectId,
+            parent_observation_id: undefined,
+            type: obsType,
+            name:
+              obsType === "GENERATION"
+                ? this.randomElement(REALISTIC_GENERATION_NAMES)
+                : obsType === "SPAN"
+                  ? this.randomElement(REALISTIC_SPAN_NAMES)
+                  : `event_${i % 10}`,
+            level: this.randomBoolean(0.85)
+              ? "DEFAULT"
+              : this.randomBoolean(0.7)
+                ? "DEBUG"
+                : this.randomBoolean(0.3)
+                  ? "ERROR"
+                  : "WARNING",
+            input:
+              obsType === "GENERATION"
+                ? this.randomBoolean(0.4)
+                  ? this.fileContent?.heavyMarkdown || "Sample input"
+                  : JSON.stringify(this.fileContent?.chatMlJson || {})
+                : undefined,
+            output:
+              obsType === "GENERATION"
+                ? this.randomBoolean(0.3)
+                  ? JSON.stringify(this.fileContent?.nestedJson || {})
+                  : JSON.stringify(this.fileContent?.chatMlJson || {})
+                : undefined,
+            provided_model_name:
+              obsType === "GENERATION"
+                ? this.randomElement(REALISTIC_MODELS)
+                : undefined,
+            model_parameters:
+              obsType === "GENERATION"
+                ? JSON.stringify({ temperature: 0.7 })
+                : undefined,
+            usage_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(20, 200),
+                    output: this.randomInt(10, 100),
+                    total: this.randomInt(30, 300),
+                  }
+                : undefined,
+            provided_usage_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(20, 200),
+                    output: this.randomInt(10, 100),
+                    total: this.randomInt(30, 300),
+                  }
+                : undefined,
+            cost_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(1, 10) / 100000,
+                    output: this.randomInt(1, 20) / 100000,
+                    total: this.randomInt(2, 30) / 100000,
+                  }
+                : undefined,
+            provided_cost_details:
+              obsType === "GENERATION"
+                ? {
+                    input: this.randomInt(1, 10) / 100000,
+                    output: this.randomInt(1, 20) / 100000,
+                    total: this.randomInt(2, 30) / 100000,
+                  }
+                : undefined,
+            environment: "langfuse-evaluation",
+          });
+
+          observations.push(observation);
+        }
+      });
+    }
+
+    return observations;
+  }
+
+  /**
+   * Creates synthetic observations with automatic prompt linking (5% rate).
+   * Use for: Large datasets, hierarchical observation structures, cost variation.
+   */
+  generateSyntheticObservations(
+    traces: TraceRecordInsertType[],
+    observationsPerTrace: number = 5,
+  ): ObservationRecordInsertType[] {
+    const observations: ObservationRecordInsertType[] = [];
+
+    traces.forEach((trace, traceIndex) => {
+      for (let i = 0; i < observationsPerTrace; i++) {
+        const obsType = this.randomElement(["GENERATION", "SPAN", "EVENT"]);
+
+        const observation: ObservationRecordInsertType = createObservation({
+          id: `obs-synthetic-${traceIndex}-${i}`,
+          trace_id: trace.id,
+          project_id: trace.project_id,
+          parent_observation_id:
+            i > 0 ? `obs-synthetic-${traceIndex}-${i - 1}` : undefined,
+          type: obsType as any,
+          name:
+            obsType === "GENERATION"
+              ? this.randomElement(REALISTIC_GENERATION_NAMES)
+              : this.randomElement(REALISTIC_SPAN_NAMES),
+          input:
+            obsType === "GENERATION"
+              ? this.generateObservationInput()
+              : undefined,
+          output:
+            obsType === "GENERATION"
+              ? this.generateObservationOutput()
+              : undefined,
+          provided_model_name:
+            obsType === "GENERATION"
+              ? this.randomElement(REALISTIC_MODELS)
+              : undefined,
+          model_parameters:
+            obsType === "GENERATION"
+              ? JSON.stringify({ temperature: 0.7 })
+              : undefined,
+          usage_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(20, 200),
+                  output: this.randomInt(10, 100),
+                  total: this.randomInt(30, 300),
+                }
+              : undefined,
+          provided_usage_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(20, 200),
+                  output: this.randomInt(10, 100),
+                  total: this.randomInt(30, 300),
+                }
+              : undefined,
+          cost_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(1, 10) / 100000,
+                  output: this.randomInt(1, 20) / 100000,
+                  total: this.randomInt(2, 30) / 100000,
+                }
+              : undefined,
+          provided_cost_details:
+            obsType === "GENERATION"
+              ? {
+                  input: this.randomInt(1, 10) / 100000,
+                  output: this.randomInt(1, 20) / 100000,
+                  total: this.randomInt(2, 30) / 100000,
+                }
+              : undefined,
+          level: this.randomBoolean(0.85)
+            ? "DEFAULT"
+            : this.randomBoolean(0.7)
+              ? "DEBUG"
+              : this.randomBoolean(0.5)
+                ? "WARNING"
+                : "ERROR",
+          environment: trace.environment,
+        });
+
+        observations.push(observation);
+      }
+    });
+
+    return observations;
+  }
+
+  generateSyntheticScores(
+    traces: TraceRecordInsertType[],
+    observations: ObservationRecordInsertType[],
+    scoresPerTrace: number = 2,
+  ): ScoreRecordInsertType[] {
+    const scores: ScoreRecordInsertType[] = [];
+
+    traces.forEach((trace, traceIndex) => {
+      for (let i = 0; i < scoresPerTrace; i++) {
+        const scoreType = this.randomElement([
+          "NUMERIC",
+          "CATEGORICAL",
+          "BOOLEAN",
+        ]);
+
+        let value: number | undefined;
+        let stringValue: string | undefined;
+
+        switch (scoreType) {
+          case "NUMERIC":
+            value = Math.random() * 100;
+            break;
+          case "CATEGORICAL":
+            stringValue = `category_${this.randomInt(1, 5)}`;
+            break;
+          case "BOOLEAN":
+            value = this.randomBoolean() ? 1 : 0;
+            stringValue = value === 1 ? "true" : "false";
+            break;
+        }
+
+        const score: ScoreRecordInsertType = createTraceScore({
+          id: `score-synthetic-${traceIndex}-${i}`,
+          project_id: trace.project_id,
+          trace_id: trace.id,
+          observation_id: this.randomBoolean(0.1)
+            ? this.randomElement(
+                observations.filter((o) => o.trace_id === trace.id),
+              )?.id
+            : undefined,
+          name: `metric_${this.randomInt(1, 10)}`,
+          value,
+          string_value: stringValue,
+          data_type: scoreType as any,
+          source: "API",
+          comment: "Generated score\ntest",
+          environment: trace.environment,
+        });
+
+        scores.push(score);
+      }
+    });
+
+    return scores;
+  }
+
+  /**
+   * Creates evaluation traces for testing evaluator configurations.
+   * Use for: Evaluation testing, score validation, evaluator development.
+   */
+  generateEvaluationTraces(
+    projectId: string,
+    count: number,
+  ): TraceRecordInsertType[] {
+    const traces: TraceRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      for (let i = 0; i < count; i++) {
+        const traceId = generateEvalTraceId(
+          evalJobConfiguration.evalTemplateId,
+          i,
+          projectId,
+        );
+        const trace = createTrace({
+          id: traceId,
+          session_id: null,
+          project_id: projectId,
+          name: this.randomElement(REALISTIC_TRACE_NAMES),
+          input: this.generateEvaluationInput(),
+          output: this.generateEvaluationOutput(),
+          user_id: this.randomBoolean(0.3)
+            ? `user_${this.randomInt(1, 1000)}`
+            : null,
+          environment: "langfuse-evaluation",
+          metadata: { purpose: "evaluation" },
+          tags: this.randomBoolean(0.3) ? ["production", "ai-agent"] : [],
+          public: this.randomBoolean(0.8),
+          bookmarked: this.randomBoolean(0.1),
+          release: this.randomBoolean(0.4)
+            ? `v${this.randomInt(1, 5)}.${this.randomInt(0, 10)}`
+            : null,
+          version: this.randomBoolean(0.4)
+            ? `v${this.randomInt(1, 3)}.${this.randomInt(0, 20)}`
+            : null,
+        });
+
+        traces.push(trace);
+      }
+    }
+
+    return traces;
+  }
+
+  private generateTraceInput(): string {
+    if (!this.fileContent) return "Sample input";
+
+    // Match original logic: 30% chance of heavy markdown, otherwise chatML
+    return this.randomBoolean(0.3)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateTraceOutput(): string {
+    if (!this.fileContent) return "Sample output";
+
+    // Match original logic: 20% chance of nested JSON, otherwise chatML
+    return this.randomBoolean(0.2)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateObservationInput(): string {
+    if (!this.fileContent) return "Sample observation input";
+
+    // Match original logic: 40% chance of heavy markdown, otherwise chatML
+    return this.randomBoolean(0.4)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateObservationOutput(): string {
+    if (!this.fileContent) return "Sample observation output";
+
+    // Match original logic: 30% chance of nested JSON, otherwise chatML
+    return this.randomBoolean(0.3)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateEvaluationInput(): string {
+    if (!this.fileContent) return "Evaluation input";
+
+    return this.randomBoolean(0.3)
+      ? this.fileContent.heavyMarkdown
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  private generateEvaluationOutput(): string {
+    if (!this.fileContent) return "Evaluation output";
+
+    return this.randomBoolean(0.2)
+      ? JSON.stringify(this.fileContent.nestedJson)
+      : JSON.stringify(this.fileContent.chatMlJson);
+  }
+
+  /**
+   * Creates exactly one score per evaluation trace with prefixed IDs.
+   * Use for: Evaluation traces that need score validation, evaluator testing.
+   */
+  generateEvaluationScores(
+    traces: TraceRecordInsertType[],
+    _observations: ObservationRecordInsertType[],
+    projectId: string,
+  ): ScoreRecordInsertType[] {
+    const scores: ScoreRecordInsertType[] = [];
+
+    for (const evalJobConfiguration of SEED_EVALUATOR_CONFIGS) {
+      traces.forEach((trace, traceIndex) => {
+        if (traceIndex % FAILED_EVAL_TRACE_INTERVAL === 0) return;
+        // Create exactly one score per evaluation trace with prefixed ID
+        const score: ScoreRecordInsertType = createTraceScore({
+          id: generateEvalScoreId(
+            evalJobConfiguration.evalTemplateId,
+            traceIndex,
+            projectId,
+          ), // Use prefixed ID pattern
+          project_id: projectId,
+          trace_id: trace.id,
+          observation_id: undefined, // Score is for the entire trace, not a specific observation
+          name: `evaluation_score-${evalJobConfiguration.evalTemplateId}`,
+          value: Math.random() * 100, // Random evaluation score 0-100
+          string_value: undefined,
+          data_type: "NUMERIC",
+          source: "EVAL",
+          comment: "Evaluation trace score",
+          environment: trace.environment,
+        });
+
+        scores.push(score);
+      });
+    }
+
+    return scores;
+  }
+}

packages/shared/scripts/seeder/utils/markdown.txt

@@ -0,0 +1,266 @@
+# 🚀 Advanced AI Development Guide
+
+## 📋 Table of Contents
+- [Getting Started](#getting-started)
+- [Architecture Overview](#architecture)
+- [Implementation Details](#implementation)
+- [Best Practices](#best-practices)
+
+---
+
+## 🎯 Getting Started
+
+### Prerequisites
+\`\`\`bash
+npm install @langfuse/core
+pip install langfuse
+\`\`\`
+
+### Quick Setup
+1. **Initialize your project**
+   \`\`\`typescript
+   import { Langfuse } from 'langfuse'
+   
+   const langfuse = new Langfuse({
+     secretKey: process.env.LANGFUSE_SECRET_KEY,
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
+     baseUrl: 'https://cloud.langfuse.com'
+   })
+   \`\`\`
+
+2. **Create your first trace**
+   \`\`\`python
+   from langfuse import Langfuse
+   
+   langfuse = Langfuse()
+   trace = langfuse.trace(name="chat-application")
+   \`\`\`
+
+---
+
+## 🏗️ Architecture Overview
+
+### System Components
+
+| Component | Description | Status |
+|-----------|-------------|--------|
+| **Core Engine** | Main processing unit | ✅ Active |
+| **API Gateway** | Request routing | ✅ Active |
+| **Data Store** | Persistence layer | ⚠️ Maintenance |
+| **Analytics** | Metrics & insights | 🚧 Development |
+
+### Data Flow
+
+```mermaid
+graph TD
+    A[User Request] --> B[API Gateway]
+    B --> C{Route Decision}
+    C -->|Trace| D[Trace Handler]
+    C -->|Generation| E[Generation Handler]
+    C -->|Score| F[Score Handler]
+    D --> G[Database]
+    E --> G
+    F --> G
+```
+
+---
+
+## ⚙️ Implementation Details
+
+### Trace Management
+
+> **Note:** Traces are the foundation of observability in LLM applications.
+
+#### Creating Traces
+\`\`\`typescript
+// Basic trace creation
+const trace = langfuse.trace({
+  name: "user-query-processing",
+  userId: "user-123",
+  sessionId: "session-456",
+  metadata: {
+    environment: "production",
+    version: "2.1.0"
+  }
+})
+
+// Nested observations
+const span = trace.span({
+  name: "document-retrieval",
+  input: { query: "What is machine learning?" },
+  metadata: { vectorStore: "pinecone" }
+})
+
+const generation = span.generation({
+  name: "answer-generation",
+  model: "gpt-4",
+  input: retrievedDocs,
+  output: generatedAnswer,
+  usage: {
+    promptTokens: 1250,
+    completionTokens: 420,
+    totalTokens: 1670
+  }
+})
+\`\`\`
+
+### Advanced Features
+
+#### 🔄 Async Processing
+\`\`\`python
+import asyncio
+from langfuse import Langfuse
+
+async def process_batch():
+    langfuse = Langfuse()
+    
+    tasks = []
+    for item in batch_items:
+        task = asyncio.create_task(
+            process_item_with_tracing(langfuse, item)
+        )
+        tasks.append(task)
+    
+    results = await asyncio.gather(*tasks)
+    return results
+\`\`\`
+
+#### 🎯 Custom Scoring
+\`\`\`typescript
+// Automated scoring
+trace.score({
+  name: "relevance",
+  value: 0.95,
+  comment: "Highly relevant response"
+})
+
+// Human feedback scoring
+trace.score({
+  name: "user-satisfaction", 
+  value: 1,
+  source: "user-feedback",
+  comment: "User rated 5/5 stars"
+})
+\`\`\`
+
+---
+
+## 🎨 Best Practices
+
+### 📊 Monitoring & Observability
+
+#### Key Metrics to Track
+- **Latency**: P50, P95, P99 response times
+- **Token Usage**: Cost optimization
+- **Error Rates**: System reliability
+- **User Satisfaction**: Quality metrics
+
+#### Dashboard Setup
+\`\`\`yaml
+# monitoring-config.yml
+dashboards:
+  - name: "LLM Performance"
+    panels:
+      - type: "time-series"
+        title: "Response Latency"
+        query: "avg(response_time) by (model)"
+      - type: "stat"
+        title: "Daily Token Usage" 
+        query: "sum(tokens_used)"
+      - type: "table"
+        title: "Top Errors"
+        query: "topk(10, count by (error_type))"
+\`\`\`
+
+### 🔐 Security Considerations
+
+> ⚠️ **Important**: Never log sensitive user data in traces
+
+#### Data Sanitization
+\`\`\`python
+def sanitize_input(data):
+    """Remove PII from trace data"""
+    sanitized = data.copy()
+    
+    # Remove email addresses
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
+                      '[EMAIL_REDACTED]', sanitized)
+    
+    # Remove phone numbers
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
+                      '[PHONE_REDACTED]', sanitized)
+    
+    return sanitized
+\`\`\`
+
+### 🚀 Performance Optimization
+
+#### Batch Processing
+\`\`\`typescript
+// Efficient batch uploads
+const batchSize = 100
+const traces = []
+
+for (let i = 0; i < data.length; i += batchSize) {
+  const batch = data.slice(i, i + batchSize)
+  const processedBatch = await Promise.all(
+    batch.map(item => processWithLangfuse(item))
+  )
+  traces.push(...processedBatch)
+}
+
+// Flush all traces at once
+await langfuse.flushAsync()
+\`\`\`
+
+---
+
+## 📚 Advanced Examples
+
+### Multi-Agent System Tracing
+\`\`\`python
+class MultiAgentTracer:
+    def __init__(self):
+        self.langfuse = Langfuse()
+    
+    async def orchestrate_agents(self, task):
+        # Main orchestration trace
+        main_trace = self.langfuse.trace(
+            name="multi-agent-orchestration",
+            input={"task": task}
+        )
+        
+        # Agent 1: Research
+        research_span = main_trace.span(name="research-agent")
+        research_result = await self.research_agent.process(task)
+        research_span.end(output=research_result)
+        
+        # Agent 2: Analysis  
+        analysis_span = main_trace.span(name="analysis-agent")
+        analysis_result = await self.analysis_agent.process(research_result)
+        analysis_span.end(output=analysis_result)
+        
+        # Agent 3: Synthesis
+        synthesis_span = main_trace.span(name="synthesis-agent")
+        final_result = await self.synthesis_agent.process(analysis_result)
+        synthesis_span.end(output=final_result)
+        
+        main_trace.end(output=final_result)
+        return final_result
+\`\`\`
+
+---
+
+## 🎉 Conclusion
+
+With proper implementation of Langfuse tracing, you can:
+
+- ✅ **Monitor** your LLM applications in real-time
+- ✅ **Debug** issues with detailed trace information  
+- ✅ **Optimize** performance and costs
+- ✅ **Scale** your applications with confidence
+
+### Next Steps
+1. Review the [official documentation](https://langfuse.com/docs)
+2. Join our [Discord community](https://discord.gg/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/scripts/seeder/utils/postgres-seed-constants.ts

@@ -0,0 +1,432 @@
+// Datasets
+const SEED_DATASET_ITEMS_COUNTRIES = [
+  { input: { country: "France" }, output: "Paris" },
+  { input: { country: "Germany" }, output: "Berlin" },
+  { input: { country: "Italy" }, output: "Rome" },
+  { input: { country: "Spain" }, output: "Madrid" },
+  { input: { country: "United Kingdom" }, output: "London" },
+  { input: { country: "Japan" }, output: "Tokyo" },
+  { input: { country: "China" }, output: "Beijing" },
+  { input: { country: "India" }, output: "New Delhi" },
+  { input: { country: "Brazil" }, output: "Brasília" },
+  { input: { country: "Canada" }, output: "Ottawa" },
+  { input: { country: "Australia" }, output: "Canberra" },
+  { input: { country: "South Africa" }, output: "Pretoria" },
+  { input: { country: "Mexico" }, output: "Mexico City" },
+  { input: { country: "Russia" }, output: "Moscow" },
+  { input: { country: "Egypt" }, output: "Cairo" },
+  { input: { country: "Turkey" }, output: "Ankara" },
+  { input: { country: "Indonesia" }, output: "Jakarta" },
+  { input: { country: "South Korea" }, output: "Seoul" },
+  { input: { country: "Saudi Arabia" }, output: "Riyadh" },
+  { input: { country: "Argentina" }, output: "Buenos Aires" },
+  { input: { country: "Nigeria" }, output: "Abuja" },
+  { input: { country: "Pakistan" }, output: "Islamabad" },
+  { input: { country: "Thailand" }, output: "Bangkok" },
+  { input: { country: "Vietnam" }, output: "Hanoi" },
+  { input: { country: "Malaysia" }, output: "Kuala Lumpur" },
+  { input: { country: "Philippines" }, output: "Manila" },
+  { input: { country: "Singapore" }, output: "Singapore" },
+  { input: { country: "New Zealand" }, output: "Wellington" },
+  { input: { country: "Sweden" }, output: "Stockholm" },
+  { input: { country: "Norway" }, output: "Oslo" },
+  { input: { country: "Denmark" }, output: "Copenhagen" },
+  { input: { country: "Finland" }, output: "Helsinki" },
+  { input: { country: "Netherlands" }, output: "Amsterdam" },
+  { input: { country: "Belgium" }, output: "Brussels" },
+  { input: { country: "Switzerland" }, output: "Bern" },
+  { input: { country: "Austria" }, output: "Vienna" },
+  { input: { country: "Portugal" }, output: "Lisbon" },
+  { input: { country: "Greece" }, output: "Athens" },
+  { input: { country: "Poland" }, output: "Warsaw" },
+  { input: { country: "Ukraine" }, output: "Kyiv" },
+  { input: { country: "Romania" }, output: "Bucharest" },
+  { input: { country: "Hungary" }, output: "Budapest" },
+  { input: { country: "Czech Republic" }, output: "Prague" },
+  { input: { country: "Slovakia" }, output: "Bratislava" },
+  { input: { country: "Croatia" }, output: "Zagreb" },
+  { input: { country: "Serbia" }, output: "Belgrade" },
+  { input: { country: "Bulgaria" }, output: "Sofia" },
+  { input: { country: "Ireland" }, output: "Dublin" },
+  { input: { country: "Iceland" }, output: "Reykjavik" },
+  { input: { country: "Estonia" }, output: "Tallinn" },
+  { input: { country: "Latvia" }, output: "Riga" },
+  { input: { country: "Lithuania" }, output: "Vilnius" },
+];
+
+const SEED_DATASET_ITEMS_IPA = [
+  { input: { word: "the" }, output: "/ðə/" },
+  { input: { word: "be" }, output: "/bi/" },
+  { input: { word: "to" }, output: "/tu/" },
+  { input: { word: "of" }, output: "/əv/" },
+  { input: { word: "and" }, output: "/ænd/" },
+  { input: { word: "a" }, output: "/ə/" },
+  { input: { word: "in" }, output: "/ɪn/" },
+  { input: { word: "that" }, output: "/ðæt/" },
+  { input: { word: "have" }, output: "/hæv/" },
+  { input: { word: "I" }, output: "/aɪ/" },
+  { input: { word: "it" }, output: "/ɪt/" },
+  { input: { word: "for" }, output: "/fɔr/" },
+  { input: { word: "not" }, output: "/nɑt/" },
+  { input: { word: "on" }, output: "/ɑn/" },
+  { input: { word: "with" }, output: "/wɪð/" },
+  { input: { word: "he" }, output: "/hi/" },
+  { input: { word: "as" }, output: "/æz/" },
+  { input: { word: "you" }, output: "/ju/" },
+  { input: { word: "do" }, output: "/du/" },
+  { input: { word: "at" }, output: "/æt/" },
+  { input: { word: "this" }, output: "/ðɪs/" },
+  { input: { word: "but" }, output: "/bʌt/" },
+  { input: { word: "his" }, output: "/hɪz/" },
+  { input: { word: "by" }, output: "/baɪ/" },
+  { input: { word: "from" }, output: "/frʌm/" },
+  { input: { word: "they" }, output: "/ðeɪ/" },
+  { input: { word: "we" }, output: "/wi/" },
+  { input: { word: "say" }, output: "/seɪ/" },
+  { input: { word: "her" }, output: "/hər/" },
+  { input: { word: "she" }, output: "/ʃi/" },
+  { input: { word: "or" }, output: "/ɔr/" },
+  { input: { word: "an" }, output: "/æn/" },
+  { input: { word: "will" }, output: "/wɪl/" },
+  { input: { word: "my" }, output: "/maɪ/" },
+  { input: { word: "one" }, output: "/wʌn/" },
+  { input: { word: "all" }, output: "/ɔl/" },
+  { input: { word: "would" }, output: "/wʊd/" },
+  { input: { word: "there" }, output: "/ðɛr/" },
+  { input: { word: "their" }, output: "/ðɛr/" },
+  { input: { word: "what" }, output: "/wʌt/" },
+  { input: { word: "so" }, output: "/soʊ/" },
+  { input: { word: "up" }, output: "/ʌp/" },
+  { input: { word: "out" }, output: "/aʊt/" },
+  { input: { word: "if" }, output: "/ɪf/" },
+  { input: { word: "about" }, output: "/əˈbaʊt/" },
+  { input: { word: "who" }, output: "/hu/" },
+  { input: { word: "get" }, output: "/gɛt/" },
+  { input: { word: "which" }, output: "/wɪtʃ/" },
+  { input: { word: "go" }, output: "/goʊ/" },
+  { input: { word: "me" }, output: "/mi/" },
+  { input: { word: "when" }, output: "/wɛn/" },
+  { input: { word: "make" }, output: "/meɪk/" },
+  { input: { word: "can" }, output: "/kæn/" },
+  { input: { word: "like" }, output: "/laɪk/" },
+  { input: { word: "time" }, output: "/taɪm/" },
+  { input: { word: "no" }, output: "/noʊ/" },
+  { input: { word: "just" }, output: "/dʒʌst/" },
+  { input: { word: "him" }, output: "/hɪm/" },
+  { input: { word: "know" }, output: "/noʊ/" },
+  { input: { word: "take" }, output: "/teɪk/" },
+  { input: { word: "person" }, output: "/ˈpərsən/" },
+  { input: { word: "into" }, output: "/ˈɪntu/" },
+  { input: { word: "year" }, output: "/jɪr/" },
+  { input: { word: "your" }, output: "/jʊər/" },
+  { input: { word: "good" }, output: "/gʊd/" },
+  { input: { word: "some" }, output: "/sʌm/" },
+  { input: { word: "could" }, output: "/kʊd/" },
+  { input: { word: "them" }, output: "/ðɛm/" },
+  { input: { word: "see" }, output: "/si/" },
+  { input: { word: "other" }, output: "/ˈʌðər/" },
+  { input: { word: "than" }, output: "/ðæn/" },
+  { input: { word: "then" }, output: "/ðɛn/" },
+  { input: { word: "now" }, output: "/naʊ/" },
+  { input: { word: "look" }, output: "/lʊk/" },
+  { input: { word: "only" }, output: "/ˈoʊnli/" },
+  { input: { word: "come" }, output: "/kʌm/" },
+  { input: { word: "its" }, output: "/ɪts/" },
+  { input: { word: "over" }, output: "/ˈoʊvər/" },
+  { input: { word: "think" }, output: "/θɪŋk/" },
+  { input: { word: "also" }, output: "/ˈɔlsoʊ/" },
+  { input: { word: "back" }, output: "/bæk/" },
+  { input: { word: "after" }, output: "/ˈæftər/" },
+  { input: { word: "use" }, output: "/juz/" },
+  { input: { word: "two" }, output: "/tu/" },
+  { input: { word: "how" }, output: "/haʊ/" },
+  { input: { word: "our" }, output: "/aʊər/" },
+  { input: { word: "work" }, output: "/wɜrk/" },
+  { input: { word: "first" }, output: "/fɜrst/" },
+  { input: { word: "well" }, output: "/wɛl/" },
+  { input: { word: "way" }, output: "/weɪ/" },
+  { input: { word: "even" }, output: "/ˈivɪn/" },
+  { input: { word: "new" }, output: "/nu/" },
+  { input: { word: "want" }, output: "/wɑnt/" },
+  { input: { word: "because" }, output: "/bɪˈkɔz/" },
+  { input: { word: "any" }, output: "/ˈɛni/" },
+  { input: { word: "these" }, output: "/ðiz/" },
+  { input: { word: "give" }, output: "/gɪv/" },
+  { input: { word: "day" }, output: "/deɪ/" },
+  { input: { word: "most" }, output: "/moʊst/" },
+  { input: { word: "us" }, output: "/ʌs/" },
+];
+
+export const SEED_DATASETS = [
+  {
+    name: "demo-countries-dataset",
+    description: "Dataset for countries",
+    metadata: {
+      key: "value",
+    },
+    items: SEED_DATASET_ITEMS_COUNTRIES,
+  },
+  {
+    name: "demo-english-transcription-dataset",
+    description:
+      "Dataset for english transcription, where words are represented in their international phonetic alphabet (IPA)",
+    metadata: {
+      key: "value",
+    },
+    items: SEED_DATASET_ITEMS_IPA,
+  },
+];
+
+// Prompts
+export const SEED_TEXT_PROMPTS = [
+  {
+    id: `prompt-parent`,
+    createdBy: "user-1",
+    prompt:
+      'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format. Refer to the respective links of the documentation.\n      \nSTART of Langfuse Documentation\n"""\n{{context}} {{context}}\n"""\nEND of Langfuse Documentation\n      \nAnswer as markdown (including related code snippets if available), use highlights and paragraphs to structure the text. Use emojis in your answers. Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers. Only use information that is available in the context, do not make up any code that is not in the context. If you are unsure and the answer is not explicitly written in the documentation, say "Sorry, I don\'t know how to help with that." If the user is having problems using Langfuse, tell her to reach out to the founders directly via the chat widget. Make it crisp.\n\n@@@langfusePrompt:name=child-prompt|label=production@@@',
+    name: "parent-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-child`,
+    createdBy: "user-1",
+    prompt: `Please follow these guidelines:
+- Refer to the respective links of the documentation
+- Be kind.
+- Include emojis where it makes sense.
+- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.
+- Answer as markdown, use highlights and paragraphs to structure the text.
+- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.`,
+    name: "child-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-123`,
+    createdBy: "user-1",
+    prompt: "Prompt 1 content",
+    name: "prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-456`,
+    createdBy: "user-1",
+    prompt: "Prompt 2 content",
+    name: "prompt-2",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-789`,
+    createdBy: "API",
+    prompt: "Prompt 3 content",
+    name: "prompt-3-by-api",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: "Prompt 4 content",
+    name: "prompt-4",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `countries-experiment-prompt`,
+    createdBy: "user-1",
+    prompt: "What is the capital of {{country}}?",
+    name: "countries-experiment-prompt",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-customer-prompt-1`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 1 content",
+    name: "folder/customer/prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-customer-prompt-2`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 2 content",
+    name: "folder/customer/prompt-2",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `folder-prompt-1`,
+    createdBy: "user-1",
+    prompt: "Folder prompt 1 content",
+    name: "folder/prompt-1",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
+
+export const SEED_CHAT_ML_PROMPTS = [
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: [
+      {
+        role: "system",
+        content:
+          'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format.\n\nPlease follow these guidelines:\n- Refer to the respective links of the documentation and select quality examples\n- Be kind.\n- Include emojis where it makes sense.\n- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.\n- Answer as markdown, use highlights and paragraphs to structure the text.\n- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.\n- Only use information that is available in the context, do not make up any code that is not in the context.\n- Always put an empji at the end of the message.',
+      },
+      {
+        role: "assistant",
+        content:
+          "All right, what is the documentation that I am meant to exclusively use to answer the question?",
+      },
+      {
+        role: "user",
+        content: "<documentation>\n```\n{{context}}\n```\n</documentation>",
+      },
+      {
+        role: "assistant",
+        content:
+          "Answering in next message based on your instructions only. What is the question?",
+      },
+      {
+        role: "user",
+        content: "{{question}}",
+      },
+    ],
+    name: "prompt-chat-ml",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+  {
+    id: `prompt-chat-placeholder`,
+    createdBy: "user-1",
+    prompt: [
+      {
+        role: "system",
+        content:
+          'You are a very enthusiastic Langfuse representative who loves to help people! Langfuse is an open-source observability tool for developers of applications that use Large Language Models (LLMs). Given the following sections from the Langfuse documentation, answer the question using only that information, outputted in markdown format.\n\nPlease follow these guidelines:\n- Refer to the respective links of the documentation and select quality examples\n- Be kind.\n- Include emojis where it makes sense.\n- If the users have problems using Langfuse, tell them to reach out to the founders directly via the chat widget or GitHub at the end of your answer.\n- Answer as markdown, use highlights and paragraphs to structure the text.\n- Do not mention that you are "enthusiastic", the user does not need to know, will feel it from the style of your answers.\n- Only use information that is available in the context, do not make up any code that is not in the context.\n- Always put an empji at the end of the message.',
+      },
+      {
+        type: "placeholder",
+        name: "message_history",
+      },
+      {
+        role: "assistant",
+        content:
+          "Answering in next message based on your instructions only. What is the question?",
+      },
+      {
+        role: "user",
+        content: "{{question}}",
+      },
+    ],
+    name: "prompt-chat-ml-with-placeholder",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
+
+export const SEED_PROMPT_VERSIONS = [
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 1 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+    },
+    version: 1,
+  },
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 2 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+      topP: 0.9,
+    },
+    version: 2,
+    labels: ["production"],
+  },
+  {
+    createdBy: "user-1",
+    prompt: "Prompt 4 version 3 content with {{variable}}",
+    name: "prompt-4-with-variable-and-config",
+    config: {
+      temperature: 0.7,
+      topP: 0.9,
+      frequencyPenalty: 0.5,
+    },
+    version: 3,
+    labels: ["production", "latest"],
+  },
+];
+
+// evaluators
+export const SEED_EVALUATOR_TEMPLATES = [
+  {
+    id: "toxicity-template",
+    name: "toxicity-template",
+    version: 1,
+    prompt:
+      "Please evaluate the toxicity of the following text {{input}} {{output}}",
+    model: "gpt-3.5-turbo",
+    vars: ["input", "output"],
+    provider: "openai",
+    outputSchema: {
+      score: "provide a score between 0 and 1",
+      reasoning: "one sentence reasoning for the score",
+    },
+    modelParams: {
+      temperature: 0.7,
+      outputTokenLimit: 100,
+      topP: 0.9,
+    },
+  },
+];
+
+export const SEED_EVALUATOR_CONFIGS = [
+  {
+    id: "toxicity-job",
+    evalTemplateId: "toxicity-template",
+    jobType: "EVAL",
+    status: "ACTIVE",
+    scoreName: "toxicity",
+    filter: [
+      {
+        type: "string",
+        value: "user",
+        column: "User ID",
+        operator: "contains",
+      },
+    ],
+    variableMapping: [
+      {
+        langfuseObject: "trace",
+        selectedColumnId: "input",
+        templateVariable: "input",
+      },
+      {
+        langfuseObject: "trace",
+        selectedColumnId: "metadata",
+        templateVariable: "output",
+      },
+    ],
+    targetObject: "trace",
+    sampling: 1,
+    delay: 5_000,
+  },
+];
+
+export const EVAL_TRACE_COUNT = 100;
+export const FAILED_EVAL_TRACE_INTERVAL = 10;

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -0,0 +1,32 @@
+export const generateDatasetRunTraceId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `trace-dataset-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateEvalTraceId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `trace-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};
+
+export const generateEvalObservationId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `observation-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};
+
+export const generateEvalScoreId = (
+  evalTemplateId: string,
+  index: number,
+  projectId: string,
+) => {
+  return `score-eval-${evalTemplateId}-${projectId.slice(-8)}-${index}`;
+};

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -0,0 +1,328 @@
+import { FileContent, SeederOptions } from "./types";
+import { DataGenerator } from "./data-generators";
+import { ClickHouseQueryBuilder } from "./clickhouse-builder";
+import { EVAL_TRACE_COUNT, SEED_DATASETS } from "./postgres-seed-constants";
+import {
+  clickhouseClient,
+  logger,
+  ObservationRecordInsertType,
+  TraceRecordInsertType,
+} from "../../../src/server";
+import path from "path";
+import { readFileSync } from "fs";
+
+/**
+ * Orchestrates seeding operations across ClickHouse and PostgreSQL.
+ *
+ * Use createXxxData() for specific data types:
+ * - createDatasetExperimentData(): Dataset runs in langfuse-prompt-experiments env
+ * - createEvaluationData(): Evaluation data in langfuse-evaluation env
+ * - createSyntheticData(): Large synthetic data in default env
+ * - executeFullSeed(): All data types together
+ */
+export class SeederOrchestrator {
+  private dataGenerator: DataGenerator;
+  private queryBuilder: ClickHouseQueryBuilder;
+  private fileContent: FileContent | null = null;
+
+  constructor() {
+    this.dataGenerator = DataGenerator.getInstance();
+    this.queryBuilder = new ClickHouseQueryBuilder();
+    this.loadFileContent();
+  }
+
+  private loadFileContent() {
+    try {
+      const nestedJsonPath = path.join(__dirname, "./nested_json.json");
+      const heavyMarkdownPath = path.join(__dirname, "./markdown.txt");
+      const chatMlJsonPath = path.join(__dirname, "./chat_ml_json.json");
+
+      const nestedJsonContent = JSON.parse(
+        readFileSync(nestedJsonPath, "utf-8"),
+      );
+      const heavyMarkdownContent = readFileSync(heavyMarkdownPath, "utf-8");
+      const chatMlJsonContent = JSON.parse(
+        readFileSync(chatMlJsonPath, "utf-8"),
+      );
+
+      // Truncate large content for reasonable test data size
+      const truncatedNestedJson = {
+        ...nestedJsonContent,
+        products: nestedJsonContent.products?.slice(0, 3) || [],
+      };
+
+      const truncatedChatMlJson = {
+        ...chatMlJsonContent,
+        messages: chatMlJsonContent.messages?.slice(0, 4) || [],
+      };
+
+      this.fileContent = {
+        nestedJson: truncatedNestedJson,
+        heavyMarkdown: heavyMarkdownContent,
+        chatMlJson: truncatedChatMlJson,
+      };
+
+      this.dataGenerator.setFileContent(this.fileContent);
+    } catch (error) {
+      logger.warn(
+        "Could not load file content for seeding, using fallback data",
+        error,
+      );
+    }
+  }
+
+  /**
+   * Creates dataset experiment data for A/B testing and prompt comparisons.
+   * Use for: Experiment tracking, dataset-based evaluations, prompt testing.
+   */
+  async createDatasetExperimentData(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info(
+      `Creating dataset experiment data for ${projectIds.length} projects.`,
+    );
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing project ${projectId}`);
+
+      const numberOfRuns = opts.numberOfRuns || 1;
+
+      for (let runNumber = 0; runNumber < numberOfRuns; runNumber++) {
+        logger.info(
+          `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
+        );
+
+        const traces: TraceRecordInsertType[] = [];
+        const observations: ObservationRecordInsertType[] = [];
+
+        for (const seedDataset of SEED_DATASETS) {
+          for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // Generate trace data
+            const trace = this.dataGenerator.generateDatasetTrace(
+              {
+                datasetName: seedDataset.name,
+                itemIndex,
+                item: datasetItem,
+                runNumber,
+              },
+              projectId,
+            );
+
+            // Generate observation data
+            const observation = this.dataGenerator.generateDatasetObservation(
+              trace,
+              {
+                datasetName: seedDataset.name,
+                itemIndex,
+                item: datasetItem,
+                runNumber,
+              },
+              projectId,
+            );
+
+            traces.push(trace);
+            observations.push(observation);
+          }
+        }
+
+        try {
+          await this.queryBuilder.executeTracesInsert(traces);
+          await this.queryBuilder.executeObservationsInsert(observations);
+        } catch (error) {
+          logger.error(`✗ Insert failed:`, error);
+          throw error;
+        }
+      }
+    }
+  }
+
+  /**
+   * Creates evaluation data for testing evaluator configurations.
+   * Use for: Evaluator development, score validation, evaluation testing.
+   */
+  async createEvaluationData(projectIds: string[]): Promise<void> {
+    logger.info(`Creating evaluation data for ${projectIds.length} projects.`);
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing evaluation data for project ${projectId}`);
+
+      const evalTracesPerProject = EVAL_TRACE_COUNT;
+      const evalObservationsPerTrace = 10;
+
+      // Generate evaluation traces
+      const traces = this.dataGenerator.generateEvaluationTraces(
+        projectId,
+        evalTracesPerProject,
+      );
+
+      // Generate evaluation observations
+      const observations = this.dataGenerator.generateEvaluationObservations(
+        traces,
+        evalObservationsPerTrace,
+        projectId,
+      );
+
+      // Generate scores - exactly one score per evaluation trace
+      const scores = this.dataGenerator.generateEvaluationScores(
+        traces,
+        observations,
+        projectId,
+      );
+
+      await this.queryBuilder.executeTracesInsert(traces);
+      await this.queryBuilder.executeObservationsInsert(observations);
+      await this.queryBuilder.executeScoresInsert(scores);
+    }
+  }
+
+  /**
+   * Creates large-scale synthetic data for performance testing and demos.
+   * Use for: Load testing, dashboard demos, realistic usage simulation.
+   */
+  async createSyntheticData(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info(`Creating synthetic data for ${projectIds.length} projects.`);
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing synthetic data for project ${projectId}`);
+
+      const observationsPerTrace = 15;
+      const tracesPerProject = Math.floor(
+        (opts.totalObservations || 1000) / observationsPerTrace,
+      );
+      const scoresPerTrace = 10;
+
+      // For large datasets, use bulk generation for better performance
+      if (tracesPerProject > 100) {
+        logger.info(`Using bulk generation for ${tracesPerProject} traces`);
+
+        const traceQuery = this.queryBuilder.buildBulkTracesInsert(
+          projectId,
+          tracesPerProject,
+          "default",
+          this.fileContent || undefined,
+          { numberOfDays: opts.numberOfDays },
+        );
+        const observationQuery = this.queryBuilder.buildBulkObservationsInsert(
+          projectId,
+          tracesPerProject,
+          observationsPerTrace,
+          "default",
+          this.fileContent || undefined,
+          { numberOfDays: opts.numberOfDays },
+        );
+        const scoreQuery = this.queryBuilder.buildBulkScoresInsert(
+          projectId,
+          tracesPerProject,
+          scoresPerTrace,
+          "default",
+          { numberOfDays: opts.numberOfDays },
+        );
+
+        await this.executeQuery(traceQuery);
+        await this.executeQuery(observationQuery);
+        await this.executeQuery(scoreQuery);
+      } else {
+        // Use detailed generation for smaller datasets
+        const traces = this.dataGenerator.generateSyntheticTraces(
+          projectId,
+          tracesPerProject,
+        );
+        const observations = this.dataGenerator.generateSyntheticObservations(
+          traces,
+          observationsPerTrace,
+        );
+        const scores = this.dataGenerator.generateSyntheticScores(
+          traces,
+          observations,
+          scoresPerTrace,
+        );
+
+        await this.queryBuilder.executeTracesInsert(traces);
+        await this.queryBuilder.executeObservationsInsert(observations);
+        await this.queryBuilder.executeScoresInsert(scores);
+      }
+    }
+  }
+
+  /**
+   * Executes complete seeding: datasets + evaluation + synthetic data.
+   * Use for: Full system setup, comprehensive testing, complete data reset.
+   */
+  async executeFullSeed(
+    projectIds: string[],
+    opts: SeederOptions,
+  ): Promise<void> {
+    logger.info("Starting full seed process");
+
+    try {
+      // Create dataset experiment data
+      await this.createDatasetExperimentData(projectIds, opts);
+
+      // Create evaluation data
+      await this.createEvaluationData(projectIds);
+
+      // Create synthetic data
+      await this.createSyntheticData(projectIds, opts);
+
+      // Log completion statistics (commented out to reduce terminal noise)
+      await this.logStatistics();
+
+      logger.info("Full seed process completed successfully");
+    } catch (error) {
+      logger.error("Seed process failed:", error);
+      throw error;
+    }
+  }
+
+  private async executeQuery(query: string): Promise<void> {
+    try {
+      await clickhouseClient().command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        },
+      });
+    } catch (error) {
+      logger.error("Query execution failed:", error);
+      logger.error("Failed query:", query);
+      throw error;
+    }
+  }
+
+  private async logStatistics(): Promise<void> {
+    const tables = ["traces", "scores", "observations"];
+
+    for (const table of tables) {
+      try {
+        const query = `
+          SELECT
+            project_id,
+            count() AS per_project_count,
+            bar(per_project_count, 0, (
+              SELECT count(*)
+              FROM ${table}
+            ), 50) AS bar_representation
+          FROM ${table}
+          GROUP BY project_id
+          ORDER BY count() desc
+        `;
+
+        const result = await clickhouseClient().query({
+          query,
+          format: "TabSeparated",
+        });
+
+        logger.info(
+          `${table.charAt(0).toUpperCase() + table.slice(1)} per Project: \n` +
+            (await result.text()),
+        );
+      } catch (error) {
+        logger.warn(`Could not log statistics for ${table}:`, error);
+      }
+    }
+  }
+}

packages/shared/scripts/seeder/utils/types.ts

@@ -0,0 +1,18 @@
+export interface SeederOptions {
+  numberOfDays: number;
+  numberOfRuns?: number;
+  totalObservations?: number;
+}
+
+export interface DatasetItemInput {
+  datasetName: string;
+  itemIndex: number;
+  item: any;
+  runNumber?: number;
+}
+
+export interface FileContent {
+  nestedJson: any;
+  heavyMarkdown: string;
+  chatMlJson: any;
+}

packages/shared/src/db.ts

@@ -88,7 +88,7 @@ declare const globalThis: {
   kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
 } & typeof global;
 
-if (process.env.NODE_ENV === "development") {
+if (process.env.NODE_ENV === "development") { // eslint-disable-line turbo/no-undeclared-env-vars
   globalThis.prismaGlobal ??= createPrismaInstance(); // regular instantiation
   globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
     kyselyExtension({

packages/shared/src/domain/observations.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { jsonSchema } from "../utils/zod";
 import { MetadataDomain } from "./traces";
 

packages/shared/src/domain/scores.ts

@@ -1,5 +1,5 @@
 import { ScoreDataType } from "@prisma/client";
-import z from "zod";
+import z from "zod/v4";
 import { MetadataDomain } from "./traces";
 
 export const ScoreSource = {
@@ -31,7 +31,7 @@ export const ScoreSchema = z.object({
   queueId: z.string().nullable(),
   createdAt: z.date(),
   updatedAt: z.date(),
-  dataType: z.nativeEnum(ScoreDataType),
+  dataType: z.enum(ScoreDataType),
 });
 
 export type ScoreDomain = z.infer<typeof ScoreSchema>;

packages/shared/src/domain/table-view-presets.ts

@@ -1,13 +1,13 @@
 import { singleFilter } from "../interfaces/filters";
 import { orderBy } from "../interfaces/orderBy";
-import z from "zod";
+import z from "zod/v4";
 
 export enum TableViewPresetTableName {
-  Traces = "traces",
-  Observations = "observations",
-  Scores = "scores",
-  Sessions = "sessions",
-  Datasets = "datasets",
+  Traces = "traces", // eslint-disable-line no-unused-vars
+  Observations = "observations", // eslint-disable-line no-unused-vars
+  Scores = "scores", // eslint-disable-line no-unused-vars
+  Sessions = "sessions", // eslint-disable-line no-unused-vars
+  Datasets = "datasets", // eslint-disable-line no-unused-vars
 }
 
 const TableViewPresetDomainSchema = z.object({
@@ -17,7 +17,7 @@ const TableViewPresetDomainSchema = z.object({
   updatedAt: z.date(),
   createdBy: z.string().nullable(),
   name: z.string(),
-  tableName: z.nativeEnum(TableViewPresetTableName),
+  tableName: z.enum(TableViewPresetTableName),
   filters: z.array(singleFilter),
   columnOrder: z.array(z.string()),
   columnVisibility: z.record(z.string(), z.boolean()),

packages/shared/src/domain/traces.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { jsonSchema, jsonSchemaNullable } from "../utils/zod";
 
 export const MetadataDomain = z.record(

packages/shared/src/env.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
@@ -9,10 +9,7 @@ const EnvSchema = z.object({
   NEXTAUTH_URL: z.string().url().optional(),
   REDIS_HOST: z.string().nullish(),
   REDIS_PORT: z.coerce
-    .number({
-      description:
-        ".env files convert numbers to strings, therefore we have to enforce them to be numbers",
-    })
+    .number() // .env files convert numbers to strings, therefore we have to enforce them to be numbers
     .positive()
     .max(65536, `options.port should be >= 0 and < 65536`)
     .default(6379)
@@ -24,6 +21,9 @@ const EnvSchema = z.object({
   REDIS_TLS_CERT_PATH: z.string().optional(),
   REDIS_TLS_KEY_PATH: z.string().optional(),
   REDIS_ENABLE_AUTO_PIPELINING: z.enum(["true", "false"]).default("true"),
+  // Redis Cluster Configuration
+  REDIS_CLUSTER_ENABLED: z.enum(["true", "false"]).default("false"),
+  REDIS_CLUSTER_NODES: z.string().optional(),
   ENCRYPTION_KEY: z
     .string()
     .length(
@@ -45,18 +45,23 @@ const EnvSchema = z.object({
     .number()
     .nonnegative()
     .default(15_000),
+  LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
     .optional(),
   LANGFUSE_LOG_FORMAT: z.enum(["text", "json"]).default("text"),
+  LANGFUSE_LOG_PROPAGATED_HEADERS: z
+    .string()
+    .optional()
+    .transform((s) =>
+      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
+    ),
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
   LANGFUSE_S3_CONCURRENT_WRITES: z.coerce.number().positive().default(50),
-  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
-    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
-  }),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string(), // Langfuse requires a bucket name for S3 Event Uploads.
   LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
   LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
   LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
@@ -83,6 +88,7 @@ const EnvSchema = z.object({
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
+  LANGFUSE_S3_LIST_MAX_KEYS: z.coerce.number().positive().default(200),
   LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
@@ -99,9 +105,11 @@ const EnvSchema = z.object({
     .number()
     .default(80e6), // 80MB
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
+  LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
+  LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
 });
 
 export const env: z.infer<typeof EnvSchema> =
-  process.env.DOCKER_BUILD === "1"
+  process.env.DOCKER_BUILD === "1" // eslint-disable-line turbo/no-undeclared-env-vars
     ? (process.env as any)
     : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/errors/index.ts

@@ -7,3 +7,4 @@ export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
 export { InternalServerError } from "./InternalServerError";
 export { LangfuseConflictError } from "./ConflictError";
+export { QUEUE_ERROR_MESSAGES } from "./utils/constants";

packages/shared/src/errors/utils/constants.ts

@@ -0,0 +1,10 @@
+export const QUEUE_ERROR_MESSAGES = {
+  API_KEY_ERROR: "API key for provider",
+  NO_DEFAULT_MODEL_ERROR: "No default model or custom model found for project",
+  MAPPED_DATA_ERROR:
+    "Please ensure the mapped data exists and consider extending the job delay.",
+  INVALID_JSON_ERROR: "is not valid JSON",
+  TOO_LOW_MAX_TOKENS_ERROR: "Error: Unterminated string in JSON at position",
+  OUTPUT_TOKENS_TOO_LONG_ERROR:
+    "Could not parse response content as the length limit was reached",
+};

packages/shared/src/features/annotation/types.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { type ScoreDataType } from "../../db";
 
 const NUMERIC: ScoreDataType = "NUMERIC";

packages/shared/src/features/batchAction/types.ts

@@ -1,7 +1,8 @@
-import z from "zod";
+import z from "zod/v4";
 import { singleFilter } from "../../interfaces/filters";
 import { orderBy } from "../../interfaces/orderBy";
-import { BatchExportTableName } from "../batchExport/types";
+import { BatchTableNames } from "../../interfaces/tableNames";
+import { TracingSearchType } from "../../interfaces/search";
 
 /* eslint-disable no-unused-vars */
 export enum BatchActionType {
@@ -20,6 +21,8 @@ export type ActionId = z.infer<typeof ActionIdSchema>;
 export const BatchActionQuerySchema = z.object({
   filter: z.array(singleFilter).nullable(),
   orderBy,
+  searchQuery: z.string().optional(),
+  searchType: z.array(TracingSearchType).optional(),
 });
 
 export type BatchActionQuery = z.infer<typeof BatchActionQuerySchema>;
@@ -29,11 +32,11 @@ export const CreateBatchActionSchema = z.object({
   actionId: ActionIdSchema,
   targetId: z.string().optional(),
   query: BatchActionQuerySchema,
-  tableName: z.nativeEnum(BatchExportTableName),
+  tableName: z.enum(BatchTableNames),
 });
 
 export const GetIsBatchActionInProgressSchema = z.object({
   projectId: z.string(),
   actionId: ActionIdSchema,
-  tableName: z.nativeEnum(BatchExportTableName),
+  tableName: z.enum(BatchTableNames),
 });

packages/shared/src/features/batchExport/types.ts

@@ -1,30 +1,28 @@
-import z from "zod";
+import z from "zod/v4";
 
 import { BatchExport } from "@prisma/client";
 
 import { singleFilter } from "../../interfaces/filters";
 import { orderBy } from "../../interfaces/orderBy";
+import { BatchTableNames } from "../../interfaces/tableNames";
+import { TracingSearchType } from "../../interfaces/search";
 
 export enum BatchExportStatus {
-  QUEUED = "QUEUED",
-  PROCESSING = "PROCESSING",
-  COMPLETED = "COMPLETED",
-  FAILED = "FAILED",
+  QUEUED = "QUEUED", // eslint-disable-line no-unused-vars
+  PROCESSING = "PROCESSING", // eslint-disable-line no-unused-vars
+  COMPLETED = "COMPLETED", // eslint-disable-line no-unused-vars
+  FAILED = "FAILED", // eslint-disable-line no-unused-vars
 }
 
 export enum BatchExportFileFormat {
-  JSON = "JSON",
-  CSV = "CSV",
-  JSONL = "JSONL",
+  JSON = "JSON", // eslint-disable-line no-unused-vars
+  CSV = "CSV", // eslint-disable-line no-unused-vars
+  JSONL = "JSONL", // eslint-disable-line no-unused-vars
 }
 
-export enum BatchExportTableName {
-  Scores = "scores",
-  Sessions = "sessions",
-  Traces = "traces",
-  Observations = "observations",
-  DatasetRunItems = "dataset_run_items",
-}
+// Use shared BatchTableNames enum for consistency across batch operations
+// Keep BatchExportTableName as alias for backward compatibility
+export { BatchTableNames as BatchExportTableName };
 
 export const exportOptions: Record<
   BatchExportFileFormat,
@@ -44,8 +42,10 @@ export const exportOptions: Record<
 } as const;
 
 export const BatchExportQuerySchema = z.object({
-  tableName: z.nativeEnum(BatchExportTableName),
+  tableName: z.enum(BatchTableNames),
   filter: z.array(singleFilter).nullable(),
+  searchQuery: z.string().optional(),
+  searchType: z.array(TracingSearchType).optional(),
   orderBy,
   limit: z.number().optional(),
   page: z.number().optional(),
@@ -57,7 +57,7 @@ export const CreateBatchExportSchema = z.object({
   projectId: z.string(),
   name: z.string(),
   query: BatchExportQuerySchema,
-  format: z.nativeEnum(BatchExportFileFormat),
+  format: z.enum(BatchExportFileFormat),
 });
 
 export const BatchExportSchema = z.object({
@@ -69,9 +69,9 @@ export const BatchExportSchema = z.object({
   finishedAt: z.date().nullable(),
   expiresAt: z.date().nullable(),
   name: z.string(),
-  status: z.nativeEnum(BatchExportStatus),
+  status: z.enum(BatchExportStatus),
   query: BatchExportQuerySchema,
-  format: z.nativeEnum(BatchExportFileFormat),
+  format: z.enum(BatchExportFileFormat),
   url: z.string().nullable(),
   log: z.string().nullable(),
 });

packages/shared/src/features/comments/types.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 
 const MAX_COMMENT_LENGTH = 3000;
 

packages/shared/src/features/evals/types.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 
 export const langfuseObjects = [
   "trace",
@@ -8,6 +8,9 @@ export const langfuseObjects = [
   "dataset_item",
 ] as const;
 
+const langfuseObject = z.enum(langfuseObjects);
+export type LangfuseEvaluationObject = z.infer<typeof langfuseObject>;
+
 // variable mapping stored in the db for eval templates
 export const variableMapping = z
   .object({
@@ -15,7 +18,7 @@ export const variableMapping = z
     // name of the observation to extract the variable from
     // not required for trace, as we only have one.
     objectName: z.string().nullish(),
-    langfuseObject: z.enum(langfuseObjects),
+    langfuseObject: langfuseObject,
     selectedColumnId: z.string(),
     jsonSelector: z.string().nullish(),
   })
@@ -31,7 +34,7 @@ export const variableMappingList = z.array(variableMapping);
 export const wipVariableMapping = z.object({
   templateVariable: z.string(),
   objectName: z.string().nullish(),
-  langfuseObject: z.enum(langfuseObjects),
+  langfuseObject: langfuseObject,
   selectedColumnId: z.string().nullish(),
   jsonSelector: z.string().nullish(),
 });

packages/shared/src/features/evals/utilities.ts

@@ -0,0 +1,71 @@
+import z from "zod/v4";
+import { JSONPath } from "jsonpath-plus";
+import { variableMapping } from "./types";
+
+/**
+ * Parses an unknown value to a string representation
+ * This is used for any evaluation variable that needs string conversion
+ */
+export const parseUnknownToString = (value: unknown): string => {
+  if (value === null || value === undefined) {
+    return "";
+  }
+  if (
+    typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean"
+  ) {
+    return value.toString();
+  }
+  if (typeof value === "object") {
+    return JSON.stringify(value);
+  }
+  if (typeof value === "symbol") {
+    return value.toString();
+  }
+
+  return String(value);
+};
+
+function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
+  const result = JSONPath({
+    path: jsonSelector,
+    json:
+      typeof selectedColumn === "string"
+        ? JSON.parse(selectedColumn)
+        : selectedColumn,
+  });
+
+  return result.length > 0 ? result[0] : undefined;
+}
+
+export function extractValueFromObject(
+  obj: Record<string, unknown>,
+  mapping: z.infer<typeof variableMapping>,
+  parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown, // eslint-disable-line no-unused-vars
+): { value: string; error: Error | null } {
+  const selectedColumn = obj[mapping.selectedColumnId];
+  const jsonParser = parseJson || parseJsonDefault;
+
+  let jsonSelectedColumn;
+  let error: Error | null = null;
+
+  if (mapping.jsonSelector && selectedColumn) {
+    try {
+      jsonSelectedColumn = jsonParser(selectedColumn, mapping.jsonSelector);
+    } catch (err) {
+      error =
+        err instanceof Error
+          ? err
+          : new Error("There was an unknown error parsing the JSON");
+      jsonSelectedColumn = selectedColumn; // Fallback to original value
+    }
+  } else {
+    jsonSelectedColumn = selectedColumn;
+  }
+
+  return {
+    value: parseUnknownToString(jsonSelectedColumn),
+    error,
+  };
+}

packages/shared/src/features/prompts/parsePromptDependencyTags.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 
 export const PromptDependencyRegex = /@@@langfusePrompt:(.*?)@@@/g;
 

packages/shared/src/features/prompts/types.ts

@@ -1,15 +1,17 @@
-import { z } from "zod";
-import { jsonSchema } from "@langfuse/shared";
-import type { Prompt } from "@langfuse/shared";
-import { COMMIT_MESSAGE_MAX_LENGTH } from "@/src/features/prompts/constants";
+import { z } from "zod/v4";
+import type { Prompt } from "../../../prisma/generated/types";
+import { jsonSchema } from "../../utils/zod";
+import { COMMIT_MESSAGE_MAX_LENGTH } from "./constants";
+import { PromptChatMessageSchema } from "../../server/llm/types";
+import { PromptNameSchema } from "./validation";
 
-export const ChatMessageSchema = z.object({
-  role: z.string(),
-  content: z.string(),
-});
+export const SingleChatMessageSchema = PromptChatMessageSchema;
+export type SingleChatMessage = z.infer<typeof SingleChatMessageSchema>;
 
 export enum PromptType {
+  // eslint-disable-next-line no-unused-vars
   Chat = "chat",
+  // eslint-disable-next-line no-unused-vars
   Text = "text",
 }
 
@@ -22,13 +24,6 @@ export const PromptLabelSchema = z
     "Label must be lowercase alphanumeric with optional underscores, hyphens, or periods",
   );
 
-export const PromptNameSchema = z
-  .string()
-  .min(1)
-  .regex(/^[^|]*$/, "Name cannot contain '|' character")
-  .transform((s) => s.trim())
-  .refine((s) => s.length > 0, "Name cannot be empty");
-
 const BaseCreateTextPromptSchema = z.object({
   name: PromptNameSchema,
   labels: z.array(PromptLabelSchema).default([]),
@@ -48,7 +43,7 @@ const BaseCreateChatPromptSchema = z.object({
   name: PromptNameSchema,
   labels: z.array(PromptLabelSchema).default([]),
   type: z.literal(PromptType.Chat),
-  prompt: z.array(ChatMessageSchema),
+  prompt: z.array(PromptChatMessageSchema),
   config: jsonSchema.nullable().default({}),
   tags: z.array(z.string()).nullish(),
 });
@@ -141,7 +136,7 @@ export const BaseChatPromptSchema = z.object({
   tags: z.array(z.string()),
   labels: z.array(PromptLabelSchema),
   type: z.literal(PromptType.Chat),
-  prompt: z.array(ChatMessageSchema),
+  prompt: z.array(PromptChatMessageSchema),
   config: jsonSchema,
 });
 

packages/shared/src/features/prompts/validation.ts

@@ -0,0 +1,15 @@
+import { z } from "zod/v4";
+
+/**
+ * Prompt name validation schema for API, tRPC and client
+ */
+export const PromptNameSchema = z
+  .string()
+  .min(1, "Enter a name")
+  .regex(/^[^|]*$/, "Prompt name cannot contain '|' character")
+  .regex(/^[^/]/, "Name cannot start with a slash")
+  .regex(/^(?!.*\/\/)/, "Name cannot contain consecutive slashes")
+  .regex(/^.*[^/]$/, "Name cannot end with a slash")
+  .transform((s) => s.trim())
+  .refine((s) => s.length > 0, "Name cannot be empty")
+  .refine((name) => name !== "new", "Prompt name cannot be 'new'");

packages/shared/src/features/scores/interfaces/api/shared.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { jsonSchema, publicApiPaginationZod } from "../../../../utils/zod";
 import { stringDateTime } from "../../../../utils/typeChecks";
 import { applyScoreValidation } from "../../../../utils/scores";

packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { paginationMetaResponseZod } from "../../../../../utils/zod";
 import {
   DeleteScoreQuery,

packages/shared/src/features/scores/interfaces/api/v1/schemas.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import {
   CategoricalData,
   NumericData,

packages/shared/src/features/scores/interfaces/api/v1/validation.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { GetScoreResponseDataV1 } from "./endpoints";
 
 /**

packages/shared/src/features/scores/interfaces/api/v2/endpoints.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { paginationMetaResponseZod } from "../../../../../utils/zod";
 import { GetScoreQuery, GetScoresQuery } from "../shared";
 import { APIScoreSchemaV2 } from "./schemas";

packages/shared/src/features/scores/interfaces/api/v2/schemas.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import {
   CategoricalData,
   NumericData,

packages/shared/src/features/scores/interfaces/api/v2/validation.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { GetScoreResponseDataV2 } from "./endpoints";
 
 /**

packages/shared/src/features/scores/interfaces/application/validation.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { APIScoreSchemaV2, APIScoreV2 } from "../api/v2/schemas";
 import { APIScoreSchemaV1, APIScoreV1 } from "../api/v1/schemas";
 import { ScoreDomain } from "../../../../domain";

packages/shared/src/features/scores/interfaces/ingestion/validation.ts

@@ -1,4 +1,4 @@
-import z from "zod";
+import z from "zod/v4";
 import { applyScoreValidation } from "../../../../utils/scores";
 import { PostScoreBodyFoundationSchema } from "../shared";
 import { isPresent } from "../../../../utils/typeChecks";
@@ -37,13 +37,13 @@ const ScorePropsAgainstConfigNumeric = z
     dataType: z.literal("NUMERIC"),
   })
   .superRefine((data, ctx) => {
-    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+    if (isPresent(data.maxValue) && data.value > data.maxValue) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,
         message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
       });
     }
-    if (isPresent(data.minValue) && data.value <= data.minValue) {
+    if (isPresent(data.minValue) && data.value < data.minValue) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,
         message: `Value is below minimum value of ${data.minValue} defined in config`,

packages/shared/src/features/scores/interfaces/shared.ts

@@ -1,5 +1,5 @@
 import { jsonSchema } from "../../../utils/zod";
-import z from "zod";
+import z from "zod/v4";
 import { NonEmptyString } from "../../../utils/zod";
 
 /**

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 
 import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
 
@@ -68,10 +68,10 @@ const CategoricalScoreConfig = z.object({
     const parseResult = Categories.safeParse(categories);
     if (!parseResult.success) {
       ctx.addIssue({
-        code: z.ZodIssueCode.custom,
+        code: "custom",
         message:
           "Category must be an array of objects with label value pairs, where labels and values are unique.",
-      });
+      } as z.core.$ZodIssueCustom);
       return;
     }
 
@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void,
+  onParseError?: (error: z.ZodError) => void, // eslint-disable-line no-unused-vars
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
@@ -198,7 +198,7 @@ export const GetScoreConfigResponse = ValidatedScoreConfigSchema;
 
 // POST /score-configs
 export const PostScoreConfigBody = z
-  .union([
+  .discriminatedUnion("dataType", [
     ScoreConfigPostBase.merge(CategoricalScoreConfig),
     ScoreConfigPostBase.merge(NumericScoreConfig),
     ScoreConfigPostBase.merge(

packages/shared/src/index.ts

@@ -13,6 +13,7 @@ export * from "./utils/json";
 export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
+export * from "./utils/prompts";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
@@ -22,10 +23,11 @@ export * from "./server/llm/types";
 
 // evals
 export * from "./features/evals/types";
-
+export * from "./features/evals/utilities";
 // table actions
 export * from "./features/batchExport/types";
 export * from "./features/batchAction/types";
+export { BatchTableNames } from "./interfaces/tableNames";
 
 // annotation
 export * from "./features/annotation/types";
@@ -41,6 +43,10 @@ export * from "./features/experiments/utils";
 
 // prompts
 export * from "./features/prompts/parsePromptDependencyTags";
+export * from "./features/prompts/validation";
+export * from "./features/prompts/types";
+export * from "./features/prompts/constants";
+export { compileChatMessages, compileChatMessagesWithIds, isPlaceholder, type MessagePlaceholderValues, type PromptMessage as ServerPromptMessage } from "./server/llm/compileChatMessages";
 
 // export db types only
 export * from "@prisma/client";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { CloudConfigRateLimit } from "./rate-limits";
 import { cloudConfigPlans } from "../features/entitlements/plans";