chore: release v3.173.0

ci: harden prettier check file arguments (#13513 )
* ci: harden prettier check file arguments Prevent changed filenames from being interpreted as prettier options in CI. Co-Authored-By: Codex Opus 4.6 (1M context) <noreply@anthropic.com> * ci: ignore deleted files in prettier check Exclude deleted paths so delete-only changes do not pass missing files to Prettier. Co-Authored-By: Codex Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Codex Opus 4.6 (1M context) <noreply@anthropic.com>
2026-05-08 10:47:01 +02:00 · 2026-05-08 07:47:01 +00:00 · 2026-05-08 07:42:05 +00:00 · 2026-05-07 19:25:12 +00:00 · 2026-05-07 16:30:04 +00:00 · 2026-05-07 16:09:11 +00:00
778 changed files with 40876 additions and 19876 deletions
@@ -34,6 +34,9 @@ evaluating, and debugging AI applications.
  [`skills/add-model-price/SKILL.md`](skills/add-model-price/SKILL.md)
 - Code review tasks:
  [`skills/code-review/SKILL.md`](skills/code-review/SKILL.md)
+- Debugging a Linear issue, GitHub issue, or incident report using Datadog
+  (APM, logs, metrics) to establish a root cause:
+  [`skills/debug-issue-with-datadog/SKILL.md`](skills/debug-issue-with-datadog/SKILL.md)
 - Changelog drafting for completed feature branches:
  [`skills/changelog-writing/SKILL.md`](skills/changelog-writing/SKILL.md)
 - ClickHouse schema/query review:
@@ -98,8 +101,8 @@ langfuse/
 - Build check: `pnpm run build:check`
 - Full build: `pnpm run build`
 - Full reset/bootstrap (destructive): `pnpm run dx`
- Codex environment bootstrap: `bash scripts/codex/setup.sh`
- Codex environment maintenance: `bash scripts/codex/maintenance.sh`
+- Environment/worktree bootstrap: `bash scripts/codex/setup.sh`
+- Environment/worktree maintenance: `bash scripts/codex/maintenance.sh`
 - Install Playwright Chromium for agent browser review: `pnpm run playwright:install`

 Minimum verification matrix:
@@ -124,7 +127,9 @@ Minimum verification matrix:
  - `*/dist/*`
  - `packages/shared/prisma/generated/*`
 - Public API contract changes must update Fern sources in `fern/apis/**` and
-  regenerated outputs; never hand-edit `generated/**`.
+  regenerated outputs. Never hand-edit `generated/**`.
+- Before adding constants, value lists, or display mappings, search for an
+  existing owner and reuse or extend that source of truth.
 - Keep tests independent and parallel-safe.
 - For bug fixes, write the failing test first, confirm it fails, then fix the
  bug.
@@ -2,7 +2,12 @@

 ## Purpose

-Establish consistency and best practices across Langfuse's backend packages (web, worker, packages/shared) using Next.js 14, tRPC, BullMQ, and TypeScript patterns.
+Establish consistency and best practices across Langfuse's backend packages
+(`web`, `worker`, `packages/shared`) using Next.js, tRPC, BullMQ, and TypeScript
+patterns. Check package manifests such as `web/package.json` for current
+framework versions before version-sensitive work.
+Keep this file as an entrypoint; open reference files only when the task needs
+their details.

 ## When to Use This Skill

@@ -64,7 +69,7 @@ Use this guide when working on:
 ### Layered Architecture

 ```
-# Web Package (Next.js 14)
+# Web Package (Next.js)

 ┌─ tRPC API ──────────────────┐   ┌── Public REST API ──────────┐
 │                             │   │                             │
@@ -105,150 +110,6 @@ for complete details.

 ---

-## Directory Structure
-
-### Web Package (`/web/`)
-
-```
-web/src/
-├── features/                # Feature-organized code
-│   ├── [feature-name]/
-│   │   ├── server/          # Backend logic
-│   │   │   ├── *Router.ts   # tRPC router
-│   │   │   └── service.ts   # Business logic
-│   │   ├── components/      # React components
-│   │   └── types/           # Feature types
-├── server/
-│   ├── api/
-│   │   ├── routers/         # tRPC routers
-│   │   ├── trpc.ts          # tRPC setup & middleware
-│   │   └── root.ts          # Main router
-│   ├── auth.ts              # NextAuth.js config
-│   └── db.ts                # Database client
-├── pages/
-│   ├── api/
-│   │   ├── public/          # Public REST APIs
-│   │   └── trpc/            # tRPC endpoint
-│   └── [routes].tsx         # Next.js pages
-├── __tests__/               # Jest tests
-│   └── async/               # Integration tests
-├── instrumentation.ts       # OpenTelemetry (FIRST IMPORT)
-└── env.mjs                  # Environment config
-```
-
-### Worker Package (`/worker/`)
-
-```
-worker/src/
-├── queues/                  # BullMQ processors
-│   ├── evalQueue.ts
-│   ├── ingestionQueue.ts
-│   └── workerManager.ts
-├── features/                # Business logic
-│   └── [feature]/
-│       └── service.ts
-├── instrumentation.ts       # OpenTelemetry (FIRST IMPORT)
-├── app.ts                   # Express setup + queue registration
-├── env.ts                   # Environment config
-└── index.ts                 # Server start
-```
-
-### Shared Package (`/packages/shared/`)
-
-```
-shared/src/
-├── server/                  # Server utilities
-│   ├── auth/                # Authentication helpers
-│   ├── clickhouse/          # ClickHouse client & schema
-│   ├── instrumentation/     # OpenTelemetry helpers
-│   ├── llm/                 # LLM integration utilities
-│   ├── redis/               # Redis queues & cache
-│   ├── repositories/        # Data repositories
-│   ├── services/            # Shared services
-│   ├── utils/               # Server utilities
-│   ├── logger.ts
-│   └── queues.ts
-├── encryption/              # Encryption utilities
-├── features/                # Feature-specific code
-├── tableDefinitions/        # Table schemas
-├── utils/                   # Shared utilities
-├── constants.ts
-├── db.ts                    # Prisma client
-├── env.ts                   # Environment config
-└── index.ts                 # Main exports
-```
-
-**Import Paths (package.json exports):**
-
-The shared package exposes specific import paths for different use cases:
-
-| Import Path                                | Maps To                           | Use For                                                         |
-| ------------------------------------------ | --------------------------------- | --------------------------------------------------------------- |
-| `@langfuse/shared`                         | `dist/src/index.js`               | General types, schemas, utilities, constants                    |
-| `@langfuse/shared/src/db`                  | `dist/src/db.js`                  | Prisma client and database types                                |
-| `@langfuse/shared/src/server`              | `dist/src/server/index.js`        | Server-side utilities (queues, auth, services, instrumentation) |
-| `@langfuse/shared/src/server/auth/apiKeys` | `dist/src/server/auth/apiKeys.js` | API key management utilities                                    |
-| `@langfuse/shared/encryption`              | `dist/src/encryption/index.js`    | Encryption and signature utilities                              |
-
-**Usage Examples:**
-
-```typescript
-// General imports - types, schemas, constants, interfaces
-import {
-  CloudConfigSchema,
-  StringNoHTML,
-  AnnotationQueueObjectType,
-  type APIScoreV2,
-  type ColumnDefinition,
-  Role,
-} from "@langfuse/shared";
-
-// Database - Prisma client and types
-import { prisma, Prisma, JobExecutionStatus } from "@langfuse/shared/src/db";
-import { type DB as Database } from "@langfuse/shared";
-
-// Server utilities - queues, services, auth, instrumentation
-import {
-  logger,
-  instrumentAsync,
-  traceException,
-  redis,
-  getTracesTable,
-  StorageService,
-  sendMembershipInvitationEmail,
-  invalidateApiKeysForProject,
-  recordIncrement,
-  recordHistogram,
-} from "@langfuse/shared/src/server";
-
-// API key management (specific path)
-import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";
-
-// Encryption utilities
-import { encrypt, decrypt, sign, verify } from "@langfuse/shared/encryption";
-```
-
-**What Goes Where:**
-
-The shared package provides types, utilities, and server code used by both web and worker packages. It has **5 export paths** that control frontend vs backend access:
-
-| Import Path                                | Usage                 | What's Included                                                                    |
-| ------------------------------------------ | --------------------- | ---------------------------------------------------------------------------------- |
-| `@langfuse/shared`                         | ✅ Frontend + Backend | Prisma types, Zod schemas, constants, table definitions, domain models, utilities  |
-| `@langfuse/shared/src/db`                  | 🔒 Backend only       | Prisma client instance                                                             |
-| `@langfuse/shared/src/server`              | 🔒 Backend only       | Services, repositories, queues, auth, ClickHouse, LLM integration, instrumentation |
-| `@langfuse/shared/src/server/auth/apiKeys` | 🔒 Backend only       | API key management (separated to avoid circular deps)                              |
-| `@langfuse/shared/encryption`              | 🔒 Backend only       | Database field encryption/decryption                                               |
-
-**Naming Conventions:**
-
- tRPC Routers: `camelCaseRouter.ts` - `datasetRouter.ts`
- Services: `service.ts` in feature directory
- Queue Processors: `camelCaseQueue.ts` - `evalQueue.ts`
- Public APIs: `kebab-case.ts` - `dataset-items.ts`
-
---
-
 ## Core Principles

 ### 1. tRPC Procedures Delegate to Services
@@ -355,51 +216,10 @@ const result = await instrumentAsync(

 ### 7. Comprehensive Testing Required

-Write tests for all new features and bug fixes. See [testing-guide.md](references/testing-guide.md) for detailed examples.
-
-**Test Types:**
-
-| Type        | Framework | Location                                | Purpose                      |
-| ----------- | --------- | --------------------------------------- | ---------------------------- |
-| Integration | Jest      | `web/src/__tests__/async/`              | Full API endpoint testing    |
-| tRPC        | Jest      | `web/src/__tests__/async/`              | tRPC procedures with auth    |
-| Service     | Jest      | `web/src/__tests__/async/repositories/` | Repository/service functions |
-| Worker      | Vitest    | `worker/src/__tests__/`                 | Queue processors & streams   |
-
-**Quick Examples:**
-
-```typescript
-// Integration Test (Public API)
-const res = await makeZodVerifiedAPICall(
-  PostDatasetsV1Response, "POST", "/api/public/datasets",
-  { name: "test-dataset" }, auth
-);
-expect(res.status).toBe(200);
-
-// tRPC Test
-const { caller } = await prepare(); // Creates session + caller
-const response = await caller.automations.getAutomations({ projectId });
-expect(response).toHaveLength(1);
-
-// Service Test
-const result = await getObservationsWithModelDataFromEventsTable({
-  projectId, filter: [...], limit: 1000, offset: 0
-});
-expect(result.length).toBeGreaterThan(0);
-
-// Worker Test (vitest)
-const stream = await getObservationStream({ projectId, filter: [] });
-const rows = [];
-for await (const chunk of stream) rows.push(chunk);
-expect(rows).toHaveLength(2);
-```
-
-**Key Principles:**
-
- Use unique IDs (`randomUUID()`) to avoid test interference
- Clean up test data or use unique project IDs
- Tests must be independent and runnable in any order
- Prefer scoped cleanup or unique project IDs over global reset helpers
+Add targeted tests for new backend behavior and bug fixes. Keep tests
+independent and parallel-safe. See
+[testing-guide.md](references/testing-guide.md) for tRPC, public API, service,
+repository, and worker examples.

 ### 8. Always Filter by projectId for Tenant Isolation

@@ -448,72 +268,26 @@ Trace:

 ---

-## Common Imports
+## Live Examples

-```typescript
-// tRPC (Web)
-import { z } from "zod/v4";
-import {
-  createTRPCRouter,
-  protectedProjectProcedure,
-} from "@/src/server/api/trpc";
-import { TRPCError } from "@trpc/server";
-
-// Database
-import { prisma } from "@langfuse/shared/src/db";
-import type { Prisma } from "@prisma/client";
-
-// ClickHouse
-import {
-  queryClickhouse,
-  queryClickhouseStream,
-  upsertClickhouse,
-} from "@langfuse/shared/src/server";
-
-// Observability - OpenTelemetry + DataDog (NOT Sentry for backend)
-import {
-  logger,          // Winston logger with OTEL/DataDog trace context
-  traceException,  // Record exceptions to OpenTelemetry spans
-  instrumentAsync, // Create instrumented spans for operations
-} from "@langfuse/shared/src/server";
-
-// Config
-import { env } from "@/src/env.mjs"; // web
-// or
-import { env } from "./env"; // worker
-
-// Public API (Web)
-import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
-import { createAuthedProjectAPIRoute } from "@/src/features/public-api/server/createAuthedProjectAPIRoute";
-
-// Queue Processing (Worker)
-import { Job } from "bullmq";
-import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";
-```
+Reference existing Langfuse features for implementation patterns:
+- tRPC router with project auth and Zod input:
+  `web/src/features/events/server/eventsRouter.ts`
+- Public API route with middleware and typed request/response schemas:
+  `web/src/pages/api/public/datasets.ts`
+- Worker queue processor with typed jobs, logging, and retry behavior:
+  `worker/src/queues/evalQueue.ts`
+- Tenant filters for Prisma and ClickHouse:
+  `references/database-patterns.md`

 ---

-## Quick Reference
+## Naming Conventions

-### HTTP Status Codes
-
-| Code | Use Case     |
-| ---- | ------------ |
-| 200  | Success      |
-| 201  | Created      |
-| 400  | Bad Request  |
-| 401  | Unauthorized |
-| 403  | Forbidden    |
-| 404  | Not Found    |
-| 500  | Server Error |
-
-### Example Features to Reference
-
-Reference existing Langfuse features for implementation patterns:
- **Datasets** (`web/src/features/datasets/`) - Complete feature with tRPC router, public API, and service
- **Prompts** (`web/src/features/prompts/`) - Feature with versioning and templates
- **Evaluations** (`web/src/features/evals/`) - Complex feature with worker integration
- **Public API** (`web/src/features/public-api/`) - Middleware and route patterns
+- tRPC routers: `camelCaseRouter.ts`, e.g. `datasetRouter.ts`
+- Services: `service.ts` in the feature server directory
+- Queue processors: `camelCaseQueue.ts`, e.g. `evalQueue.ts`
+- Public API routes: kebab-case filenames, e.g. `dataset-items.ts`

 ---

@@ -530,6 +304,9 @@ Reference existing Langfuse features for implementation patterns:

 ## Navigation Guide

+Keep detailed backend guidance in these focused reference files and open only
+the one that matches the task.
+
 | Need to...                | Read this                                                    |
 | ------------------------- | ------------------------------------------------------------ |
 | Understand architecture   | [architecture-overview.md](references/architecture-overview.md)         |
@@ -541,37 +318,3 @@ Reference existing Langfuse features for implementation patterns:
 | Write tests               | [testing-guide.md](references/testing-guide.md)                         |

 ---
-
-## Reference Files
-
-### [architecture-overview.md](references/architecture-overview.md)
-
-Three-layer architecture (tRPC/Public API → Services → Data Access), request lifecycle for tRPC/Public API/Worker, Next.js 14 directory structure, dual database system (PostgreSQL + ClickHouse), separation of concerns, repository pattern for complex queries
-
-### [routing-and-controllers.md](references/routing-and-controllers.md)
-
-Next.js file-based routing, tRPC router patterns, Public REST API routes, layered architecture (Entry Points → Services → Repositories → Database), service layer organization, anti-patterns to avoid
-
-### [services-and-repositories.md](references/services-and-repositories.md)
-
-Service layer overview, dependency injection patterns, singleton patterns, repository pattern for data access, service design principles, caching strategies, testing services
-
-### [middleware-guide.md](references/middleware-guide.md)
-
-tRPC middleware (withErrorHandling, withOtelInstrumentation, enforceUserIsAuthed), seven tRPC procedure types (publicProcedure, authenticatedProcedure, protectedProjectProcedure, etc.), Public API middleware (withMiddlewares, createAuthedProjectAPIRoute), authentication patterns (NextAuth for tRPC, Basic Auth for Public API)
-
-### [database-patterns.md](references/database-patterns.md)
-
-Dual database architecture (PostgreSQL via Prisma + ClickHouse via direct client), PostgreSQL CRUD operations, ClickHouse query patterns (queryClickhouse, queryClickhouseStream, upsertClickhouse), repository pattern for complex queries, tenant isolation with projectId filtering, when to use which database
-
-### [configuration.md](references/configuration.md)
-
-Environment variable validation with Zod, package-specific configs (web/env.mjs with t3-oss/env-nextjs, worker/env.ts, shared/env.ts), NEXT_PUBLIC_LANGFUSE_CLOUD_REGION usage, LANGFUSE_EE_LICENSE_KEY for enterprise features, best practices for env management
-
-### [testing-guide.md](references/testing-guide.md)
-
-Integration tests (Public API with makeZodVerifiedAPICall), tRPC tests (createInnerTRPCContext, appRouter.createCaller), service-level tests (repository/service functions), worker tests (vitest with streams), test isolation principles, running tests (Jest for web, vitest for worker)
-
-**Skill Status**: COMPLETE ✅
-**Line Count**: ~540 lines
-**Progressive Disclosure**: 7 reference files ✅
@@ -1,6 +1,6 @@
 ---
 name: backend-dev-guidelines
-description: Shared backend guide for Langfuse's Next.js 14, tRPC, BullMQ, and TypeScript monorepo. Use when creating or reviewing tRPC routers, public REST endpoints, BullMQ queue processors, backend services, middleware, Prisma or ClickHouse data access, OpenTelemetry instrumentation, Zod validation, env configuration, or backend tests across web, worker, or packages/shared.
+description: Shared backend guide for Langfuse's Next.js, tRPC, BullMQ, and TypeScript monorepo. Use when creating or reviewing tRPC routers, public REST endpoints, BullMQ queue processors, backend services, middleware, Prisma or ClickHouse data access, OpenTelemetry instrumentation, Zod validation, env configuration, or backend tests across web, worker, or packages/shared.
 ---

 # Backend Development Guidelines
@@ -1,6 +1,6 @@
 # Architecture Overview - Langfuse Backend

-Complete guide to the layered architecture pattern used in Langfuse's Next.js 14/tRPC/Express monorepo.
+Complete guide to the layered architecture pattern used in Langfuse's Next.js/tRPC/Express monorepo. Check package manifests such as `web/package.json` for current framework versions before version-sensitive work.

 ## Table of Contents

@@ -20,7 +20,7 @@ Langfuse uses a **three-layer architecture** with two primary entry points (tRPC
 ### The Three Layers

 ```
-# Web Package (Next.js 14)
+# Web Package (Next.js)

 ┌─ tRPC API ──────────────────┐   ┌── Public REST API ──────────┐
 │                             │   │                             │
@@ -412,190 +412,19 @@ Repository: "Here's the Prisma query that does that"
 - ❌ Know about HTTP
 - ❌ Make decisions (that's service layer)

-### Repository Template
+### Langfuse Repository Examples

-```typescript
-// repositories/UserRepository.ts
-import { PrismaService } from "@project-lifecycle-portal/database";
-import type { User, Prisma } from "@project-lifecycle-portal/database";
+Use these current Langfuse files as repository templates:

-export class UserRepository {
-  /**
-   * Find user by ID with optimized query
-   */
-  async findById(userId: string): Promise<User | null> {
-    try {
-      return await PrismaService.main.user.findUnique({
-        where: { userID: userId },
-        select: {
-          userID: true,
-          email: true,
-          name: true,
-          isActive: true,
-          roles: true,
-          createdAt: true,
-          updatedAt: true,
-        },
-      });
-    } catch (error) {
-      console.error("[UserRepository] Error finding user by ID:", error);
-      throw new Error(`Failed to find user: ${userId}`);
-    }
-  }
+- PostgreSQL repository with project-scoped filters:
+  `packages/shared/src/server/repositories/comments.ts`
+- ClickHouse repository with project-scoped filters and query helpers:
+  `packages/shared/src/server/repositories/traces.ts`
+- Repository tests:
+  `web/src/__tests__/server/repositories/event-repository.servertest.ts`

-  /**
-   * Find all active users
-   */
-  async findActive(options?: {
-    orderBy?: Prisma.UserOrderByWithRelationInput;
-  }): Promise<User[]> {
-    try {
-      return await PrismaService.main.user.findMany({
-        where: { isActive: true },
-        orderBy: options?.orderBy || { name: "asc" },
-        select: {
-          userID: true,
-          email: true,
-          name: true,
-          roles: true,
-        },
-      });
-    } catch (error) {
-      console.error("[UserRepository] Error finding active users:", error);
-      throw new Error("Failed to find active users");
-    }
-  }
-
-  /**
-   * Find user by email
-   */
-  async findByEmail(email: string): Promise<User | null> {
-    try {
-      return await PrismaService.main.user.findUnique({
-        where: { email },
-      });
-    } catch (error) {
-      console.error("[UserRepository] Error finding user by email:", error);
-      throw new Error(`Failed to find user with email: ${email}`);
-    }
-  }
-
-  /**
-   * Create new user
-   */
-  async create(data: Prisma.UserCreateInput): Promise<User> {
-    try {
-      return await PrismaService.main.user.create({ data });
-    } catch (error) {
-      console.error("[UserRepository] Error creating user:", error);
-      throw new Error("Failed to create user");
-    }
-  }
-
-  /**
-   * Update user
-   */
-  async update(userId: string, data: Prisma.UserUpdateInput): Promise<User> {
-    try {
-      return await PrismaService.main.user.update({
-        where: { userID: userId },
-        data,
-      });
-    } catch (error) {
-      console.error("[UserRepository] Error updating user:", error);
-      throw new Error(`Failed to update user: ${userId}`);
-    }
-  }
-
-  /**
-   * Delete user (soft delete by setting isActive = false)
-   */
-  async delete(userId: string): Promise<User> {
-    try {
-      return await PrismaService.main.user.update({
-        where: { userID: userId },
-        data: { isActive: false },
-      });
-    } catch (error) {
-      console.error("[UserRepository] Error deleting user:", error);
-      throw new Error(`Failed to delete user: ${userId}`);
-    }
-  }
-
-  /**
-   * Check if email exists
-   */
-  async emailExists(email: string): Promise<boolean> {
-    try {
-      const count = await PrismaService.main.user.count({
-        where: { email },
-      });
-      return count > 0;
-    } catch (error) {
-      console.error("[UserRepository] Error checking email exists:", error);
-      throw new Error("Failed to check if email exists");
-    }
-  }
-}
-
-// Export singleton instance
-export const userRepository = new UserRepository();
-```
-
-**Using Repository in Service:**
-
-```typescript
-// services/userService.ts
-import { userRepository } from "../repositories/UserRepository";
-import { ConflictError, NotFoundError } from "../utils/errors";
-
-export class UserService {
-  /**
-   * Create new user with business rules
-   */
-  async createUser(data: {
-    email: string;
-    name: string;
-    roles: string[];
-  }): Promise<User> {
-    // Business rule: Check if email already exists
-    const emailExists = await userRepository.emailExists(data.email);
-    if (emailExists) {
-      throw new ConflictError("Email already exists");
-    }
-
-    // Business rule: Validate roles
-    const validRoles = ["admin", "operations", "user"];
-    const invalidRoles = data.roles.filter(
-      (role) => !validRoles.includes(role),
-    );
-    if (invalidRoles.length > 0) {
-      throw new ValidationError(`Invalid roles: ${invalidRoles.join(", ")}`);
-    }
-
-    // Create user via repository
-    return await userRepository.create({
-      email: data.email,
-      name: data.name,
-      roles: data.roles,
-      isActive: true,
-    });
-  }
-
-  /**
-   * Get user by ID
-   */
-  async getUser(userId: string): Promise<User> {
-    const user = await userRepository.findById(userId);
-
-    if (!user) {
-      throw new NotFoundError(`User not found: ${userId}`);
-    }
-
-    return user;
-  }
-}
-```
+Keep data-access concerns in repositories and business decisions in services.
+Project-scoped queries must include `projectId` or `project_id` filters.

 ---

@@ -803,69 +632,13 @@ class UserService {

 ## Testing Services

-### Unit Tests
+Use `testing-guide.md` for backend test patterns. Prefer current Langfuse tests
+over invented examples:

-```typescript
-// tests/userService.test.ts
-import { UserService } from "../services/userService";
-import { userRepository } from "../repositories/UserRepository";
-import { ConflictError } from "../utils/errors";
-
-// Mock repository
-jest.mock("../repositories/UserRepository");
-
-describe("UserService", () => {
-  let userService: UserService;
-
-  beforeEach(() => {
-    userService = new UserService();
-    jest.clearAllMocks();
-  });
-
-  describe("createUser", () => {
-    it("should create user when email does not exist", async () => {
-      // Arrange
-      const userData = {
-        email: "test@example.com",
-        name: "Test User",
-        roles: ["user"],
-      };
-
-      (userRepository.emailExists as jest.Mock).mockResolvedValue(false);
-      (userRepository.create as jest.Mock).mockResolvedValue({
-        userID: "123",
-        ...userData,
-      });
-
-      // Act
-      const user = await userService.createUser(userData);
-
-      // Assert
-      expect(user).toBeDefined();
-      expect(user.email).toBe(userData.email);
-      expect(userRepository.emailExists).toHaveBeenCalledWith(userData.email);
-      expect(userRepository.create).toHaveBeenCalled();
-    });
-
-    it("should throw ConflictError when email exists", async () => {
-      // Arrange
-      const userData = {
-        email: "existing@example.com",
-        name: "Test User",
-        roles: ["user"],
-      };
-
-      (userRepository.emailExists as jest.Mock).mockResolvedValue(true);
-
-      // Act & Assert
-      await expect(userService.createUser(userData)).rejects.toThrow(
-        ConflictError,
-      );
-      expect(userRepository.create).not.toHaveBeenCalled();
-    });
-  });
-});
-```
+- Repository tests:
+  `web/src/__tests__/server/repositories/event-repository.servertest.ts`
+- Pure service unit tests:
+  `web/src/__tests__/server/unit/`

 ---

@@ -4,25 +4,65 @@ Complete guide to testing Langfuse backend services across web, worker, and shar

 ## Table of Contents

+- [Key Testing Principles](#key-testing-principles)
 - [Test Types Overview](#test-types-overview)
 - [Integration Tests (Public API)](#integration-tests-public-api)
 - [Service-Level Tests (Repository/Service)](#service-level-tests-repositoryservice)
 - [tRPC Tests (Procedure Testing)](#trpc-tests-procedure-testing)
 - [Worker Tests (Queue Processing)](#worker-tests-queue-processing)
- [Key Testing Principles](#key-testing-principles)
 - [Running Tests](#running-tests)

 ---

+## Key Testing Principles
+
+### General Principles
+
+1. **Test Isolation**: Each test should be independent and runnable in any order
+2. **Unique IDs**: Use `randomUUID()` or unique project IDs to avoid test interference
+3. **Cleanup**: Always clean up test data in service tests (or use unique project IDs)
+4. **Avoid Global Resets**: Prefer scoped cleanup or unique project IDs over global reset helpers
+5. **Flags and Fallbacks**: When code branches on env flags, feature flags, or fallback data paths, test both branches and ensure fixtures are written to the same store the branch reads from
+
+### By Test Type
+
+| Test Type | Key Principles |
+|-----------|----------------|
+| **Integration** | Test HTTP endpoints, validate status codes and response shapes |
+| **tRPC** | Use `createInnerTRPCContext` and `appRouter.createCaller`, test auth/permissions |
+| **Service** | Test individual functions with isolated data, always cleanup |
+| **Worker** | Use vitest, test streams with async iteration, test filtering logic |
+
+### Test Data Management
+
+```typescript
+// ✅ GOOD: Use unique IDs
+const projectId = randomUUID();
+const traceId = randomUUID();
+
+// ✅ GOOD: Cleanup in service tests
+afterAll(async () => {
+  await prisma.model.delete({ where: { id: modelId } });
+});
+
+// ✅ GOOD: Use unique projects (no cleanup needed)
+const { projectId } = await createOrgProjectAndApiKey();
+
+// ❌ BAD: Shared test data between tests
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+```
+
+---
+
 ## Test Types Overview

 Langfuse uses multiple testing strategies for different layers:

 | Test Type | Framework | Location | Purpose |
 |-----------|-----------|----------|---------|
-| Integration | Jest | `web/src/__tests__/async/` | Full API endpoint testing |
-| tRPC | Jest | `web/src/__tests__/async/` | tRPC procedure testing with auth |
-| Service | Jest | `web/src/__tests__/async/repositories/` | Repository/service function testing |
+| Integration | Vitest | `web/src/__tests__/server/` | Full API endpoint testing |
+| tRPC | Vitest | `web/src/__tests__/server/` | tRPC procedure testing with auth |
+| Service | Vitest | `web/src/__tests__/server/repositories/` | Repository/service function testing |
 | Worker | Vitest | `worker/src/__tests__/` | Queue processors and streams |

 ---
@@ -31,7 +71,7 @@ Langfuse uses multiple testing strategies for different layers:

 Test full REST API endpoints end-to-end using HTTP requests.

-**File location:** `web/src/__tests__/async/datasets-api.servertest.ts`
+**File location:** `web/src/__tests__/server/datasets-api.servertest.ts`

 ```typescript
 import { makeZodVerifiedAPICall } from "../helpers";
@@ -73,7 +113,7 @@ describe("Dataset API", () => {

 Test individual repository/service functions with isolated data.

-**File location:** `web/src/__tests__/async/repositories/event-repository.servertest.ts`
+**File location:** `web/src/__tests__/server/repositories/event-repository.servertest.ts`

 ```typescript
 import {
@@ -186,7 +226,7 @@ describe("Event Repository Tests", () => {

 Test tRPC procedures with caller pattern and auth context.

-**File location:** `web/src/__tests__/async/automations-trpc.servertest.ts`
+**File location:** `web/src/__tests__/server/automations-trpc.servertest.ts`

 ```typescript
 import { appRouter } from "@/src/server/api/root";
@@ -464,88 +504,15 @@ describe("batch export test suite", () => {

 ---

-## Key Testing Principles
-
-### General Principles
-
-1. **Test Isolation**: Each test should be independent and runnable in any order
-2. **Unique IDs**: Use `randomUUID()` or unique project IDs to avoid test interference
-3. **Cleanup**: Always clean up test data in service tests (or use unique project IDs)
-4. **Avoid Global Resets**: Prefer scoped cleanup or unique project IDs over global reset helpers
-
-### By Test Type
-
-| Test Type | Key Principles |
-|-----------|----------------|
-| **Integration** | Test HTTP endpoints, validate status codes and response shapes |
-| **tRPC** | Use `createInnerTRPCContext` and `appRouter.createCaller`, test auth/permissions |
-| **Service** | Test individual functions with isolated data, always cleanup |
-| **Worker** | Use vitest, test streams with async iteration, test filtering logic |
-
-### Test Data Management
-
-```typescript
-// ✅ GOOD: Use unique IDs
-const projectId = randomUUID();
-const traceId = randomUUID();
-
-// ✅ GOOD: Cleanup in service tests
-afterAll(async () => {
-  await prisma.model.delete({ where: { id: modelId } });
-});
-
-// ✅ GOOD: Use unique projects (no cleanup needed)
-const { projectId } = await createOrgProjectAndApiKey();
-
-// ❌ BAD: Shared test data between tests
-const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-```
-
---
-
 ## Running Tests

-### Web Tests (Jest)
+Use the nearest package `AGENTS.md` as the source of truth for current test
+commands.

-```bash
-# Run all tests
-pnpm test
-
-# Run sync tests
-pnpm test-sync
-
-# Run async tests
-pnpm test -- --testPathPattern="async"
-
-# Run specific test file
-pnpm test -- --testPathPattern="datasets-api"
-
-# Run specific test
-pnpm test -- --testPathPattern="datasets-api" --testNamePattern="should create dataset"
-```
-
-### Worker Tests (Vitest)
-
-```bash
-# Run all worker tests
-pnpm run test --filter=worker
-
-# Run specific test file
-pnpm run test --filter=worker -- batchExport
-
-# Run specific test
-pnpm run test --filter=worker -- batchExport -t "should export observations"
-```
-
-### Coverage
-
-```bash
-# Web coverage
-pnpm test -- --coverage
-
-# Worker coverage
-pnpm run test --filter=worker -- --coverage
-```
+Common targeted forms:
+- Web server tests: `pnpm --filter web run test -- <pattern>`
+- Web client tests: `pnpm --filter web run test-client -- <pattern>`
+- Worker tests: `pnpm --filter worker run test <file-or-pattern>`

 ---

@@ -25,9 +25,20 @@ Comprehensive guidance for ClickHouse covering schema design, query optimization

 **Why rules take priority:** ClickHouse has specific behaviors (columnar storage, sparse indexes, merge tree mechanics) where general database intuition can be misleading. The rules encode validated, ClickHouse-specific guidance.

-### For Formal Reviews
+## Langfuse-Specific Rules

-When performing a formal review of schemas, queries, or data ingestion:
+- Use `packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts`
+  for queries against the `events` table. Do not hand-roll `events` SQL unless
+  you first confirm the query builder cannot express the query.
+- Never use `FINAL` on the `events` table; it is designed so `FINAL` is not
+  required and the keyword hurts performance.
+- Any migration in `packages/shared/clickhouse/migrations/clustered/**` with
+  more than one `ALTER` on the same table must end every metadata `ALTER`
+  (`ADD/DROP/MODIFY COLUMN`, `ADD/DROP INDEX`) with `SETTINGS alter_sync = 2`,
+  and every mutation-creating `ALTER` (`MATERIALIZE …`, `UPDATE`, `DELETE`)
+  with `SETTINGS mutations_sync = 2`. The matching `unclustered/` file runs
+  against plain `MergeTree` and does not need (and should not duplicate)
+  these settings.

 ---

@@ -53,6 +64,7 @@ When performing a formal review of schemas, queries, or data ingestion:
 - [ ] LowCardinality applied to appropriate string columns
 - [ ] Partition key cardinality bounded (100-1,000 values)
 - [ ] ReplacingMergeTree has version column if used
+- [ ] Clustered migration files with multiple ALTERs on the same table use `SETTINGS alter_sync = 2` (metadata) and `SETTINGS mutations_sync = 2` (`MATERIALIZE …`, `UPDATE`, `DELETE`); unclustered mirror has none

 ### For Query Reviews (SELECT, JOIN, aggregations)

@@ -227,8 +239,8 @@ Each rule file in `rules/` contains:

 ---

-## Full Compiled Document
+## Compatibility Entrypoint

-For the complete guide with all rules expanded inline: `AGENTS.md`
-
-Use `AGENTS.md` when you need to check multiple rules quickly without reading individual files.
+`AGENTS.md` is a short compatibility index for agents that open that file
+directly. The authoritative workflow lives in this `SKILL.md`, and detailed rule
+bodies live in `rules/`.
@@ -0,0 +1,115 @@
+---
+name: debug-issue-with-datadog
+description: |
+  Debug a user-reported issue, Linear ticket, or incident report by combining
+  Datadog (APM, logs, metrics) with the Langfuse repo to establish a
+  root cause. Use when given a Linear issue URL/ID (e.g. LFE-XXXX), a GitHub
+  issue, or a pasted error/report and asked to investigate, root-cause, or
+  triage. Produces a structured analysis — error breakdown, hypothesis-by-class,
+  suggested patches with code references.
+---
+
+# Debug Issue with Datadog
+
+Use this skill whenever the task is **investigative** rather than
+implementational: a user, customer, or oncall has surfaced a problem and you
+need to figure out *what is actually happening in production* and *where in the
+code it lives*. The deliverable is an analysis, not a patch — though the
+analysis should make the right patch obvious.
+
+## When to Apply
+
+- A Linear issue (typically with an `LFE-XXXX` ID) describes a production
+  failure, error spike, or customer report.
+- A GitHub issue or pasted incident/error report needs triage.
+- A monitor alerted and you need to understand *why* before deciding what to
+  fix.
+- Existing tickets under the "Make monitoring useful again" project (parent
+  `LFE-8837`) and similar — these expect the structured analysis output below.
+
+If the task is "implement this fix" rather than "figure out what's broken",
+this is the wrong skill — go to `backend-dev-guidelines` or the relevant
+package guide.
+
+## Workflow
+
+Read the inputs first, then plan the Datadog sweep, then read the code, then
+write the analysis. Do not skip ahead to suggested patches before the data
+supports them.
+
+1. **Intake.** Pull every signal already available in the report. See
+   [`references/intake.md`](references/intake.md). For a Linear URL/ID, fetch
+   the issue *and* its comments via the Linear MCP — the description is often
+   updated inline as triage proceeds. For a GitHub issue, use `gh issue view`.
+   For pasted text, treat it as the description.
+
+2. **Scope the sweep.** From the intake, pick the affected subsystem and time
+   window. Use [`references/repo-debug-map.md`](references/repo-debug-map.md)
+   to translate "PostHog integration", "ingestion failures", "evals stuck",
+   etc. into the Datadog filters and source files you should be looking at.
+
+3. **Run the broad Datadog sweep.** Default to the full sweep in
+   [`references/datadog-playbook.md`](references/datadog-playbook.md): APM
+   spans, error logs, metrics, and monitors — split across `prod-eu`
+   and `prod-us` (and `prod-hipaa` / `prod-jp` when relevant). Always check
+   regional disparity first; it usually rules whole hypotheses in or out.
+
+4. **Cluster the errors.** Group by `(projectId, error.message)` or
+   `(error.type, error.message)`. Treat each distinct cluster as its own
+   hypothesis — Langfuse incidents commonly have *multiple* coexisting root
+   causes, not one.
+
+5. **Map clusters to code.** For each cluster, open the relevant handler file
+   from the repo-debug map and read enough of it to confirm or refute the
+   hypothesis. Cite specific files and line ranges in the output.
+
+6. **Write the analysis** using
+   [`references/output-template.md`](references/output-template.md).
+
+7. **Deliver.** Default: print the analysis in chat. If the user asked for it,
+   also save under the workflow they specified (file, Linear comment via, etc.).
+
+## Datadog MCP Usage Notes
+
+Two Datadog MCP servers are typically available — one bound to the EU site
+(`datadoghq.eu`) and one to the US site (`datadoghq.com`). Always run
+region-relevant queries against **both** unless intake clearly localizes the
+incident. The `prod-eu` / `prod-us` env tags live on each side respectively.
+
+- Span search filter pattern:
+  `service:worker resource_name:"process posthog-integration-project" status:error`
+- Log search filter pattern:
+  `service:worker env:prod-eu @langfuse.project.id:cm1r6u… status:error`
+- For high-volume queries, prefer `aggregate_spans` / `aggregate_events`
+  grouped by `(error.message, projectId)` over fetching individual traces.
+- Always link to the Datadog UI for the queries you ran (final section of the
+  output template).
+
+See [`references/datadog-playbook.md`](references/datadog-playbook.md) for the
+full set of starter queries and parameter shapes.
+
+## Output Expectations
+
+From the output template:
+
+- Header: data source, time window, region split (EU vs US table).
+- Hotspots: per-`projectId` (or per-cluster) error counts.
+- Root cause by error class: each cluster gets a short hypothesis with
+  reasoning, distinguishing primary causes from symptoms.
+- Suggested patches: P0/P1/P2 grouped, with concrete file paths and short code
+  sketches. Reference the actual handler in `worker/src/features/**` or
+  `web/src/**`.
+- Dashboards: paste the Datadog query URLs at the end.
+
+Findings come first, recommendations last. If the data is thin, say so
+explicitly and propose what would need to be true to confirm each hypothesis —
+do not invent root causes.
+
+## Cross-References
+
+- Backend layout, queue contracts, instrumentation patterns:
+  [`backend-dev-guidelines`](../backend-dev-guidelines/SKILL.md)
+- ClickHouse-related findings (memory ceilings, JOIN spills, slow queries):
+  [`clickhouse-best-practices`](../clickhouse-best-practices/SKILL.md)
+- Once a fix is identified and you switch to implementation, hand off to the
+  package `AGENTS.md` for the affected directory.
@@ -0,0 +1,133 @@
+# Datadog Query Playbook
+
+The default for this skill is the **broad sweep**: APM spans + logs + metrics
+ monitors + incidents, run against both EU and US sites unless intake clearly
+localizes. Cluster results before drilling in.
+
+Two MCP servers are typically connected — one for `datadoghq.eu`, one for
+`datadoghq.com`. Run the same query on both and compare. The contrast itself
+is often the most informative finding (e.g. LFE-9475's 23.5% EU vs 0.7% US
+error rate immediately ruled out PostHog Cloud as the global cause).
+
+## Tag Vocabulary
+
+- **Site:** `datadoghq.eu` for EU, `datadoghq.com` for US.
+- **Region tag (`env`):** `prod-eu`, `prod-us`, `prod-hipaa`, `prod-jp`.
+- **Service:** `worker` (default in `worker/src/env.ts`) or `web` (default in
+  `web/src/env.mjs`). Some deployments override to `langfuse`.
+- **Span resource names** for worker async jobs follow the pattern
+  `process <queue-name>` — see `repo-debug-map.md`.
+
+## 1. APM Span Sweep — find the failing handler
+
+Use `aggregate_spans` first; only fetch individual traces once a cluster is
+identified.
+
+Starter shape (rename the resource for the relevant subsystem):
+
+```text
+service:worker resource_name:"process posthog-integration-project" status:error
+```
+
+Aggregations to run, in order:
+
+1. Count by `env` — confirms region split.
+2. Count by `error.message` — primary error classes.
+3. Count by `(projectId, error.message)` — which tenants are affected, by
+   class. `projectId` lives on span tags as `@projectId` for log search and as
+   a tag for spans (depends on instrumentation site).
+4. p50 / p95 / p99 duration by region — fingerprints timeouts vs. crashes vs.
+   slow successes.
+
+If `aggregate_spans` returns no results, check:
+
+- the resource name is right (case-sensitive, see `repo-debug-map.md`);
+- the time window covers when the issue was actually firing;
+- the region tag matches reality (the EU MCP only sees EU traces).
+
+## 2. Log Sweep — read what the handler said
+
+Logs are the right tool for *messages* the handler emitted. Spans are the
+right tool for *which handler invocations failed*.
+
+Starter shapes:
+
+```text
+service:worker env:prod-eu @projectId:cm1r6u1iq00ccfvrkoy8vg3ms status:error
+service:worker env:prod-eu "[POSTHOG]" status:error
+```
+
+Useful log facets:
+
+- `@projectId` or `@langfuse.project.id` — Langfuse project (cuid).
+- `@error.kind` / `@error.message` / `@error.stack` — when the Winston logger
+  serialized an Error.
+- `@queue` / `@jobName` — when set by BullMQ instrumentation.
+
+For high-volume subsystems (`ingestion-queue`, `otel-ingestion-queue`),
+prefer `analyze_datadog_logs` with grouping over `search_datadog_logs` — the
+raw matches are too noisy.
+
+## 3. Metric Sweep — confirm the trend
+
+Pick 2–3 metrics that match the subsystem. Common ones:
+
+- `trace.bullmq.process.errors` and `trace.bullmq.process.duration` —
+  per-queue health from the BullMQ OTel instrumentation. Filter by
+  `resource_name:"process <queue-name>"`.
+- `trace.http_request.errors` and `trace.http_request.duration` for HTTP
+  handlers (`service:web`).
+- ClickHouse: cluster-level `clickhouse.query.duration`,
+  `clickhouse.memory_usage` — the worker doesn't emit these directly, they
+  come from the `clickhouse` integration in the infra repo.
+- Postgres: `aurora.databaseconnections`, `aurora.deadlocks` — relevant when
+  the symptom is `connection_limit` / `connection pool` errors.
+
+If the subsystem isn't already known, run `search_datadog_metrics` for the
+subsystem name and pick the obvious counter / gauge / histogram triplet.
+
+## 4. Monitors & Incidents
+
+- `search_datadog_monitors` for the subsystem name — tells you what alerts
+  *would* have fired and what their thresholds are. A muted monitor on the
+  affected subsystem is itself a finding (see LFE-9475: "EU alert muted for
+  a week").
+- `search_datadog_incidents` for the time window — links any pre-existing
+  incident the user may not have referenced.
+
+## 5. RUM / Frontend (only when the symptom is user-facing)
+
+Skip unless the issue is "page broken" / "slow load". Then:
+
+- `search_datadog_rum_events` filtered by `@view.url:` patterns matching the
+  affected route.
+- Cross-reference with `service:web` API errors at the same time.
+
+## 6. Trace Drill-Down
+
+Once a cluster is identified, fetch one or two representative traces with
+`get_datadog_trace` to read the actual stack and confirm where in the handler
+the throw originates. This is what lets you point at a specific file and
+line range in the analysis.
+
+## Anti-Patterns
+
+- Don't fetch individual logs/traces before aggregating. You'll burn context
+  on noise and miss the cluster pattern.
+- Don't trust a single-region query as global. Always compare EU and US.
+- Don't read an `error.message` literally if it goes through a custom error
+  wrapper — `validateWebhookURL` rejections, for example, are re-logged as
+  "DNS lookup failed" but are actually validator rejections.
+- Don't assume monitors are firing just because errors exist — check if the
+  monitor is muted.
+
+## Linking Out
+
+End the analysis with the actual Datadog UI URLs you queried, e.g.:
+
+```text
+https://app.datadoghq.eu/apm/traces?query=resource_name%3A%22process+posthog-integration-project%22+status%3Aerror
+https://app.datadoghq.com/apm/traces?query=resource_name%3A%22process+posthog-integration-project%22+status%3Aerror
+```
+
+so the human reader can re-run the same query.
@@ -0,0 +1,72 @@
+# Intake — What Are We Actually Investigating?
+
+Goal: extract every signal from the input *before* you touch Datadog. Cheap
+to do, expensive to skip — the wrong time window or wrong service tag can
+hide the real cause.
+
+## Source-by-Source Recipes
+
+### Linear issue (URL or `LFE-XXXX` ID)
+
+1. Fetch the issue via the Linear MCP:
+   `mcp__d39d26f2-…__get_issue` with `id: "LFE-XXXX"` and
+   `includeRelations: true`.
+2. Fetch comments separately:
+   `mcp__d39d26f2-…__list_comments` with the same `issueId`.
+3. Note: Langfuse triages **inside the issue description**. The description
+   often grows reaction sections (`projectId: <one-line note>`) as oncall
+   investigates. Treat both description and comments as authoritative state.
+4. Pull `attachments` for linked PRs/commits — these show what's already been
+   tried. Reading the diff of a half-merged fix often reveals the original
+   theory of the bug.
+5. Pull labels (`integration-posthog`, `feat-exports`, etc.) — they map
+   directly to the subsystem clusters in `repo-debug-map.md`.
+
+### GitHub issue
+
+1. `gh issue view <url-or-number> --json title,body,labels,comments,assignees`.
+2. If there's a stack trace in the issue body, copy the top frames into your
+   notes — those frames usually map straight to a handler file.
+
+### Pasted error / incident text
+
+1. Treat as the issue description. If it's a stack trace, identify:
+   - the throwing module (handler vs. SDK vs. infra),
+   - whether it looks like a *user-input* failure (HTTP 4xx, validation, auth)
+     or an *infra* failure (5xx, timeout, OOM, DNS, pool exhaustion),
+   - the error class (`Error`, `TypeError`, `PrismaClientKnownRequestError`,
+     etc.).
+2. If a `projectId` appears, that's gold — anchor every Datadog query on it.
+3. If a `traceId` (Datadog's, not Langfuse's) appears, jump straight to
+   `get_datadog_trace`.
+
+## Extract The Following Before Querying
+
+Build a small notes block. If a value is missing, mark it `?` rather than
+guessing — Datadog will tell you what's missing.
+
+- **Subsystem.** PostHog integration, blob-storage export, evaluation
+  execution, OTel ingestion, batch action, webhook delivery, etc. Map to
+  `repo-debug-map.md`.
+- **Region(s).** `prod-eu` / `prod-us` / `prod-hipaa` / `prod-jp`. If unknown,
+  query both EU and US.
+- **Service.** Most issues are `service:worker`; UI/API timeouts are
+  `service:web`. Check for both when unsure.
+- **Time window.** Default to 7 days back from the issue's `createdAt`. If
+  the issue references a specific incident or alert, use that window ±1 day.
+- **`projectId`(s).** Project IDs in Langfuse are `cuid`-shaped
+  (`cl…` / `cm…`, 25 chars). The reaction blocks in Linear descriptions
+  often *are* lists of affected project IDs.
+- **Error message fragments.** Exact substrings to grep for in DD logs:
+  `Header overflow`, `Timeout error.`, `HTTP 403`, `DNS lookup failed`,
+  `Cannot write to canceled buffer`, `connection pool`, etc.
+- **Already-attempted fixes.** Linked PRs/commits on the issue. Read their
+  diffs — your analysis must not re-recommend something that's already
+  shipped.
+
+## Output Of The Intake Step
+
+A short bullet list (not yet formatted as the final analysis) with each of
+the above filled in. The remaining steps key off this — `datadog-playbook.md`
+expects subsystem + region + window, `repo-debug-map.md` expects subsystem,
+and the output template wants the affected projects.
@@ -0,0 +1,127 @@
+# Output Template
+
+The analysis should be structured so it can be pasted directly as the first
+investigative comment on the Linear issue. The example to anchor on is the
+first comment on `LFE-9475` (PostHog Integration Processing Failures).
+
+Findings come first, recommendations last. If the data doesn't support a
+hypothesis, say so — do not invent root causes to fill the template.
+
+## Section Order
+
+1. **Header** — data source, time window, scope of sweep.
+2. **Volume & error-rate split** — table by region (always).
+3. **Hotspots** — table by `(projectId, dominant cause)` or by cluster.
+4. **Root cause by error class** — one numbered subsection per cluster.
+5. **Suggested patches** — P0 / P1 / P2, each with file paths and a short
+   code sketch.
+6. **Dashboards** — Datadog UI URLs for the queries you ran.
+
+## Skeleton (fill in with your findings)
+
+````markdown
+## Datadog APM + log analysis (<N>-day window, <YYYY-MM-DD> → <YYYY-MM-DD>)
+
+Source: APM spans with `resource_name:"process <queue-name>"` across EU and US.
+
+### Volume & error rate — <one-line summary of regional split>
+
+| Region | Total spans | Errors | Error rate |
+|---|---|---|---|
+| EU (`prod-eu`) | <n> | <n> | **<pct>%** |
+| US (`prod-us`) | <n> | <n> | **<pct>%** |
+
+<One sentence explaining where the noise actually lives.>
+
+### Hotspots — concentrated on ~<N> <region> projects
+
+<Region> errors break down by `(projectId, error.message)`:
+
+| ProjectId | Errors | Dominant cause |
+|---|---|---|
+| `<projectId>` | <n> | `<error message>` (<n>) + others |
+| ...           | ... | ...                              |
+
+<Optional: contrast with another subsystem if relevant — e.g.
+"Unlike blob storage, PostHog has multiple distinct root causes — not one
+hotspot pattern.">
+
+## Root cause by error class
+
+### 1. `<error message>` — <n> errors, <n> projects
+<2–4 sentences explaining what this error class actually is at the
+implementation level (which library, which call site). Then list candidate
+causes in order of likelihood. Mark which ones are confirmed by the data
+vs. speculative.>
+
+### 2. `<error message>` — <n> errors, mostly <n> projects
+<Same pattern.>
+
+### 3. <next class>
+<...>
+
+### <N>. <Symptom of upstream failure>
+<Use this slot when a class is a *symptom* of another class rather than an
+independent bug — call it out so suggested patches don't double-count.>
+
+## Suggested patches
+
+### P0 — <one-line summary, e.g. "Auto-disable integrations on persistent
+auth failures">
+<Why this is P0 — what noise it kills, what data it stops corrupting, what
+unblocks downstream work.>
+
+```ts
+// <relative path from repo root>
+// Short code sketch (5–20 lines). It does not need to compile —
+// it must communicate the shape of the change.
+```
+
+### P0 — <next P0>
+<...>
+
+### P1 — <smaller / less urgent fix>
+<Same shape.>
+
+### P2 — <separate-but-surfaced finding>
+<E.g. a Prisma pool sizing issue surfaced incidentally by this analysis but
+not the original bug. Call it out with its own section so it doesn't get
+lost.>
+
+### Regional split explanation (only if relevant)
+<One paragraph explaining why EU vs. US asymmetry exists — usually not an
+infra bug, just where the affected tenants happen to live.>
+
+Dashboards:
+- EU APM: <url>
+- US APM: <url>
+- (logs / metrics / monitor links as relevant)
+````
+
+## Style Rules
+
+- Lead with numbers, not adjectives. "23.5% error rate" beats "very noisy".
+- Distinguish **primary causes** from **symptoms** explicitly. Symptoms
+  shouldn't get their own P0 patch.
+- Always cite specific files when proposing a code change. A patch
+  recommendation without `worker/src/features/<…>/<file>.ts` is unfinished.
+- Code sketches are illustrative — clearly mark them as sketches if they
+  hand-wave types. The next agent / human will write the real diff.
+- If the analysis surfaces a finding *outside* the original ticket scope
+  (e.g. a Prisma pool issue while debugging PostHog), include it as a P2
+  with a sentence explaining it's separate.
+- If the data refuses to converge on a single root cause, say so. The
+  template handles N classes — use as many subsections as the data warrants.
+
+## When To Skip Sections
+
+- **Single-region deployments:** if the issue clearly affects only one
+  region, you can replace the "Volume & error rate" table with a single-row
+  variant, but still note that the other region was checked and clean.
+- **No code change recommended:** if the only finding is "the affected
+  tenants have misconfigured credentials and we should reach out", the
+  Suggested-patches section can be a single sentence — but still include
+  the dashboards.
+- **Aborted investigation:** if Datadog access fails or the data is
+  insufficient, write what you tried, what was missing, and what would let
+  the next investigator pick it up.
@@ -0,0 +1,89 @@
+# Repo Debug Map — Subsystem → Code → Datadog Filters
+
+For each subsystem we ship monitors and incidents on, this is the canonical
+map between the symptom, the Datadog query that surfaces it, and the source
+files where the bug almost certainly lives.
+
+When intake gives you a subsystem (PostHog, evals, exports, etc.), start
+here to pick the right Datadog filters and the right files to read.
+
+## Worker Async Jobs
+
+Worker handlers are wrapped by `instrumentAsync` in their queue file. The
+span resource name follows the pattern `process <queue-name>`. Queue and job
+name constants live in
+`packages/shared/src/server/queues.ts`
+(`QueueName` and `QueueJobs` enums).
+
+| Subsystem | Queue file | Handler dir | Span `resource_name` | Log prefix |
+| --- | --- | --- | --- | --- |
+| PostHog integration | `worker/src/queues/postHogIntegrationQueue.ts` | `worker/src/features/posthog/` | `process posthog-integration-project` | `[POSTHOG]` |
+| Mixpanel integration | `worker/src/queues/mixpanelIntegrationQueue.ts` | `worker/src/features/mixpanel/` | `process mixpanel-integration-project` | `[MIXPANEL]` |
+| Blob storage export | `worker/src/queues/blobStorageIntegrationQueue.ts` | `worker/src/features/blobstorage/` | `process blob-storage-project` | `[BLOBSTORAGE]` |
+| Data retention | `worker/src/queues/dataRetentionQueue.ts` | `worker/src/features/batch-data-retention-cleaner/` | `process data-retention-project` | n/a |
+| Event propagation | `worker/src/queues/eventPropagationQueue.ts` | `worker/src/features/eventPropagation/` | `process event-propagation` | n/a |
+| Cloud usage metering | `worker/src/queues/cloudUsageMeteringQueue.ts` | `worker/src/ee/` (cloud-only) | `process cloud-usage-metering` | n/a |
+| Free-tier usage threshold | `worker/src/queues/cloudFreeTierUsageThresholdQueue.ts` | `worker/src/ee/usageThresholds/` | `process cloud-free-tier-usage-threshold` | n/a |
+| Ingestion (single event) | `worker/src/queues/ingestionQueue.ts` | `worker/src/features/ingestion/` (and `IngestionService`) | BullMQ default span | n/a |
+| OTel ingestion | `worker/src/queues/otelIngestionQueue.ts` | `worker/src/features/otel/` | BullMQ default span | n/a |
+| Evaluation execution | `worker/src/queues/evalQueue.ts` | `worker/src/features/evaluation/` | BullMQ default span | n/a |
+| Batch export | `worker/src/queues/batchExportQueue.ts` | `worker/src/features/batchExport/` | BullMQ default span | n/a |
+| Webhook delivery | `worker/src/queues/webhooks.ts` | `worker/src/features/webhooks/` | BullMQ default span | n/a |
+| Trace / score / dataset / project delete | `worker/src/queues/{traceDelete,scoreDelete,datasetDelete,projectDelete}.ts` | `worker/src/features/traces/`, `…/scores/`, `…/datasets/` | BullMQ default span | n/a |
+
+For queues using BullMQ default spans (no `instrumentAsync` wrapper), search
+APM with `service:worker operation_name:bullmq.process` filtered by
+`bullmq.queue:<queue-name>`.
+
+## Web (Next.js / tRPC / public API)
+
+| Subsystem | Code | Span / log filter |
+| --- | --- | --- |
+| Public REST API | `web/src/pages/api/public/**` | `service:web resource_name:"GET /api/public/<path>"` |
+| tRPC procedures | `web/src/server/api/routers/**` | `service:web resource_name:"POST /api/trpc/<router>.<proc>"` |
+| Auth / API key verification | `web/src/features/public-api/server/apiAuth.ts` | look for `verifyAuthHeaderAndReturnScope` spans |
+| Stripe billing | `web/src/ee/features/billing/server/stripeBillingService.ts` | wrapped in `instrumentAsync`; spans named after the method |
+
+## Shared Layers
+
+These are not subsystems on their own, but are *frequently the actual cause*
+behind a worker subsystem failure.
+
+| Layer | Location | Common failure modes |
+| --- | --- | --- |
+| ClickHouse access | `packages/shared/src/server/clickhouse/`, `packages/shared/src/server/repositories/` | OOM (`Code: 241`), buffer cancel (`Code: 734`), JOIN spills, slow queries on un-pre-filtered traces |
+| Prisma access | `packages/shared/src/db.ts` and per-feature repos | `connection pool timeout` (worker default `connection_limit=5`), N+1 queries |
+| Queue contracts | `packages/shared/src/server/queues.ts` | wrong queue name, missing schema validation |
+| Logger / instrumentation | `packages/shared/src/server/logger.ts`, `packages/shared/src/server/instrumentation.ts` | log silently dropped because `LANGFUSE_LOG_LEVEL` set wrong, or span missing because handler doesn't call `instrumentAsync` |
+| Webhook URL validation | `packages/shared/src/server/validateWebhookURL.ts` | rejects with messages that *look* like DNS errors but are SSRF guard rejections |
+| Encryption | `packages/shared/encryption` | bad keys → 403/auth-style failures masquerading as upstream errors |
+
+## Common Symptoms → First Files To Read
+
+- **"403 from upstream":** check the per-integration credentials table in
+  Postgres (`PostHogIntegration`, `BlobStorageIntegration`, `WebhookConfig`,
+  etc.) and the encryption layer.
+- **"Timeout":** check the SDK timeout default and the per-stream
+  flush/batch size in the handler. Worker async jobs default to long-running
+  but the upstream SDK does not.
+- **"DNS lookup failed":** distinguish actual DNS from `validateWebhookURL`
+  rejection. The error message wrapping is misleading on purpose.
+- **"Cannot write to canceled buffer" (CH):** ClickHouse stream wasn't
+  aborted when the downstream consumer threw. Look for an `AbortController`
+  threaded through the handler.
+- **"Connection pool timeout" (Prisma):** worker `connection_limit` is set
+  in the connection string; jobs doing per-row `findFirst()` exhaust it.
+  Check whether the integration row could be cached in closure scope.
+- **"memory limit exceeded" (CH):** look for unbounded JOINs without a
+  pre-filter CTE, especially in analytics integrations.
+- **"Header overflow":** Node HTTP parser's default 80 KB ceiling. Either
+  raise `--max-http-header-size` for the worker, or replace the SDK's HTTP
+  client.
+
+## Where to Look for Already-Shipped Fixes
+
+Before recommending a patch, confirm it isn't already merged or in flight:
+
+- `attachments` on the Linear issue (PRs and commits are auto-linked).
+- `git log --oneline --since=<recent-window> -- <handler-path>`.
+- Open PRs touching the file via `gh pr list --search "<filename>"`.
@@ -31,9 +31,6 @@ pnpm workspace.
     lock refresh / reinstall path before changing `package.json`.
   - If the current parent range does not cover the requested version, upgrade
     the direct parent dependency that pulls the package in.
-   - If a compatible transitive package still stays pinned after the normal
-     refresh path, you may suggest `pnpm dedupe` to the user as an optional
-     manual follow-up, but do not run it automatically and do not require it.
   - Do not add the transitive package directly unless the user explicitly asks.

 4. Ask before changing `minimumReleaseAgeExclude`.
@@ -53,6 +50,9 @@ pnpm workspace.
   - Use the nearest package `AGENTS.md` plus the root verification matrix.
   - Finish with `pnpm why -r <package>`.
   - If companions moved too, run `pnpm why -r <companion-package>` for them as well.
+   - After fixing or upgrading a package, strongly suggest that the user run
+     `pnpm dedupe` as an optional cleanup step, but do not run it automatically
+     and do not require it.

 ## Quick Commands

@@ -64,6 +64,8 @@ pnpm workspace.
  `npm view <parent>@<installedVersion> dependencies peerDependencies optionalDependencies --json`
 - Final graph verification:
  `pnpm why -r <package>`
+- Optional lockfile cleanup for the user to run after the fix:
+  `pnpm dedupe`
 - Bump in the root workspace:
  `pnpm -w up <package>@<version>`
 - Bump in one workspace:
@@ -28,9 +28,12 @@ Use this skill for interactive dependency bumps in Langfuse.
 - If the current parent range does not cover the requested transitive version,
  upgrade that parent dependency instead of adding the target package directly
  unless the user explicitly wants that.
- If a compatible transitive package still stays pinned after the normal
-  refresh path, you may suggest `pnpm dedupe` to the user as an optional manual
-  follow-up, but do not run it automatically and do not require it.
+- Never manually edit `pnpm-lock.yaml`; regenerate lockfile changes with
+  `pnpm` commands only. If a lockfile-only refresh causes unrelated churn,
+  adjust the pnpm command and rerun instead of patching the lockfile by hand.
+- After fixing or upgrading a package, strongly suggest that the user run
+  `pnpm dedupe` as an optional cleanup step, but do not run it automatically
+  and do not require it.
 - Resolve the registry latest version, but do not silently upgrade to latest
  unless the user asked for latest.
 - Compare the target version with the latest version installable under the
@@ -94,13 +94,13 @@ cd packages/utils && bun add lodash

 ```bash
 # pnpm
-pnpm add jest --save-dev --filter=web --filter=@repo/ui
+pnpm add vitest --save-dev --filter=web --filter=@repo/ui

 # npm
-npm install jest --save-dev --workspace=web --workspace=@repo/ui
+npm install vitest --save-dev --workspace=web --workspace=@repo/ui

 # yarn (v2+)
-yarn workspaces foreach -R --from '{web,@repo/ui}' add jest --dev
+yarn workspaces foreach -R --from '{web,@repo/ui}' add vitest --dev
 ```

 ### Internal Packages
@@ -0,0 +1,196 @@
+#####################################################################
+# .env (template) — OCI Object Storage / S3-compatible configuration
+#
+# IMPORTANT SECURITY NOTES (Oracle best practice)
+# - Prefer OCI-native auth (Instance Principal / Workload Identity / Resource Principal)
+#   over static keys.
+# - If you must use static keys, store them in a secure secret manager
+#   (e.g., Kubernetes Secret / OCI Vault) and inject at runtime.
+# - Rotate/revoke any credentials that were previously shared or committed.
+#####################################################################
+
+
+#####################################################################
+# 1) Storage/Auth category (CHOOSE ONE)
+#
+# The app can read/write/download to/from an OCI object store for:
+# - Batch exports (exports/)
+# - Media uploads (media/)
+# - Event uploads (events/)
+#
+# Pick exactly ONE auth mechanism for OCI-native object storage by setting:
+#   LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+#   LANGFUSE_OCI_AUTH_TYPE=<one of the values below>
+#
+# Supported values:
+#   workload_identity | instance_principal | resource_principal | oci_profile | session_token
+#####################################################################
+
+
+#####################################################################
+# Category A — OCI Object Storage with INSTANCE PRINCIPAL (recommended on OCI Compute)
+# Use when:
+# - Running on OCI Compute with IAM set up (dynamic group + policies)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=instance_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category B — OCI Object Storage with WORKLOAD IDENTITY (common on OKE)
+# Use when:
+# - Running on OKE with OCI Workload Identity configured
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=workload_identity
+#
+# Optional (only if your environment requires additional CA trust):
+# NODE_EXTRA_CA_CERTS=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category C — OCI Object Storage with RESOURCE PRINCIPAL (common for OCI services)
+# Use when:
+# - Running inside an OCI service/runtime that injects Resource Principal env vars
+#   (e.g., certain managed services / automation contexts)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=resource_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category D — OCI Object Storage with OCI CONFIG PROFILE (developer local)
+# Use when:
+# - You have an OCI config file locally or mounted in the runtime
+# - You want to use a named profile
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=oci_profile
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#
+# NOTE: Avoid adding config files into images; mount/inject securely.
+#####################################################################
+
+
+#####################################################################
+# Category E — OCI Object Storage with SESSION TOKEN (short-lived user auth)
+# Use when:
+# - You use OCI CLI session authentication (short-lived token flow)
+# - USE oci session authenticate
+# - Appropriate for interactive/dev use; less common for long-running services
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=session_token
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#####################################################################
+
+
+#####################################################################
+# Other possible setup — Non-OCI provider (AWS S3 / GCP / Azure / MinIO / etc.)
+# Use when:
+# - Your object storage is NOT OCI Object Storage
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=false
+#
+# Then configure endpoints/regions/credentials for your provider.
+#####################################################################
+
+
+
+#####################################################################
+# 2) Feature: S3 Batch Export
+#
+# Required (when enabled):
+# - *_BUCKET, *_REGION, *_ENDPOINT, *_PREFIX
+# Optional:
+# - *_EXTERNAL_ENDPOINT
+# - *_FORCE_PATH_STYLE=true (needed for many S3-compatible providers like MinIO)
+#
+# Credentials:
+# - Set *_ACCESS_KEY_ID/_SECRET_ACCESS_KEY ONLY for static-key auth
+#   (non-OCI S3-compatible providers)
+#####################################################################
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse-bucket
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# OCI example region/endpoint:
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-chicago-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+
+# MinIO / S3-compat setting (safe to keep true for many S3-compatible endpoints)
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 3) Feature: S3 Media Upload
+#####################################################################
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 4) Feature: S3 Event Upload (optional)
+#####################################################################
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 5) OCI native auth configuration (used by oci_profile / session_token)
+#####################################################################
+# Only required when LANGFUSE_OCI_AUTH_TYPE is: oci_profile OR session_token
+OCI_CONFIG_FILE=__REPLACE_ME__/config
+OCI_CONFIG_PROFILE=DEFAULT
+
+
+
+#####################################################################
+# 6) Troubleshooting notes (comments only)
+#
+# - If you see TLS errors to the endpoint in Kubernetes/OKE, set NODE_EXTRA_CA_CERTS to the
+#   correct CA bundle path for your environment.
+# - If using MinIO or certain S3-compatible providers and you get bucket addressing errors,
+#   set *_FORCE_PATH_STYLE=true.
+# - If downloads work inside the cluster but not externally, configure
+#   LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT to a publicly reachable endpoint/DNS.
+#####################################################################
@@ -81,6 +81,7 @@ REDIS_CLUSTER_NODES="127.0.0.1:6370,127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373
 LANGFUSE_INGESTION_QUEUE_SHARD_COUNT=8
 LANGFUSE_INGESTION_SECONDARY_QUEUE_SHARD_COUNT=8
 LANGFUSE_OTEL_INGESTION_QUEUE_SHARD_COUNT=4
+LANGFUSE_OTEL_INGESTION_SECONDARY_QUEUE_SHARD_COUNT=4
 LANGFUSE_EVAL_EXECUTION_QUEUE_SHARD_COUNT=4
 LANGFUSE_EVAL_EXECUTION_SECONDARY_QUEUE_SHARD_COUNT=4
 LANGFUSE_LLM_AS_JUDGE_EXECUTION_QUEUE_SHARD_COUNT=4
@@ -153,11 +153,13 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
 # Langfuse AI Bedrock credentials
 AWS_ACCESS_KEY_ID="A123456789"
 AWS_SECRET_ACCESS_KEY="SAK123456789"
+LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY="1234567890abcdef"
 LANGFUSE_AWS_BEDROCK_REGION="eu-west-1"
 LANGFUSE_AWS_BEDROCK_MODEL="eu.anthropic.claude-3-haiku-20240307-v1:0"

 # Events table migration
 LANGFUSE_ENABLE_EVENTS_TABLE_OBSERVATIONS=true
+LANGFUSE_ENABLE_EVENTS_TABLE_UI=true
 LANGFUSE_ENABLE_EVENTS_TABLE_FLAGS=true
 LANGFUSE_ENABLE_EVENTS_TABLE_V2_APIS=true
 LANGFUSE_EXPERIMENT_INSERT_INTO_EVENTS_TABLE=true
@@ -69,6 +69,7 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GOOGLE_ALLOWED_DOMAINS=langfuse.com,google.com # optional allowlist of workspace domains that can sign in via Google
 # AUTH_GOOGLE_CLIENT_AUTH_METHOD=
 # AUTH_GOOGLE_CHECKS=
+# AUTH_GOOGLE_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
@@ -86,6 +87,7 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITLAB_ISSUER=
 # AUTH_GITLAB_CLIENT_AUTH_METHOD=
 # AUTH_GITLAB_CHECKS=
+# AUTH_GITLAB_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_GITLAB_URL=
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
@@ -93,30 +95,35 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_AZURE_AD_ALLOW_ACCOUNT_LINKING=false
 # AUTH_AZURE_AD_CLIENT_AUTH_METHOD=
 # AUTH_AZURE_AD_CHECKS=
+# AUTH_AZURE_AD_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_OKTA_CLIENT_ID=
 # AUTH_OKTA_CLIENT_SECRET=
 # AUTH_OKTA_ISSUER=
 # AUTH_OKTA_ALLOW_ACCOUNT_LINKING=false
 # AUTH_OKTA_CLIENT_AUTH_METHOD=
 # AUTH_OKTA_CHECKS=
+# AUTH_OKTA_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_AUTH0_CLIENT_ID=
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
 # AUTH_AUTH0_CLIENT_AUTH_METHOD=
 # AUTH_AUTH0_CHECKS=
+# AUTH_AUTH0_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_COGNITO_CLIENT_ID=
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
 # AUTH_COGNITO_CLIENT_AUTH_METHOD=
 # AUTH_COGNITO_CHECKS=
+# AUTH_COGNITO_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_KEYCLOAK_CLIENT_ID=
 # AUTH_KEYCLOAK_CLIENT_SECRET=
 # AUTH_KEYCLOAK_ISSUER=
 # AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_KEYCLOAK_CLIENT_AUTH_METHOD=
 # AUTH_KEYCLOAK_CHECKS=
+# AUTH_KEYCLOAK_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_KEYCLOAK_NAME=
 # AUTH_WORKOS_CLIENT_ID=
 # AUTH_WORKOS_CLIENT_SECRET=
@@ -133,12 +140,14 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 # AUTH_CUSTOM_CLIENT_AUTH_METHOD=
 # AUTH_CUSTOM_CHECKS=
+# AUTH_CUSTOM_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_JUMPCLOUD_CLIENT_ID=
 # AUTH_JUMPCLOUD_CLIENT_SECRET=
 # AUTH_JUMPCLOUD_ISSUER=
 # AUTH_JUMPCLOUD_ALLOW_ACCOUNT_LINKING=
 # AUTH_JUMPCLOUD_CLIENT_AUTH_METHOD=
 # AUTH_JUMPCLOUD_CHECKS=
+# AUTH_JUMPCLOUD_ID_TOKEN_SIGNED_RESPONSE_ALG=
 # AUTH_JUMPCLOUD_SCOPE=

 # Transactional email, optional
@@ -15,10 +15,10 @@ Fixes # (issue)
 <!-- Please delete bullets that are not relevant. -->

 - [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] Chore (refactoring code, technical debt, workflow improvements)
+- [ ] Chore (tooling, dependencies, CI, workflows, repo upkeep, or other maintenance work)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] Refactor (does not change functionality, e.g. code style improvements, linting)
+- [ ] Refactor (restructures existing code without changing behavior, e.g. simplify logic, split modules, reduce duplication)
 - [ ] This change requires a documentation update

 ## Mandatory Tasks
@@ -12,7 +12,7 @@ updates:
    schedule:
      interval: "daily"
    cooldown:
-      default-days: 5
+      default-days: 7
    versioning-strategy: "increase"
    commit-message:
      prefix: chore
@@ -54,6 +54,8 @@ updates:
    directory: "/"
    schedule:
      interval: "weekly"
+    cooldown:
+      default-days: 8
    commit-message:
      prefix: ci
      include: scope
@@ -10,10 +10,21 @@ on:
        type: string
        description: Name of the service to be deployed, e.g. web-ingestion, web, or worker.
        required: true
+    # Environment secrets don't auto-resolve in reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      SENTRY_AUTH_TOKEN:
+        required: false
 jobs:
  ecs-deploy:
    runs-on: blacksmith-4vcpu-ubuntu-2404
    environment: ${{ inputs.environment }}
+    permissions:
+      contents: read
    steps:
      - name: Get app name
        uses: winterjung/split@a211a1c46e35fcdc4097d59dd6282d4a9859651b # v2
@@ -22,52 +33,68 @@ jobs:
          msg: ${{ inputs.service }}
          separator: "-"
      - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Authenticate with AWS
        # GitHub/AWS recommend to use OIDC here: https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc
        # Probably more painful to configure, but would remove all long-lived credentials.
-        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.AWS_REGION }}
      - name: Login to AWS ECR
        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2
+        uses: aws-actions/amazon-ecr-login@f2e9fc6c2b355c1890b65e6f6f0e2ac3e6e22f78 # v2.1.2
      - name: Build, tag, and push Docker image
        env:
          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          REPOSITORY: ${{ steps.split.outputs._0 }}
          IMAGE_TAG: ${{ github.sha }}
+          STEPS_SPLIT_OUTPUTS__0: ${{ steps.split.outputs._0 }}
+          VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: ${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }}
+          VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }}
+          VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT: ${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }}
+          VARS_NEXT_PUBLIC_DEMO_ORG_ID: ${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }}
+          VARS_NEXT_PUBLIC_DEMO_PROJECT_ID: ${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }}
+          VARS_NEXT_PUBLIC_SENTRY_DSN: ${{ vars.NEXT_PUBLIC_SENTRY_DSN }}
+          VARS_NEXT_PUBLIC_POSTHOG_KEY: ${{ vars.NEXT_PUBLIC_POSTHOG_KEY }}
+          VARS_NEXT_PUBLIC_POSTHOG_HOST: ${{ vars.NEXT_PUBLIC_POSTHOG_HOST }}
+          VARS_NEXT_PUBLIC_PLAIN_APP_ID: ${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }}
+          VARS_SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          VARS_SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
+          VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }}
+          SECRETS_SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
        run: |
          docker build \
            -t $REGISTRY/$REPOSITORY:$IMAGE_TAG \
-            -f ./${{ steps.split.outputs._0 }}/Dockerfile \
-            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }} \
-            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }} \
-            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }} \
-            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }} \
-            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }} \
-            --build-arg NEXT_PUBLIC_SENTRY_DSN=${{ vars.NEXT_PUBLIC_SENTRY_DSN }} \
-            --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
-            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }} \
-            --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
-            --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
-            --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
-            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
+            -f ./${STEPS_SPLIT_OUTPUTS__0}/Dockerfile \
+            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION} \
+            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE} \
+            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT} \
+            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${VARS_NEXT_PUBLIC_DEMO_ORG_ID} \
+            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${VARS_NEXT_PUBLIC_DEMO_PROJECT_ID} \
+            --build-arg NEXT_PUBLIC_SENTRY_DSN=${VARS_NEXT_PUBLIC_SENTRY_DSN} \
+            --build-arg NEXT_PUBLIC_BUILD_ID=${IMAGE_TAG} \
+            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${VARS_NEXT_PUBLIC_POSTHOG_KEY} \
+            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${VARS_NEXT_PUBLIC_POSTHOG_HOST} \
+            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${VARS_NEXT_PUBLIC_PLAIN_APP_ID} \
+            --build-arg SENTRY_AUTH_TOKEN=${SECRETS_SENTRY_AUTH_TOKEN} \
+            --build-arg SENTRY_ORG=${VARS_SENTRY_ORG} \
+            --build-arg SENTRY_PROJECT=${VARS_SENTRY_PROJECT} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE} \
            .
          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
      - name: Render AWS ECS Task Definition
        id: render-task-definition
-        uses: aws-actions/amazon-ecs-render-task-definition@77954e213ba1f9f9cb016b86a1d4f6fcdea0d57e # v1
+        uses: aws-actions/amazon-ecs-render-task-definition@77954e213ba1f9f9cb016b86a1d4f6fcdea0d57e # v1.8.4
        with:
          container-name: ${{ inputs.service }}
          image: ${{ steps.login-ecr.outputs.registry }}/${{ steps.split.outputs._0 }}:${{ github.sha }}
          task-definition-family: ${{ inputs.environment }}-${{ inputs.service }}
      - name: Update AWS ECS Service
-        uses: aws-actions/amazon-ecs-deploy-task-definition@fc8fc60f3a60ffd500fcb13b209c59d221ac8c8c # v2
+        uses: aws-actions/amazon-ecs-deploy-task-definition@fc8fc60f3a60ffd500fcb13b209c59d221ac8c8c # v2.6.1
        with:
          task-definition: ${{ steps.render-task-definition.outputs.task-definition }}
          service: ${{ inputs.environment }}-${{ inputs.service }}
@@ -9,6 +9,8 @@ on:
  issue_comment:
    types: [created]

+permissions: {}
+
 jobs:
  retrigger_cla:
    # Only run on PR comments (not issue comments) with the /check-cla command
@@ -1,14 +1,15 @@
 name: Claude Review on Maintainer PRs

 on:
-  pull_request_target:
+  pull_request:
    types:
      - opened
      - ready_for_review

 jobs:
  comment:
-    if: github.event.pull_request.draft == false
+    # Only run on PRs that are not drafts and are from the same repository (i.e., not from forks)
+    if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    permissions:
      issues: write
@@ -16,7 +17,7 @@ jobs:
    steps:
      - name: Check author permission and existing review request
        id: check
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const owner = context.repo.owner;
@@ -57,7 +58,7 @@ jobs:

      - name: Add Claude review comment
        if: steps.check.outputs.should_comment == 'true'
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            await github.rest.issues.createComment({
@@ -55,11 +55,13 @@ jobs:
        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
    steps:
      - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
-        uses: github/codeql-action/init@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        with:
          languages: ${{ matrix.language }}
          build-mode: ${{ matrix.build-mode }}
@@ -87,6 +89,6 @@ jobs:
          exit 1

      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        with:
          category: "/language:${{matrix.language}}"
@@ -22,6 +22,8 @@ jobs:

    steps:
      - name: Checkout
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Codespell
-        uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # v2
+        uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
@@ -6,6 +6,8 @@ on:
      - main
  workflow_dispatch:

+permissions: {}
+
 jobs:
  rebase-dependabot:
    runs-on: ubuntu-latest
@@ -27,6 +27,8 @@ on:
          - prod-jp
        required: true

+permissions: {}
+
 concurrency:
  # Support concurrent `push` and `workflow_dispatch`` actions
  group: deploy-${{ github.event_name }}-${{ github.ref }}
@@ -41,7 +43,7 @@ jobs:
    steps:
      - name: Get affected services
        id: affected-services
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            if (context.eventName === "workflow_dispatch") {
@@ -56,7 +58,7 @@ jobs:
            return "[]"
          result-encoding: string
      - name: Print services to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          services: ${{ steps.affected-services.outputs.result }}
        with:
@@ -71,7 +73,7 @@ jobs:
    steps:
      - name: Get affected environments
        id: affected-environments
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            if (context.eventName === "workflow_dispatch") {
@@ -88,7 +90,7 @@ jobs:
            return "[]"
          result-encoding: string
      - name: Print environments to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          environments: ${{ steps.affected-environments.outputs.result }}
        with:
@@ -99,7 +101,14 @@ jobs:
  ecs-deploy:
    uses: ./.github/workflows/_deploy_ecs_service.yml
    needs: [affected-services, affected-environments]
-    secrets: inherit
+    permissions:
+      contents: read
+    # Environment secrets must be passed explicitly to reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
    strategy:
      matrix:
        service: ${{ fromJson(needs.affected-services.outputs.services) }}
@@ -9,15 +9,21 @@ on:
    branches:
      - "main"

+permissions:
+  contents: read
+  checks: write # Needed to create a check run for the license compliance check results
+
 jobs:
  license_check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      - name: Setup node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 18

@@ -32,7 +38,7 @@ jobs:

      - name: Check license-checker CSV file without headers
        id: license_check_report
-        uses: pilosus/action-pip-license-checker@cc7a461bfa27b44ad187b8578c881ef5138c13fd # v2
+        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
        with:
          external: "npm-license-checker.csv"
          external-format: "csv"
@@ -44,6 +50,8 @@ jobs:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Echo error
        if: failure()
-        run: echo "::error::${{ steps.license_check_report.outputs.report }}"
+        run: echo "::error::${STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT}"
+        env:
+          STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT: ${{ steps.license_check_report.outputs.report }}
      - name: Delete license-checker CSV file
        run: rm npm-license-checker.csv
@@ -12,6 +12,9 @@ on:
    branches:
      - "**"

+permissions:
+  contents: read
+
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -24,19 +27,44 @@ jobs:
      llm_connections_changed: ${{ steps.filter.outputs.llm_connections }}
    timeout-minutes: 15
    permissions:
+      contents: read
      pull-requests: read
    steps:
+      # Replaces fkirc/skip-duplicate-actions — skips runs whose git tree
+      # was already tested in a prior successful run of this workflow.
      - id: skip_check
-        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5
-        with:
-          do_not_skip: '["workflow_dispatch"]'
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          GH_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          COMMIT_SHA: ${{ github.sha }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          CURRENT_TREE=$(gh api "repos/$REPOSITORY/git/commits/$COMMIT_SHA" --jq '.tree.sha')
+
+          MATCH=$(gh api "repos/$REPOSITORY/actions/workflows/pipeline.yml/runs?status=success&per_page=20" \
+            | jq -r --argjson run_id "$RUN_ID" --arg current_tree "$CURRENT_TREE" \
+              '[.workflow_runs[] | select(.id != $run_id and .head_commit.tree_id == $current_tree)] | first | .head_sha // empty')
+
+          if [[ -n "$MATCH" ]]; then
+            echo "::notice::Tree $CURRENT_TREE already tested in a prior successful run (commit $MATCH) — skipping"
+            echo "should_skip=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # Recommended for paths-filter action
          # may save additional git fetch roundtrip if
          # merge-base is found within latest N commits
          fetch-depth: 20
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
+          persist-credentials: false
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        id: filter
        with:
          filters: |
@@ -50,17 +78,19 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] lint job dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: 24
          cache: "pnpm"
          cache-dependency-path: "pnpm-lock.yaml"
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] lint cache only; publish jobs rebuild artifacts and do not restore this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-lint-${{ github.sha }}
@@ -75,6 +105,8 @@ jobs:
          cp .env.dev.example .env
      - name: lint web
        run: pnpm run lint
+      - name: typecheck
+        run: pnpm run typecheck

  prettier-check:
    runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -82,13 +114,14 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] prettier check dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: 24
          cache: "pnpm"
@@ -100,25 +133,48 @@ jobs:
        run: |
          cp .env.dev.example .env
      - name: Check formatting on changed files
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
-            BASE_SHA=${{ github.event.pull_request.base.sha }}
+          if [ "$EVENT_NAME" = "pull_request" ]; then
+            BASE_SHA="$PR_BASE_SHA"
          else
            BASE_SHA=$(git merge-base origin/main HEAD)
          fi

          echo "Checking files changed from $BASE_SHA to HEAD"

-          # Get changed files
-          CHANGED_FILES=$(git diff --name-only $BASE_SHA HEAD -- '*.js' '*.jsx' '*.ts' '*.tsx' '*.css' | tr '\n' ' ')
-
-          if [ -n "$CHANGED_FILES" ] && [ "$CHANGED_FILES" != " " ]; then
-            echo "Files to check: $CHANGED_FILES"
-            pnpm prettier --check --experimental-cli $CHANGED_FILES
-          else
+          if git diff --quiet --exit-code --diff-filter=d "$BASE_SHA" HEAD -- '*.js' '*.jsx' '*.ts' '*.tsx' '*.css'; then
            echo "No JS/TS/CSS files changed - skipping prettier check"
+          else
+            git diff --name-only -z --diff-filter=d "$BASE_SHA" HEAD -- '*.js' '*.jsx' '*.ts' '*.tsx' '*.css' \
+              | xargs -0 --no-run-if-empty pnpm prettier --check --experimental-cli --
          fi

+  tests-eslint-plugin:
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+        with:
+          version: 10.33.0
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] eslint-plugin test dependency cache only; no published artifacts are built from this cached state
+        with:
+          node-version: 24
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm install
+      - name: run eslint-plugin tests
+        run: pnpm --filter @repo/eslint-plugin run test
+
  test-docker-build:
    timeout-minutes: 20
    runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -129,10 +185,12 @@ jobs:
      DOCKER_COMPOSE_FILE: docker-compose.build.yml
    steps:
      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
@@ -178,7 +236,7 @@ jobs:
          done
      - name: Upload docker diagnostics
        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-docker-build-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
          path: /tmp/docker-diagnostics
@@ -189,31 +247,35 @@ jobs:
    needs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
-    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.deploy-mode }}, shard${{ matrix.shard }}/3)
+    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.deploy-mode }})
    strategy:
      matrix:
        node-version: [24]
        postgres-version: [12, 15]
        deploy-mode: ["", "-azure", "-redis-cluster"]
-        shard: [1, 2, 3]
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
+          curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+            --output migrate.linux-amd64.tar.gz \
+            https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+          tar xzf migrate.linux-amd64.tar.gz
          sudo mv migrate /usr/bin/migrate
          which migrate
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] test job dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -228,18 +290,20 @@ jobs:
          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
          echo "LANGFUSE_CACHE_PROMPT_ENABLED=false" >> .env
          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
+          echo "LANGFUSE_TRACE_DELETE_DELAY_MS=1" >> .env
          echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100" >> .env
          echo "ADMIN_API_KEY=admin-api-key" >> .env
          echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
+          echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true" >> .env
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test job cache only; publish jobs rebuild artifacts and do not restore this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-
      - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test-only Next.js cache; not consumed by artifact publishing or release jobs
        with:
          path: |
            ~/.npm
@@ -276,6 +340,9 @@ jobs:
        run: pnpm run build
        env:
          NODE_OPTIONS: --max_old_space_size=8192
+          # TypeScript is checked explicitly in the lint job; skip duplicate
+          # Next.js type checks in test builds to reduce CI runtime.
+          NEXT_IGNORE_BUILD_ERRORS: "true"
      - name: Start Langfuse
        run: (pnpm run start&)
        env:
@@ -291,11 +358,11 @@ jobs:
          LANGFUSE_INIT_USER_PASSWORD: "password"
      - name: run tests
        working-directory: web
-        run: npx dotenv -e ../.env.test -e ../.env -- npx jest --verbose --runInBand --detectOpenHandles --selectProjects server --shard=${{ matrix.shard }}/3
+        run: npx dotenv -e ../.env.test -e ../.env -- vitest run --project server
      - name: run test-client
-        if: matrix.deploy-mode == '' && matrix.shard == 1
+        if: matrix.deploy-mode == ''
        working-directory: web
-        run: npx dotenv -e ../.env.test -e ../.env -- npx jest --verbose --runInBand --detectOpenHandles --selectProjects client
+        run: npx dotenv -e ../.env.test -e ../.env -- vitest run --project client

  tests-worker:
    timeout-minutes: 20
@@ -310,18 +377,20 @@ jobs:
        postgres-version: [12, 15]
        deploy-mode: ["", "-azure", "-redis-cluster"]
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] worker test dependency cache only; no release artifacts are produced from this cache
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -331,7 +400,10 @@ jobs:
          pnpm install
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
+          curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+            --output migrate.linux-amd64.tar.gz \
+            https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+          tar xzf migrate.linux-amd64.tar.gz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Load default env
@@ -381,18 +453,20 @@ jobs:
    if: startsWith(github.ref, 'refs/tags/') || (needs.pre-job.outputs.should_skip != 'true' && (needs.pre-job.outputs.llm_connections_changed == 'true' || github.event_name == 'workflow_dispatch'))
    name: test-worker-llm-connections (node24, pg15)
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js 24
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] llm-connection test dependency cache only; privileged secrets are used here, but this cache is not consumed by artifact publishing
        with:
          node-version: 24
          cache: "pnpm"
@@ -402,7 +476,10 @@ jobs:
          pnpm install
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
+          curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+            --output migrate.linux-amd64.tar.gz \
+            https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+          tar xzf migrate.linux-amd64.tar.gz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Load default env
@@ -442,6 +519,7 @@ jobs:
          LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID }}
          LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY }}
          LANGFUSE_LLM_CONNECTION_BEDROCK_REGION: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY }}
          LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY }}
          LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY }}

@@ -451,23 +529,25 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] e2e dependency cache only; release images are rebuilt later without restoring this cache
        with:
          node-version: 24
          cache: "pnpm"
          cache-dependency-path: "pnpm-lock.yaml"
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e cache only; no published artifact path restores this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-e2e-${{ github.sha }}
@@ -475,7 +555,7 @@ jobs:
            ${{ runner.os }}-turbo-e2e-
            ${{ runner.os }}-turbo-
      - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e-only Next.js cache; not part of any artifact build or publishing flow
        with:
          path: |
            ~/.npm
@@ -493,7 +573,10 @@ jobs:
          cp .env.dev.example web/.env
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
+          curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+            --output migrate.linux-amd64.tar.gz \
+            https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+          tar xzf migrate.linux-amd64.tar.gz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Run + migrate
@@ -517,8 +600,11 @@ jobs:
        run: pnpm run build
        env:
          NODE_OPTIONS: --max_old_space_size=8192
+          # TypeScript is checked explicitly in the lint job; skip duplicate
+          # Next.js type checks in test builds to reduce CI runtime.
+          NEXT_IGNORE_BUILD_ERRORS: "true"
      - name: Install playwright
-        run: pnpm --filter=web exec playwright install --with-deps
+        run: pnpm --filter=web exec playwright install --with-deps --only-shell chromium
      - name: Run e2e tests
        run: pnpm --filter=web run test:e2e

@@ -528,17 +614,19 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] server e2e dependency cache only; release/publish steps rebuild separately
        with:
          node-version: 24
          cache: "pnpm"
@@ -548,7 +636,10 @@ jobs:
          pnpm install
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
+          curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+            --output migrate.linux-amd64.tar.gz \
+            https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+          tar xzf migrate.linux-amd64.tar.gz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Load default env
@@ -575,6 +666,9 @@ jobs:
        run: pnpm run build
        env:
          NODE_OPTIONS: --max_old_space_size=8192
+          # TypeScript is checked explicitly in the lint job; skip duplicate
+          # Next.js type checks in test builds to reduce CI runtime.
+          NEXT_IGNORE_BUILD_ERRORS: "true"
      - name: Run server
        run: (pnpm run start&)
      - name: Check worker health
@@ -589,9 +683,11 @@ jobs:
  all-ci-passed:
    # This allows us to have a branch protection rule for tests and deploys with matrix
    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [
+    needs:
+      [
        lint,
        prettier-check,
+        tests-eslint-plugin,
        tests-web,
        tests-worker,
        test-worker-llm-connections,
@@ -620,16 +716,58 @@ jobs:
        run: exit 1
        working-directory: .
      - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        if: failure() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ CI failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ CI Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch/Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }
      - name: Output job results
        run: |
-          echo "Job results: ${{ steps.set-success-output.outputs.success }}"
+          echo "Job results: ${STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS}"
+        env:
+          STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}

  build-docker-image-release:
    needs: all-ci-passed
@@ -668,25 +806,30 @@ jobs:
    permissions:
      packages: write
      contents: read
-    env:
-      STAGING_TAG_PREFIX: staging-${{ github.run_id }}

    steps:
      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Set NEXT_PUBLIC_BUILD_ID
        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
      - name: Log in to the GitHub Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Log in to Docker Hub
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1
+        uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1.6.0
      - name: Extract metadata (labels) for Docker
        id: meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
        with:
          images: |
            ghcr.io/langfuse/${{ matrix.image_name }}
@@ -701,15 +844,50 @@ jobs:
            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
-      - name: Build and push staged Docker image (${{ matrix.component }}, ${{ matrix.platform_tag }})
-        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2
+      - name: Build and push image by digest to GitHub Container Registry (${{ matrix.component }}, ${{ matrix.platform_tag }})
+        id: build-ghcr
+        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2.1.0
        with:
          context: .
          file: ${{ matrix.dockerfile }}
-          push: true
-          tags: ghcr.io/langfuse/${{ matrix.image_name }}:${{ env.STAGING_TAG_PREFIX }}-${{ matrix.platform_tag }}
+          outputs: type=image,name=ghcr.io/langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
          labels: ${{ steps.meta.outputs.labels }}
          platforms: ${{ matrix.platform }}
+          provenance: false
+          sbom: false
+      - name: Build and push image by digest to Docker Hub (${{ matrix.component }}, ${{ matrix.platform_tag }})
+        id: build-dockerhub
+        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2.1.0
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          outputs: type=image,name=langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
+          sbom: false
+      - name: Record pushed digests
+        env:
+          DIGEST_GHCR: ${{ steps.build-ghcr.outputs.digest }}
+          DIGEST_DOCKERHUB: ${{ steps.build-dockerhub.outputs.digest }}
+          PLATFORM_TAG: ${{ matrix.platform_tag }}
+        run: |
+          if [ -z "$DIGEST_GHCR" ] || [ -z "$DIGEST_DOCKERHUB" ]; then
+            echo "Missing registry digest output"
+            exit 1
+          fi
+
+          mkdir -p "$RUNNER_TEMP/digests/ghcr" "$RUNNER_TEMP/digests/dockerhub"
+          printf '%s\n' "$DIGEST_GHCR" > "$RUNNER_TEMP/digests/ghcr/${PLATFORM_TAG}.txt"
+          printf '%s\n' "$DIGEST_DOCKERHUB" > "$RUNNER_TEMP/digests/dockerhub/${PLATFORM_TAG}.txt"
+      - name: Upload release digests
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-digests-${{ matrix.component }}-${{ matrix.platform_tag }}
+          path: |
+            ${{ runner.temp }}/digests/ghcr/${{ matrix.platform_tag }}.txt
+            ${{ runner.temp }}/digests/dockerhub/${{ matrix.platform_tag }}.txt
+          if-no-files-found: error

  publish-docker-image-release:
    needs:
@@ -728,26 +906,30 @@ jobs:
    permissions:
      packages: write
      contents: read
-    env:
-      STAGING_TAG_PREFIX: staging-${{ github.run_id }}

    steps:
-      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Log in to the GitHub Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Log in to Docker Hub
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Setup Blacksmith Builder
+        uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1.6.0
+      - name: Download release digests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: release-digests-${{ matrix.component }}-*
+          merge-multiple: true
+          path: ${{ runner.temp }}/digests
      - name: Extract metadata (tags) for GitHub Container Registry
        id: meta-ghcr
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
        with:
          images: ghcr.io/langfuse/${{ matrix.image_name }}
          flavor: |
@@ -762,7 +944,7 @@ jobs:
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
      - name: Extract metadata (tags) for Docker Hub
        id: meta-dockerhub
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
        with:
          images: langfuse/${{ matrix.image_name }}
          flavor: |
@@ -776,51 +958,71 @@ jobs:
            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
      - name: Publish multi-platform manifest to GitHub Container Registry
+        env:
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
        run: |
          ghcr_tags=()
+          ghcr_sources=()
+
          while IFS= read -r tag; do
            [ -n "$tag" ] || continue
            ghcr_tags+=("-t" "$tag")
-          done <<'EOF'
-          ${{ steps.meta-ghcr.outputs.tags }}
+          done <<EOF
+          ${STEPS_META_GHCR_OUTPUTS_TAGS}
          EOF

-          docker buildx imagetools create "${ghcr_tags[@]}" \
-            "ghcr.io/langfuse/${{ matrix.image_name }}:${STAGING_TAG_PREFIX}-amd64" \
-            "ghcr.io/langfuse/${{ matrix.image_name }}:${STAGING_TAG_PREFIX}-arm64"
+          shopt -s nullglob
+          for digest_file in "$RUNNER_TEMP"/digests/ghcr/*.txt; do
+            digest="$(cat "$digest_file")"
+            ghcr_sources+=("ghcr.io/langfuse/${IMAGE_NAME}@$digest")
+          done
+
+          if [ "${#ghcr_sources[@]}" -lt 2 ]; then
+            echo "Expected amd64 and arm64 GHCR digests for $COMPONENT"
+            exit 1
+          fi
+
+          docker buildx imagetools create "${ghcr_tags[@]}" "${ghcr_sources[@]}"
      - name: Publish multi-platform manifest to Docker Hub
+        env:
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
        run: |
          dockerhub_tags=()
-          ghcr_first_tag=""
+          dockerhub_sources=()

          while IFS= read -r tag; do
            [ -n "$tag" ] || continue
            dockerhub_tags+=("-t" "$tag")
-          done <<'EOF'
-          ${{ steps.meta-dockerhub.outputs.tags }}
+          done <<EOF
+          ${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}
          EOF

-          while IFS= read -r tag; do
-            [ -n "$tag" ] || continue
-            ghcr_first_tag="$tag"
-            break
-          done <<'EOF'
-          ${{ steps.meta-ghcr.outputs.tags }}
-          EOF
+          shopt -s nullglob
+          for digest_file in "$RUNNER_TEMP"/digests/dockerhub/*.txt; do
+            digest="$(cat "$digest_file")"
+            dockerhub_sources+=("langfuse/${IMAGE_NAME}@$digest")
+          done

-          if [ -z "$ghcr_first_tag" ]; then
-            echo "No GHCR tag available to copy to Docker Hub"
+          if [ "${#dockerhub_sources[@]}" -lt 2 ]; then
+            echo "Expected amd64 and arm64 Docker Hub digests for $COMPONENT"
            exit 1
          fi

-          docker buildx imagetools create "${dockerhub_tags[@]}" "$ghcr_first_tag"
+          docker buildx imagetools create "${dockerhub_tags[@]}" "${dockerhub_sources[@]}"
      - name: Inspect published manifests
        run: |
-          ghcr_first_tag="$(printf '%s\n' "${{ steps.meta-ghcr.outputs.tags }}" | sed -n '1p')"
-          dockerhub_first_tag="$(printf '%s\n' "${{ steps.meta-dockerhub.outputs.tags }}" | sed -n '1p')"
+          ghcr_first_tag="$(printf '%s\n' "${STEPS_META_GHCR_OUTPUTS_TAGS}" | sed -n '1p')"
+          dockerhub_first_tag="$(printf '%s\n' "${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}" | sed -n '1p')"

          docker buildx imagetools inspect "$ghcr_first_tag"
          docker buildx imagetools inspect "$dockerhub_first_tag"
+        env:
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}

  notify-docker-image-release:
    needs:
@@ -833,10 +1035,50 @@ jobs:
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: exit 1
      - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always()
+        if: failure()
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ Docker release failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ Docker Release Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }
@@ -12,6 +12,8 @@ concurrency:
  group: promote-main-to-production
  cancel-in-progress: false

+permissions: {}
+
 jobs:
  promote:
    runs-on: ubuntu-latest
@@ -19,16 +21,19 @@ jobs:
    steps:
      - name: Validate confirmation
        run: |
-          if [ "${{ github.event.inputs.confirm }}" != "promote" ]; then
+          if [ "${INPUT_CONFIRM}" != "promote" ]; then
            echo "Input 'confirm' must be 'promote'."
            exit 1
          fi
+        env:
+          INPUT_CONFIRM: ${{ github.event.inputs.confirm }}

-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: main
          fetch-depth: 0
          token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false

      - name: Print commit refs
        run: |
@@ -41,4 +46,6 @@ jobs:
          fi

      - name: Force push main to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
@@ -5,15 +5,20 @@ on:
    tags:
      - "v3.[0-9]+.[0-9]+" # Semantic version tags

+permissions: {}
+
 jobs:
  release:
    runs-on: ubuntu-latest
    environment: "protected branches"
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: main # Always checkout main even for tagged releases
          fetch-depth: 0
          token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false
      - name: Push to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
@@ -12,20 +12,26 @@ concurrency:
  group: sdk-api-spec-${{ github.ref }}
  cancel-in-progress: true

+permissions:
+  contents: read
+
 jobs:
  generate-sdk-api-specs:
    runs-on: ubuntu-latest
+    environment: "protected branches"
    steps:
      - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      - name: Install pnpm
-        uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b # v2
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0

      - name: Setup Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: "24"
          cache: "pnpm"
@@ -39,7 +45,7 @@ jobs:
        run: npx fern-api generate --api server --force

      - name: Install uv
-        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
        with:
          version: "0.11.2"

@@ -3,46 +3,55 @@ on:
  push:
    branches: ["production", "main"]

+permissions:
+  contents: read
+  security-events: write
+
 jobs:
  snyk:
    runs-on: ubuntu-latest
+    environment: snyk
    steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
      - name: Run Snyk to check Docker image for vulnerabilities
        # Snyk can be used to break the build when it detects vulnerabilities.
        # In this case we want to upload the issues to GitHub Code Scanning
        continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
        env:
          # In order to use the Snyk Action you will need to have a Snyk API token.
          # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
          # or you can sign up for free at https://snyk.io/login
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse \
+            --file=web/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
        if: always()
        run: |
          if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
            echo "SARIF file fixed"
          else
            echo "No SARIF file found"
          fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
+        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        if: always()
        with:
          sarif_file: snyk.sarif
+          category: snyk-container-web
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif
@@ -3,46 +3,55 @@ on:
  push:
    branches: ["production", "main"]

+permissions:
+  contents: read
+  security-events: write
+
 jobs:
  snyk:
    runs-on: ubuntu-latest
+    environment: snyk
    steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
      - name: Run Snyk to check Docker image for vulnerabilities
        # Snyk can be used to break the build when it detects vulnerabilities.
        # In this case we want to upload the issues to GitHub Code Scanning
        continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
        env:
          # In order to use the Snyk Action you will need to have a Snyk API token.
          # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
          # or you can sign up for free at https://snyk.io/login
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse-worker
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse-worker \
+            --file=worker/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
        if: always()
        run: |
          if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
            echo "SARIF file fixed"
          else
            echo "No SARIF file found"
          fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
+        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        if: always()
        with:
          sarif_file: snyk.sarif
+          category: snyk-container-worker
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif
@@ -10,7 +10,7 @@ jobs:
      issues: write
      pull-requests: write
    steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
        with:
          days-before-issue-stale: 30
          days-before-issue-close: 14
@@ -19,7 +19,7 @@ jobs:
      pull-requests: read
    steps:
      - name: Validate PR title follows conventional commits
-        uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6
+        uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -0,0 +1,43 @@
+---
+name: Check GitHub Actions
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+  merge_group:
+  pull_request:
+    branches:
+      - "main"
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: Check GitHub Actions security
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Run zizmor (fork PR - fail on findings)
+        # Fork PRs don't get security-events: write on GITHUB_TOKEN, so SARIF
+        # upload isn't possible. Fail the job on findings instead so contributors
+        # see the error directly in the PR check.
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
+
+      - name: Run zizmor (trusted - upload SARIF)
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
+        with:
+          # Means that the action will only report issues, but not fail the workflow.
+          # Blocking merges are handled by rulesets:
+          # https://docs.github.com/en/code-security/concepts/code-scanning/about-code-scanning-alerts#pull-request-check-failures-for-code-scanning-alerts
+          advanced-security: true
@@ -0,0 +1,21 @@
+rules:
+  secrets-outside-env:
+    config:
+      allow:
+        # Shared read-only CI credentials used to avoid Docker Hub rate limits in test jobs.
+        - DOCKERHUB_USERNAME_READ
+        - DOCKERHUB_TOKEN_READ
+        # Shared integration-test credentials intentionally kept together for the multi-provider LLM connection test job.
+        - LANGFUSE_LLM_CONNECTION_OPENAI_KEY
+        - LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL
+        - LANGFUSE_LLM_CONNECTION_AZURE_MODEL
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_REGION
+        - LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY
+        - LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY
+        # Fern token is generation-only; GitHub write actions are handled by GH_ACCESS_TOKEN in a protected environment.
+        - FERN_TOKEN
@@ -47,6 +47,7 @@ yarn-error.log*
 !.env.dev-redis-cluster.example
 !.env.prod.example
 !.env.test.example
+!.env.dev-oci.example

 # vercel
 .vercel
@@ -92,3 +93,4 @@ web/test-results/*

 # Refactoring planning files (local only)
 **/.refactor/
+**/.pi-lens/
@@ -316,18 +316,16 @@ Tests automatically create the PostgreSQL test database if it doesn't exist and

 ### Tests in the `web` package (public API)

-We're using Jest with in the `web` package. Therefore, if you want to provide an argument to the test runner, do it directly without an intermittent `--`.
-
-There are two types of unit tests:
+We're using Vitest in the `web` package. There are two types of unit tests:

 - `test` (server tests)
 - `test-client`

-To run a specific test, for example the test: `"should handle special characters in prompt names"` in `prompts.v2.servertest.ts`, run:
+To run a specific test by name within a file, run:

 ```sh
 cd web  # or with --filter=web
-pnpm test --testPathPatterns="prompts\.v2\.servertest" --testNamePattern="should handle special characters in prompt names"
+pnpm test -- prompts.v2.servertest -t "should handle special characters in prompt names"
 ```

 To run all tests:
@@ -344,7 +342,7 @@ pnpm run test:watch

 ### Tests in the `worker` package

-For the `worker` package, we're using `vitest` to run unit tests.
+For the `worker` package, we're also using Vitest to run unit tests.

 ```sh
 pnpm run test --filter=worker -- FILE_YOU_WANT_TO_TEST.ts -t "test name"
@@ -357,7 +355,7 @@ We use GitHub Actions for CI/CD, the configuration is in [`.github/workflows/pip
 CI on `main` and `pull_request`

 - Check Linting
- E2E test of API using Jest
+- E2E test of API using Vitest
 - E2E tests of UI using Playwright

 CD on `main`
@@ -31,6 +31,8 @@ services:
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
      LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
+      LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
+      LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
@@ -29,7 +29,7 @@
    "@langfuse/shared": "workspace:*",
    "@opentelemetry/api": ">=1.0.0 <1.10.0",
    "https-proxy-agent": "^7.0.6",
-    "next": "16.2.3",
+    "next": "16.2.6",
    "next-auth": "^4.24.13",
    "zod": "^4.3.6"
  },
@@ -39,7 +39,7 @@
    "@types/node": "^24.3.0",
    "@typescript/native-preview": "7.0.0-dev.20260122.3",
    "eslint": "^9.39.2",
-    "prettier": "^3.8.1",
+    "prettier": "^3.8.3",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
    "typescript": "^5.7.2"
@@ -32,6 +32,9 @@ types:
      configId:
        type: optional<string>
        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+      source:
+        type: optional<CreateScoreSource>
+        docs: The source of the score. Defaults to API. Set to ANNOTATION to prefill scores (e.g. from an LLM) for a human reviewer to verify in an annotation queue. When source is ANNOTATION, a configId is required unless dataType is CORRECTION. EVAL is reserved for internal evaluator outputs and is not accepted on this endpoint.
    examples:
      - value:
          name: "novelty"
@@ -77,6 +80,22 @@ types:
          name: "hallucination"
          value: 0
          datasetRunId: "7891-5678-90ab-hijk"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+          source: "ANNOTATION"
+
+  CreateScoreSource:
+    docs: |
+      Source values accepted when creating a score via the public API.
+      EVAL is reserved for internal evaluator outputs and is intentionally not
+      exposed here.
+    enum:
+      - API
+      - ANNOTATION

  ScoreDataType:
    enum:
@@ -56,6 +56,7 @@ types:

  BlobStorageExportFrequency:
    enum:
+      - every_20_minutes
      - hourly
      - daily
      - weekly
@@ -138,8 +139,8 @@ types:
      - `up_to_date` — all available data has been exported; next export is scheduled for the future

      **ETL usage**: poll this endpoint and check for `up_to_date` status. Compare `lastSyncAt` against your
-      ETL bookmark to determine if new data is available. Note that exports run with a 30-minute lag buffer,
-      so `lastSyncAt` will always be at least 30 minutes behind real-time.
+      ETL bookmark to determine if new data is available. Note that exports run with a 20-minute lag buffer,
+      so `lastSyncAt` will always be at least 20 minutes behind real-time.
    enum:
      - idle
      - queued
@@ -46,6 +46,9 @@ types:
      configId:
        type: optional<string>
        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+      source:
+        type: optional<CreateScoreSource>
+        docs: The source of the score. Defaults to API. Set to ANNOTATION to prefill scores (e.g. from an LLM) for a human reviewer to verify in an annotation queue. When source is ANNOTATION, a configId is required unless dataType is CORRECTION. EVAL is reserved for internal evaluator outputs and is not accepted on this endpoint.
    examples:
      - value:
          name: "novelty"
@@ -90,6 +93,22 @@ types:
          value: "Great explanation of the concept"
          dataType: "TEXT"
          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+          source: "ANNOTATION"
+          queueId: "aq-1234-5678-90ab-cdef"
+  CreateScoreSource:
+    docs: |
+      Source values accepted when creating a score via the public REST API.
+      EVAL is reserved for internal evaluator outputs and is intentionally not
+      exposed here — use commons.ScoreSource when reading scores.
+    enum:
+      - API
+      - ANNOTATION
  CreateScoreResponse:
    properties:
      id:
@@ -26,6 +26,13 @@ service:
      path: /llm-connections
      request: UpsertLlmConnectionRequest
      response: LlmConnection
+    delete:
+      method: DELETE
+      docs: Delete an LLM connection by id. Evaluators that depend on the deleted connection are automatically paused.
+      path: /llm-connections/{id}
+      path-parameters:
+        id: string
+      response: DeleteLlmConnectionResponse

 types:
  LlmConnection:
@@ -96,6 +103,10 @@ types:
          - **VertexAI**: Optional. If provided, must be `{"location": "<gcp-location>"}` (e.g., `{"location":"us-central1"}`)
          - **Other adapters**: Not supported. Omit this field or set to null.

+  DeleteLlmConnectionResponse:
+    properties:
+      message: string
+
  LlmAdapter:
    enum:
      - value: anthropic
@@ -54,7 +54,9 @@ types:
      meta: pagination.MetaResponse
  CreateScoreConfigRequest:
    properties:
-      name: string
+      name:
+        type: string
+        docs: "Name of the score config. Max 35 characters. Only letters, numbers, underscores, spaces, periods, parentheses, and hyphens are allowed."
      dataType: commons.ScoreConfigDataType
      categories:
        type: optional<list<commons.ConfigCategory>>
@@ -75,7 +77,7 @@ types:
        docs: The status of the score config showing if it is archived or not
      name:
        type: optional<string>
-        docs: The name of the score config
+        docs: "Name of the score config. Max 35 characters. Only letters, numbers, underscores, spaces, periods, parentheses, and hyphens are allowed."
      categories:
        type: optional<list<commons.ConfigCategory>>
        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
@@ -0,0 +1,564 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+types:
+  EvaluatorType:
+    docs: |
+      The evaluator engine type.
+
+      The unstable public API currently supports only LLM-as-a-judge evaluators.
+    enum:
+      - llm_as_judge
+
+  EvaluatorScope:
+    docs: |
+      Where an evaluator comes from.
+
+      - `project`: created in your project
+      - `managed`: provided by Langfuse
+    enum:
+      - project
+      - managed
+
+  EvaluationRuleTarget:
+    docs: |
+      The ingestion object type that should trigger evaluation runs.
+
+      Choose the target first, because it changes both the valid filter columns and the valid variable-mapping sources:
+      - `observation` evaluates live-ingested observations such as generations, spans, and events.
+        It supports mapping from `input`, `output`, and `metadata`.
+      - `experiment` evaluates live experiment executions and can additionally map `expected_output`.
+        It currently supports filtering by `datasetId`.
+        Discover valid dataset IDs with `GET /api/public/v2/datasets`, then use the returned dataset `id` values in your filter.
+    enum:
+      - observation
+      - experiment
+
+  EvaluationRuleStatus:
+    docs: |
+      Effective runtime status of the evaluation rule.
+
+      - `active`: enabled and currently runnable.
+      - `inactive`: disabled by configuration.
+      - `paused`: enabled, but Langfuse has blocked execution until the underlying issue is resolved.
+    enum:
+      - active
+      - inactive
+      - paused
+
+  EvaluationRuleMappingSource:
+    docs: |
+      Source field used to populate a prompt variable.
+
+      Use these values when mapping evaluator prompt variables to live data.
+
+      Target-specific rules:
+      - `target=observation` supports `input`, `output`, and `metadata`
+      - `target=experiment` supports `input`, `output`, `metadata`, and `expected_output`
+
+      Source semantics:
+      - `input`: the observation or experiment input payload
+      - `output`: the observation or experiment output payload
+      - `metadata`: the metadata object for the target. Combine with `jsonPath` when you need one nested field instead of the whole object.
+      - `expected_output`: the experiment item's expected output. Only valid for `target=experiment`.
+    enum:
+      - input
+      - output
+      - metadata
+      - expected_output
+
+  EvaluatorModelConfig:
+    docs: |
+      Optional explicit model configuration for an evaluator.
+
+      If omitted, Langfuse uses the project's default evaluation model.
+      If provided, the model must be available to the project when the evaluator or evaluation rule is enabled.
+
+      To discover valid configured `provider` values for a project, call `GET /api/public/llm-connections` and read the `provider` field from the returned connections.
+      Use a `provider` value that matches one of the connections already configured in the same project.
+
+      Recovery guidance:
+      - If evaluator creation returns `422` with `code=evaluator_preflight_failed`, either provide a valid explicit `modelConfig` here or configure the project's default evaluation model, then retry the same request.
+    properties:
+      provider:
+        type: string
+        docs: |
+          Provider identifier to use for this evaluator, for example `openai` or `anthropic`.
+
+          To discover valid values for the current project, call `GET /api/public/llm-connections` and use one of the returned `provider` values.
+      model:
+        type: string
+        docs: Model identifier exposed by the provider, for example `gpt-4.1-mini`.
+    examples:
+      - value:
+          provider: openai
+          model: gpt-4.1-mini
+
+  EvaluatorOutputDataType:
+    docs: |
+      Structured score type returned by an evaluator.
+
+      This controls the type of score value Langfuse stores for evaluation results:
+      - `NUMERIC`: a numeric score such as `0.82`
+      - `BOOLEAN`: a boolean score such as `true`
+      - `CATEGORICAL`: one or more category labels from a fixed list
+    enum:
+      - NUMERIC
+      - BOOLEAN
+      - CATEGORICAL
+
+  EvaluatorOutputFieldDefinition:
+    properties:
+      description:
+        type: string
+        docs: Human-readable instructions for what the evaluator should return in this field.
+
+  EvaluatorOutputDefinition:
+    docs: |
+      Structured output definition to send when creating an evaluator.
+
+      Agent guidance:
+      - `dataType` is required.
+      - Do not send `version`; that is an internal storage detail and is not part of the public request contract.
+      - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`.
+      - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`.
+    discriminant: dataType
+    union:
+      NUMERIC:
+        type: PublicNumericEvaluatorOutputDefinition
+      BOOLEAN:
+        type: PublicBooleanEvaluatorOutputDefinition
+      CATEGORICAL:
+        type: PublicCategoricalEvaluatorOutputDefinition
+    examples:
+      - name: Numeric
+        value:
+          dataType: NUMERIC
+          reasoning:
+            description: Explain why the answer is correct or incorrect.
+          score:
+            description: Return a score between 0 and 1.
+      - name: Boolean
+        value:
+          dataType: BOOLEAN
+          reasoning:
+            description: Explain why the output satisfies the requirement.
+          score:
+            description: Return true if the output satisfies the requirement, otherwise false.
+      - name: Categorical
+        value:
+          dataType: CATEGORICAL
+          reasoning:
+            description: Explain which category best fits the output.
+          score:
+            description: Choose the best category.
+            categories:
+              - correct
+              - partially_correct
+              - incorrect
+            shouldAllowMultipleMatches: false
+
+  PublicNumericEvaluatorOutputDefinition:
+    properties:
+      dataType:
+        type: EvaluatorOutputDataType
+        docs: Always `NUMERIC`.
+      reasoning:
+        type: EvaluatorOutputFieldDefinition
+      score:
+        type: EvaluatorOutputFieldDefinition
+
+  PublicBooleanEvaluatorOutputDefinition:
+    properties:
+      dataType:
+        type: EvaluatorOutputDataType
+        docs: Always `BOOLEAN`.
+      reasoning:
+        type: EvaluatorOutputFieldDefinition
+      score:
+        type: EvaluatorOutputFieldDefinition
+
+  PublicCategoricalEvaluatorOutputScoreDefinition:
+    properties:
+      description:
+        type: string
+      categories:
+        type: list<string>
+      shouldAllowMultipleMatches:
+        type: boolean
+
+  PublicCategoricalEvaluatorOutputDefinition:
+    properties:
+      dataType:
+        type: EvaluatorOutputDataType
+        docs: Always `CATEGORICAL`.
+      reasoning:
+        type: EvaluatorOutputFieldDefinition
+      score:
+        type: PublicCategoricalEvaluatorOutputScoreDefinition
+
+  PublicEvaluatorOutputDefinition:
+    docs: |
+      Evaluator output definition returned by the public API.
+
+      This response always includes `dataType` and never includes an internal output-definition `version`.
+      Legacy stored evaluator definitions are normalized into this shape before they are returned.
+
+      Use this response shape when deciding how to interpret future evaluation scores:
+      - `NUMERIC`: expect numeric score values
+      - `BOOLEAN`: expect `true` / `false`
+      - `CATEGORICAL`: expect one or more values from `score.categories`
+    discriminant: dataType
+    union:
+      NUMERIC:
+        type: PublicNumericEvaluatorOutputDefinition
+      BOOLEAN:
+        type: PublicBooleanEvaluatorOutputDefinition
+      CATEGORICAL:
+        type: PublicCategoricalEvaluatorOutputDefinition
+    examples:
+      - name: PublicNumeric
+        value:
+          dataType: NUMERIC
+          reasoning:
+            description: Explain why the answer is correct or incorrect.
+          score:
+            description: Return a score between 0 and 1.
+      - name: PublicCategorical
+        value:
+          dataType: CATEGORICAL
+          reasoning:
+            description: Explain which label best fits the output.
+          score:
+            description: Choose the best label.
+            categories:
+              - correct
+              - partially_correct
+              - incorrect
+            shouldAllowMultipleMatches: false
+
+  EvaluationRuleStringFilterOperator:
+    enum:
+      - name: Equals
+        value: "="
+      - name: Contains
+        value: contains
+      - name: DoesNotContain
+        value: does not contain
+      - name: StartsWith
+        value: starts with
+      - name: EndsWith
+        value: ends with
+
+  EvaluationRuleNumberFilterOperator:
+    enum:
+      - name: Equals
+        value: "="
+      - name: GreaterThan
+        value: ">"
+      - name: LessThan
+        value: "<"
+      - name: GreaterThanOrEqual
+        value: ">="
+      - name: LessThanOrEqual
+        value: "<="
+
+  EvaluationRuleOptionsFilterOperator:
+    enum:
+      - name: AnyOf
+        value: any of
+      - name: NoneOf
+        value: none of
+
+  EvaluationRuleArrayOptionsFilterOperator:
+    enum:
+      - name: AnyOf
+        value: any of
+      - name: NoneOf
+        value: none of
+      - name: AllOf
+        value: all of
+
+  EvaluationRuleBooleanFilterOperator:
+    enum:
+      - name: Equals
+        value: "="
+      - name: NotEquals
+        value: "<>"
+
+  EvaluationRuleNullFilterOperator:
+    enum:
+      - name: IsNull
+        value: is null
+      - name: IsNotNull
+        value: is not null
+
+  DateTimeEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleNumberFilterOperator
+        docs: Comparison operator for datetime values.
+      value:
+        type: datetime
+        docs: Datetime value to compare against.
+
+  StringEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleStringFilterOperator
+      value:
+        type: string
+
+  NumberEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleNumberFilterOperator
+      value:
+        type: double
+
+  StringOptionsEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleOptionsFilterOperator
+      value:
+        type: list<string>
+        docs: One or more allowed string values.
+
+  ArrayOptionsEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleArrayOptionsFilterOperator
+      value:
+        type: list<string>
+        docs: One or more array elements to match.
+
+  StringObjectEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Object-valued column to filter on. In the unstable public API this is currently `metadata`.
+      key:
+        type: string
+        docs: Top-level key inside the object-valued column to filter on.
+      operator:
+        type: EvaluationRuleStringFilterOperator
+      value:
+        type: string
+
+  NumberObjectEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Object-valued column to filter on.
+      key:
+        type: string
+        docs: Key inside the object-valued column to filter on.
+      operator:
+        type: EvaluationRuleNumberFilterOperator
+      value:
+        type: double
+
+  CategoryOptionsEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Object-valued column to filter on.
+      key:
+        type: string
+        docs: Key inside the object-valued column to filter on.
+      operator:
+        type: EvaluationRuleOptionsFilterOperator
+      value:
+        type: list<string>
+
+  BooleanEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on.
+      operator:
+        type: EvaluationRuleBooleanFilterOperator
+      value:
+        type: boolean
+
+  NullEvaluationRuleFilter:
+    properties:
+      column:
+        type: string
+        docs: Column to filter on. In the unstable public API this is currently `parentObservationId`.
+      operator:
+        type: EvaluationRuleNullFilterOperator
+      value:
+        type: optional<string>
+        docs: Ignored placeholder value. Clients may omit it or send an empty string.
+
+  EvaluationRuleMapping:
+    docs: |
+      Maps one evaluator prompt variable to one source field from the target object.
+
+      How to build a valid mapping list:
+      1. Create the evaluator or fetch it with `GET /evaluators/{id}`.
+      2. Read the evaluator `variables` array.
+      3. Add exactly one mapping object for each variable in that array.
+      4. Use the variable name exactly as returned, without braces such as `{{` or `}}`.
+      5. Choose a `source` that is valid for the selected `target`.
+
+      `jsonPath` is optional. Use it only when the selected source is a JSON object and you want to extract one nested field before inserting it into the evaluator prompt.
+
+      Recovery guidance:
+      - `invalid_variable_mapping`: the variable name is unknown for this evaluator, or the selected `source` is not valid for the chosen `target`
+      - `missing_variable_mapping`: one or more evaluator variables are not mapped yet
+      - `duplicate_variable_mapping`: the same evaluator variable appears more than once
+      - `invalid_json_path`: the JSONPath expression is malformed. Remove it or correct it.
+    properties:
+      variable:
+        type: string
+        docs: |
+          Prompt variable name without braces.
+
+          Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`.
+      source:
+        type: EvaluationRuleMappingSource
+        docs: |
+          Source field that should populate the prompt variable.
+
+          Quick reference:
+          - `target=observation`: `input`, `output`, `metadata`
+          - `target=experiment`: `input`, `output`, `metadata`, `expected_output`
+      jsonPath:
+        type: optional<string>
+        docs: |
+          Optional JSONPath selector applied to the selected source before it is passed to the evaluator prompt.
+
+          Requirements:
+          - Must start with `$`
+          - Must be a syntactically valid JSONPath expression
+          - Most useful with `source=metadata`
+    examples:
+      - name: BasicObservationMapping
+        value:
+          variable: input
+          source: input
+      - name: MetadataProjectionMapping
+        value:
+          variable: customer_tier
+          source: metadata
+          jsonPath: "$.customer.tier"
+      - name: ExperimentExpectedOutputMapping
+        value:
+          variable: expected_output
+          source: expected_output
+
+  EvaluationRuleFilter:
+    docs: |
+      One filter condition used to decide whether a live-ingested target should be evaluated.
+
+      An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run.
+
+      How to build a valid filter object:
+      - Pick the `target` first, because it changes the supported columns.
+      - Pick the filter `type`. That determines which fields are required.
+      - Use `key` only for object filters such as `metadata`.
+      - Use the correct `value` shape for the chosen filter `type`.
+
+      Operator quick reference by filter `type`:
+      - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`
+      - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+      - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+      - `stringOptions`: `any of`, `none of`
+      - `arrayOptions`: `any of`, `none of`, `all of`
+      - `stringObject`: same operators as `string`
+      - `null`: `is null`, `is not null`
+
+      Supported columns by target:
+      - `target=observation`
+        - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT`
+        - `name`: `stringOptions`, operators `any of` / `none of`
+        - `environment`: `stringOptions`, operators `any of` / `none of`
+        - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR`
+        - `version`: `string`
+        - `traceName`: `stringOptions`, operators `any of` / `none of`
+        - `userId`: `string`
+        - `sessionId`: `string`
+        - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of`
+        - `metadata`: `stringObject` with `key`
+        - `parentObservationId`: `null`, operators `is null` / `is not null`
+        - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of`
+        - `toolCalls`: `number`
+      - `target=experiment`
+        - `datasetId`: `stringOptions`, operators `any of` / `none of`
+          Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
+
+      Recovery guidance:
+      - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target`
+      - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided.
+      - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response.
+    discriminant: type
+    union:
+      "datetime":
+        type: DateTimeEvaluationRuleFilter
+      "string":
+        type: StringEvaluationRuleFilter
+      "number":
+        type: NumberEvaluationRuleFilter
+      "stringOptions":
+        type: StringOptionsEvaluationRuleFilter
+      "categoryOptions":
+        type: CategoryOptionsEvaluationRuleFilter
+      "arrayOptions":
+        type: ArrayOptionsEvaluationRuleFilter
+      "stringObject":
+        type: StringObjectEvaluationRuleFilter
+      "numberObject":
+        type: NumberObjectEvaluationRuleFilter
+      "boolean":
+        type: BooleanEvaluationRuleFilter
+      "null":
+        type: NullEvaluationRuleFilter
+    examples:
+      - name: ObservationTypeFilter
+        value:
+          type: stringOptions
+          column: type
+          operator: any of
+          value:
+            - GENERATION
+      - name: ObservationMetadataFilter
+        value:
+          type: stringObject
+          column: metadata
+          key: customerTier
+          operator: "="
+          value: enterprise
+      - name: ObservationRootOnlyFilter
+        value:
+          type: "null"
+          column: parentObservationId
+          operator: is null
+      - name: ObservationTagsFilter
+        value:
+          type: arrayOptions
+          column: tags
+          operator: any of
+          value:
+            - production
+      - name: ExperimentDatasetFilter
+        value:
+          type: stringOptions
+          column: datasetId
+          operator: any of
+          value:
+            - "550e8400-e29b-41d4-a716-446655440000"
@@ -0,0 +1,269 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+types:
+  PublicApiErrorCode:
+    docs: |
+      Machine-readable error code returned by the unstable evaluators API.
+
+      SDKs, CLIs, and agents should branch on `code` rather than parsing the human-readable `message`.
+      The HTTP status still indicates the broad error class, while `code` gives the specific failure reason.
+    enum:
+      - authentication_failed
+      - access_denied
+      - invalid_request
+      - invalid_query
+      - invalid_body
+      - invalid_filter_value
+      - invalid_json_path
+      - invalid_variable_mapping
+      - missing_variable_mapping
+      - duplicate_variable_mapping
+      - resource_not_found
+      - name_conflict
+      - evaluator_preflight_failed
+      - conflict
+      - unprocessable_content
+      - rate_limited
+      - method_not_allowed
+      - internal_error
+
+  PublicApiValidationIssue:
+    docs: |
+      One validation issue returned for malformed request bodies or query parameters.
+
+      This mirrors the most important parts of a Zod issue: a machine-readable `code`,
+      a human-readable `message`, and a structured `path`.
+    properties:
+      code:
+        type: string
+        docs: Machine-readable validation issue code emitted by the server validator.
+      message:
+        type: string
+        docs: Human-readable explanation of the validation failure.
+      path:
+        type: list<unknown>
+        docs: Path to the invalid field, for example `["mapping", 0, "jsonPath"]`.
+
+  PublicApiErrorDetails:
+    docs: |
+      Optional structured context attached to an unstable-evals error.
+
+      The populated fields depend on the error `code`:
+      - request parsing failures populate `issues`
+      - filter validation failures populate `field`, `column`, `invalidValues`, and `allowedValues`
+      - variable mapping failures populate `field`, `variable`, or `variables`
+      - JSONPath validation failures populate `field`, `variable`, and `value`
+      - evaluator preflight failures populate `evaluatorName`, `provider`, and `model`
+      - rate limiting populates `retryAfterSeconds`, `limit`, `remaining`, and `resetAt`
+    properties:
+      issues:
+        type: optional<list<PublicApiValidationIssue>>
+        docs: Validation issues for malformed request bodies or query parameters.
+      field:
+        type: optional<string>
+        docs: Path-like reference to the failing field, for example `mapping[1].jsonPath`.
+      column:
+        type: optional<string>
+        docs: Filter column that failed validation.
+      invalidValues:
+        type: optional<list<string>>
+        docs: Unsupported values supplied by the caller.
+      allowedValues:
+        type: optional<list<string>>
+        docs: Allowed values for the failing filter column.
+      variable:
+        type: optional<string>
+        docs: Evaluator variable involved in the failure.
+      variables:
+        type: optional<list<string>>
+        docs: Multiple evaluator variables involved in the failure, for example missing mappings.
+      value:
+        type: optional<string>
+        docs: Raw invalid value supplied by the caller.
+      evaluatorName:
+        type: optional<string>
+        docs: Evaluator name used during preflight validation.
+      provider:
+        type: optional<nullable<string>>
+        docs: Provider resolved during evaluator preflight, if any.
+      model:
+        type: optional<nullable<string>>
+        docs: Model resolved during evaluator preflight, if any.
+      retryAfterSeconds:
+        type: optional<integer>
+        docs: Suggested retry delay for rate-limited requests.
+      limit:
+        type: optional<integer>
+        docs: Numeric limit associated with the failure, for example the active evaluation-rule cap or the current rate-limit window.
+      remaining:
+        type: optional<integer>
+        docs: Remaining requests in the current rate-limit window.
+      resetAt:
+        type: optional<string>
+        docs: ISO-8601 timestamp when the current rate-limit window resets.
+
+  PublicApiError:
+    docs: |
+      Standard error envelope for the unstable evaluators API.
+
+      Response handling guidance:
+      - Use the HTTP status code for the broad class of failure.
+      - Use `code` for precise branching in SDKs, CLIs, or agents.
+      - Inspect `details` for field-level validation context such as invalid filter values, malformed JSONPath expressions, or missing variable mappings.
+      - Retry only after fixing the specific issue described by `code` and `details`.
+    properties:
+      message:
+        type: string
+        docs: Human-readable description of the failure.
+      code:
+        type: PublicApiErrorCode
+        docs: Stable machine-readable error code.
+      details:
+        type: optional<PublicApiErrorDetails>
+        docs: Optional structured error context. Inspect the populated fields based on `code`.
+    examples:
+      - name: InvalidFilterValue
+        value:
+          message: 'Filter column "type" contains unsupported value(s): INVALID'
+          code: invalid_filter_value
+          details:
+            field: filter[0].value
+            column: type
+            invalidValues:
+              - INVALID
+            allowedValues:
+              - GENERATION
+              - SPAN
+              - EVENT
+      - name: InvalidExperimentDatasetFilterValue
+        value:
+          message: 'Filter column "datasetId" contains dataset id(s) that do not exist in this project: dataset-prod'
+          code: invalid_filter_value
+          details:
+            field: filter[0].value
+            column: datasetId
+            invalidValues:
+              - dataset-prod
+      - name: EvaluatorPreflightFailed
+        value:
+          message: 'No valid LLM model found for evaluator "answer-correctness". No default model or custom model configured for project project_123'
+          code: evaluator_preflight_failed
+          details:
+            evaluatorName: answer-correctness
+            provider: null
+            model: null
+      - name: MissingVariableMapping
+        value:
+          message: "Missing mappings for evaluator variable(s): output"
+          code: missing_variable_mapping
+          details:
+            field: mapping
+            variables:
+              - output
+      - name: InvalidJsonPath
+        value:
+          message: 'Mapping for variable "customer_tier" has an invalid jsonPath "$[". JSONPath expressions must use balanced quotes, brackets, and parentheses.'
+          code: invalid_json_path
+          details:
+            field: mapping[0].jsonPath
+            variable: customer_tier
+            value: "$["
+      - name: NameConflict
+        value:
+          message: 'An evaluation rule named "answer-quality-live" already exists in this project. Use PATCH /api/public/unstable/evaluation-rules/erule_123 to update it instead of creating a duplicate.'
+          code: name_conflict
+          details:
+            field: name
+      - name: ActiveEvaluationRuleLimit
+        value:
+          message: This project already has the maximum number of active evaluation rules (50). Disable an existing active evaluation rule before enabling another one.
+          code: conflict
+          details:
+            limit: 50
+      - name: RateLimited
+        value:
+          message: Rate limit exceeded
+          code: rate_limited
+          details:
+            retryAfterSeconds: 60
+            limit: 1000
+            remaining: 0
+            resetAt: "2026-03-30T10:00:00.000Z"
+
+errors:
+  BadRequestError:
+    docs: |
+      Request parsing or validation failed.
+
+      Typical unstable-eval examples:
+      - malformed request body or query parameters
+      - unsupported filter value
+      - invalid JSONPath selector
+      - missing or duplicate variable mappings
+
+      Recovery guidance:
+      - read `details.issues` for malformed bodies or queries
+      - read `details.column`, `details.invalidValues`, and `details.allowedValues` for filter problems
+      - for `details.column=datasetId`, call `GET /api/public/v2/datasets` and retry with dataset `id` values from that response
+      - read `details.variable` or `details.variables` for mapping problems
+    status-code: 400
+    type: PublicApiError
+  UnauthorizedError:
+    docs: |
+      Authentication failed.
+
+      Typical unstable-eval examples:
+      - the API key or secret key is invalid
+      - the request uses the wrong host or stale credentials
+    status-code: 401
+    type: PublicApiError
+  AccessDeniedError:
+    docs: |
+      Authenticated, but the caller is not allowed to access the requested resource.
+
+      Typical unstable-eval examples:
+      - using an organization-scoped key against project-scoped evaluator endpoints
+      - using a valid key that lacks the required access level for this resource
+    status-code: 403
+    type: PublicApiError
+  NotFoundError:
+    docs: The requested evaluator or evaluation rule does not exist within the authorized project.
+    status-code: 404
+    type: PublicApiError
+  MethodNotAllowedError:
+    docs: The HTTP method is not supported for this endpoint.
+    status-code: 405
+    type: PublicApiError
+  ConflictError:
+    docs: |
+      The request conflicts with existing state.
+
+      Typical unstable-eval examples:
+      - evaluator version changed during concurrent creation
+      - an evaluation rule name already exists in the project, so the caller should PATCH the existing resource instead
+      - enabling another evaluation rule would exceed the project limit of 50 active evaluation rules
+      - the underlying state changed between read and write operations, so the client should retry
+
+      Recovery guidance:
+      - for `name_conflict`, PATCH the existing evaluation rule instead of creating another one
+      - for the active-limit conflict, disable another active evaluation rule before enabling a new one
+    status-code: 409
+    type: PublicApiError
+  UnprocessableContentError:
+    docs: |
+      The request is syntactically valid, but Langfuse cannot accept it in its current form.
+
+      The most important unstable-eval case is `code=evaluator_preflight_failed`, which means the evaluator cannot currently run with the resolved model configuration.
+
+      Recovery guidance:
+      - configure the project's default evaluation model, or send a valid explicit evaluator `modelConfig`
+      - once the model configuration is valid, retry the same request
+    status-code: 422
+    type: PublicApiError
+  TooManyRequestsError:
+    docs: The project is rate limited. Retry after the interval described in the response headers and error details.
+    status-code: 429
+    type: PublicApiError
+  InternalServerError:
+    docs: An unexpected server-side error occurred.
+    status-code: 500
+    type: PublicApiError
@@ -0,0 +1,494 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  errors: ./errors.yml
+  pagination: ../utils/pagination.yml
+
+service:
+  auth: true
+  base-path: /api/public/unstable
+  endpoints:
+    create:
+      docs: |
+        Create an evaluation rule.
+
+        An evaluation rule defines **what** incoming data should be evaluated and **how prompt variables should be populated** from that data.
+
+        Use this resource after choosing an evaluator from the evaluator endpoints.
+
+        Key rules:
+        - `name` must be unique within the project for public evaluation rules
+        - `target` must be `observation` or `experiment`
+        - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints
+        - Langfuse resolves that family to its latest version before saving the evaluation rule
+        - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId`
+        - every evaluator prompt variable must be mapped exactly once
+        - `expected_output` mappings are only valid for `target=experiment`
+        - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run
+        - at most 50 evaluation rules can be effectively active in one project at the same time
+
+        If an evaluation rule with the same `name` already exists in the project, the API returns `409`.
+        In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one.
+
+        If enabling this resource would exceed the 50-active limit, the API also returns `409`.
+        In that case, disable or pause another active evaluation rule before enabling a new one.
+
+        Current scope:
+        - evaluation rules are live-ingestion rules only
+        - they do not trigger historical backfills
+
+        Recovery guidance:
+        - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues`
+        - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response
+        - `400 missing_variable_mapping`: fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping`
+        - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable
+        - `400 invalid_variable_mapping`: switch to a valid `source` for the selected `target`, or fix the variable name
+        - `400 invalid_json_path`: remove or correct the `jsonPath`
+        - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request.
+      method: POST
+      path: /evaluation-rules
+      request: CreateEvaluationRuleRequest
+      response: EvaluationRule
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.NotFoundError
+        - errors.ConflictError
+        - errors.MethodNotAllowedError
+        - errors.UnprocessableContentError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+      examples:
+        - name: CreateObservationEvaluationRule
+          docs: Deploy an evaluator to score live generations only.
+          request:
+            name: answer-correctness-live
+            evaluator:
+              name: answer-correctness
+              scope: project
+            target: observation
+            enabled: true
+            sampling: 1
+            filter:
+              - type: stringOptions
+                column: type
+                operator: any of
+                value:
+                  - GENERATION
+            mapping:
+              - variable: input
+                source: input
+              - variable: output
+                source: output
+          response:
+            body:
+              id: erule_123
+              name: answer-correctness-live
+              evaluator:
+                id: evaltmpl_123
+                name: answer-correctness
+                scope: project
+              target: observation
+              enabled: true
+              status: active
+              pausedReason: null
+              pausedMessage: null
+              sampling: 1
+              filter:
+                - type: stringOptions
+                  column: type
+                  operator: any of
+                  value:
+                    - GENERATION
+              mapping:
+                - variable: input
+                  source: input
+                - variable: output
+                  source: output
+              createdAt: "2026-03-30T09:20:00.000Z"
+              updatedAt: "2026-03-30T09:20:00.000Z"
+        - name: CreateExperimentEvaluationRule
+          docs: Deploy an evaluator to compare experiment outputs against expected outputs. Discover valid dataset IDs with `GET /api/public/v2/datasets` first.
+          request:
+            name: experiment-expected-output-match
+            evaluator:
+              name: expected-output-match
+              scope: project
+            target: experiment
+            enabled: true
+            sampling: 0.5
+            filter:
+              - type: stringOptions
+                column: datasetId
+                operator: any of
+                value:
+                  - "550e8400-e29b-41d4-a716-446655440000"
+            mapping:
+              - variable: output
+                source: output
+              - variable: expected_output
+                source: expected_output
+          response:
+            body:
+              id: erule_456
+              name: experiment-expected-output-match
+              evaluator:
+                id: evaltmpl_456
+                name: expected-output-match
+                scope: project
+              target: experiment
+              enabled: true
+              status: active
+              pausedReason: null
+              pausedMessage: null
+              sampling: 0.5
+              filter:
+                - type: stringOptions
+                  column: datasetId
+                  operator: any of
+                  value:
+                    - "550e8400-e29b-41d4-a716-446655440000"
+              mapping:
+                - variable: output
+                  source: output
+                - variable: expected_output
+                  source: expected_output
+              createdAt: "2026-03-30T09:30:00.000Z"
+              updatedAt: "2026-03-30T09:30:00.000Z"
+
+    list:
+      docs: |
+        List evaluation rules in the authenticated project.
+
+        Each item describes one live evaluation rule and its effective runtime status.
+      method: GET
+      path: /evaluation-rules
+      request:
+        name: ListEvaluationRulesRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: 1-based page number. Defaults to `1`.
+          limit:
+            type: optional<integer>
+            docs: Maximum number of items per page. Defaults to `50`.
+      response: EvaluationRules
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.MethodNotAllowedError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+    get:
+      docs: |
+        Get one evaluation rule by its identifier.
+
+        Use this endpoint to inspect the current evaluator, target, mapping, filters, and effective runtime status.
+      method: GET
+      path: /evaluation-rules/{evaluationRuleId}
+      path-parameters:
+        evaluationRuleId:
+          type: string
+          docs: Evaluation rule identifier returned by the evaluation rule endpoints.
+      response: EvaluationRule
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.NotFoundError
+        - errors.MethodNotAllowedError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+    update:
+      docs: |
+        Update an evaluation rule.
+
+        Typical uses:
+        - enable or disable live execution
+        - switch to another evaluator
+        - adjust sampling
+        - change filters
+        - update variable mappings
+
+        Important behavior:
+        - provide only the fields you want to change
+        - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving
+        - changing `target`, `filter`, or `mapping` must still produce a valid target-specific configuration
+        - if you change `target`, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target
+        - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run
+        - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409`
+
+        Recovery guidance:
+        - if the update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping`
+        - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter`
+      method: PATCH
+      path: /evaluation-rules/{evaluationRuleId}
+      path-parameters:
+        evaluationRuleId:
+          type: string
+          docs: Evaluation rule identifier.
+      request: UpdateEvaluationRuleRequest
+      response: EvaluationRule
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.NotFoundError
+        - errors.MethodNotAllowedError
+        - errors.UnprocessableContentError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+    delete:
+      docs: |
+        Delete an evaluation rule.
+
+        This removes the live-ingestion rule only. It does not delete the referenced evaluator.
+      method: DELETE
+      path: /evaluation-rules/{evaluationRuleId}
+      path-parameters:
+        evaluationRuleId:
+          type: string
+          docs: Evaluation rule identifier.
+      response: DeleteEvaluationRuleResponse
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.NotFoundError
+        - errors.MethodNotAllowedError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+types:
+  EvaluationRule:
+    docs: |
+      Live evaluation rule for incoming data.
+
+      An evaluation rule answers:
+      - which evaluator should be used
+      - which target objects should trigger scoring
+      - how often scoring should run
+      - which target fields should populate each evaluator variable
+      - whether the deployment is active, inactive, or paused
+
+      Important status semantics:
+      - `enabled` is the desired on/off setting from the client
+      - `status` is the effective runtime state after Langfuse applies validation and blocking rules
+      - `enabled=true` with `status=paused` means the rule should run, but Langfuse has paused it until the underlying problem is fixed
+    properties:
+      id:
+        type: string
+        docs: Stable evaluation rule identifier.
+      name:
+        type: string
+        docs: Human-readable deployment name. This is independent from the evaluator name.
+      evaluator:
+        type: EvaluationRuleEvaluator
+        docs: |
+          Evaluator currently used by this rule.
+
+          `name` and `scope` identify the evaluator family conceptually.
+          `id` is the currently active evaluator version in that family.
+          If you create a newer project version with the same evaluator name later, existing evaluation rules are moved to it automatically.
+      target:
+        type: commons.EvaluationRuleTarget
+        docs: Target object type that should trigger scoring.
+      enabled:
+        type: boolean
+        docs: Desired enabled state configured by the client.
+      status:
+        type: commons.EvaluationRuleStatus
+        docs: Effective runtime status after Langfuse applies validation and blocking rules.
+      pausedReason:
+        type: nullable<string>
+        docs: Machine-readable reason when `status=paused`, otherwise `null`.
+      pausedMessage:
+        type: nullable<string>
+        docs: Human-readable explanation when `status=paused`, otherwise `null`.
+      sampling:
+        type: double
+        docs: |
+          Fraction of matching target objects that should be evaluated.
+
+          Must be greater than `0` and less than or equal to `1`.
+          - `1` means evaluate every matching target.
+          - `0.25` means evaluate approximately 25% of matching targets.
+      filter:
+        type: list<commons.EvaluationRuleFilter>
+        docs: List of filter conditions used to decide whether a target should be evaluated.
+      mapping:
+        type: list<commons.EvaluationRuleMapping>
+        docs: Variable mappings used to populate the evaluator prompt from the live target object.
+      createdAt:
+        type: datetime
+        docs: Timestamp when the evaluation rule was created.
+      updatedAt:
+        type: datetime
+        docs: Timestamp when the evaluation rule was last updated.
+    examples:
+      - value:
+          id: erule_123
+          name: answer-correctness-live
+          evaluator:
+            id: evaltmpl_123
+            name: answer-correctness
+            scope: project
+          target: observation
+          enabled: true
+          status: active
+          pausedReason: null
+          pausedMessage: null
+          sampling: 1
+          filter:
+            - type: stringOptions
+              column: type
+              operator: any of
+              value:
+                - GENERATION
+          mapping:
+            - variable: input
+              source: input
+            - variable: output
+              source: output
+          createdAt: "2026-03-30T09:20:00.000Z"
+          updatedAt: "2026-03-30T09:20:00.000Z"
+
+  EvaluationRules:
+    docs: Paginated list of evaluation rules.
+    properties:
+      data:
+        type: list<EvaluationRule>
+        docs: Evaluation rules in the current page.
+      meta:
+        type: pagination.MetaResponse
+        docs: Standard pagination metadata.
+
+  CreateEvaluationRuleRequest:
+    docs: |
+      Request body for creating an evaluation rule.
+
+      Checklist for agents and SDK clients:
+      - reference an existing evaluator family by `evaluator.name` and `evaluator.scope`
+      - choose `target=observation` or `target=experiment`
+      - if `target=experiment` and you want a dataset filter, call `GET /api/public/v2/datasets` first and use dataset `id` values in `filter[].value`
+      - fetch or inspect the evaluator first, then provide a complete variable mapping for every evaluator variable listed in `variables`
+      - optionally narrow execution with `filter`
+      - set `enabled=true` only when you want live execution immediately
+    properties:
+      name:
+        type: string
+        docs: Human-readable deployment name.
+      evaluator:
+        type: EvaluationRuleEvaluatorReference
+        docs: |
+          Evaluator family to use.
+
+          Use `name` and `scope` from the evaluator endpoints.
+          Langfuse resolves that family to its latest version before saving the rule.
+      target:
+        type: commons.EvaluationRuleTarget
+        docs: Target object type to evaluate.
+      enabled:
+        type: boolean
+        docs: Whether the deployment should be active immediately after creation.
+      sampling:
+        type: optional<double>
+        docs: Optional sampling fraction. Defaults to `1`.
+      filter:
+        type: optional<list<commons.EvaluationRuleFilter>>
+        docs: |
+          Optional filter list.
+
+          Omit or pass an empty list to evaluate all matching targets for the selected `target`.
+          Each filter object must use a column that is valid for that `target`.
+          For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
+      mapping:
+        type: list<commons.EvaluationRuleMapping>
+        docs: |
+          Required variable mappings.
+
+          Every evaluator variable must appear exactly once.
+          Build this list from the evaluator `variables` array returned by the evaluator endpoints.
+
+  UpdateEvaluationRuleRequest:
+    docs: |
+      Partial update body for an evaluation rule.
+
+      Provide only the fields you want to change.
+      An empty body is rejected.
+
+      Practical guidance:
+      - If you only want to rename the rule or change sampling, send just those fields.
+      - If you change `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the evaluator variables.
+      - If you change `target`, usually send both `filter` and `mapping` in the same request.
+      - If you change an experiment `datasetId` filter, call `GET /api/public/v2/datasets` and use dataset `id` values from that response.
+    properties:
+      name:
+        type: optional<string>
+        docs: Updated deployment name.
+      evaluator:
+        type: optional<EvaluationRuleEvaluatorReference>
+        docs: |
+          Updated evaluator family.
+
+          Langfuse resolves the provided evaluator family to its latest version before saving the rule.
+      target:
+        type: optional<commons.EvaluationRuleTarget>
+        docs: Updated target object type.
+      enabled:
+        type: optional<boolean>
+        docs: Updated desired enabled state.
+      sampling:
+        type: optional<double>
+        docs: Updated sampling fraction.
+      filter:
+        type: optional<list<commons.EvaluationRuleFilter>>
+        docs: |
+          Updated filter list.
+
+          For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
+      mapping:
+        type: optional<list<commons.EvaluationRuleMapping>>
+        docs: Updated variable mappings.
+
+  DeleteEvaluationRuleResponse:
+    docs: Confirmation response returned after successful deletion.
+    properties:
+      message:
+        type: string
+        docs: Always `Evaluation rule successfully deleted`.
+
+  EvaluationRuleEvaluatorReference:
+    docs: |
+      Evaluator family reference used when creating or updating an evaluation rule.
+
+      `name` and `scope` are enough to identify the evaluator family in the authenticated project context.
+    properties:
+      name:
+        type: string
+        docs: Evaluator family name.
+      scope:
+        type: commons.EvaluatorScope
+        docs: Whether the evaluator family is project-owned or Langfuse-managed.
+
+  EvaluationRuleEvaluator:
+    docs: |
+      Resolved evaluator currently used by the evaluation rule.
+
+      `id` is the exact active evaluator version.
+      `name` and `scope` identify the evaluator family conceptually.
+    properties:
+      id:
+        type: string
+        docs: Identifier of the exact evaluator version currently used by the rule.
+      name:
+        type: string
+        docs: Evaluator family name.
+      scope:
+        type: commons.EvaluatorScope
+        docs: Whether the evaluator family is project-owned or Langfuse-managed.
@@ -0,0 +1,250 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  errors: ./errors.yml
+  pagination: ../utils/pagination.yml
+
+service:
+  auth: true
+  base-path: /api/public/unstable
+  endpoints:
+    create:
+      docs: |
+        Create an evaluator in the authenticated project.
+
+        Use evaluators to define **how** Langfuse should score data: the prompt, the expected structured output, and the optional model configuration.
+
+        Naming behavior:
+        - If this is a new evaluator name in your project, Langfuse creates version `1`.
+        - If the name already exists in your project, Langfuse creates the next version and returns it.
+        - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name.
+
+        Recommended workflow:
+        1. Create the evaluator.
+        2. Read the returned `variables` array.
+        3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical.
+        4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`.
+
+        Recovery guidance:
+        - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request.
+        - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry.
+        - `400` with `code=invalid_body` on `outputDefinition`: send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape.
+
+        Unstable API note:
+        - This surface may evolve while the underlying evaluation data model is being redesigned.
+      method: POST
+      path: /evaluators
+      request: CreateEvaluatorRequest
+      response: Evaluator
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.MethodNotAllowedError
+        - errors.ConflictError
+        - errors.UnprocessableContentError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+      examples:
+        - name: CreateEvaluatorVersion
+          docs: Create a new version of an evaluator named `answer-correctness`.
+          request:
+            name: answer-correctness
+            prompt: |
+              You are grading an answer.
+
+              Input:
+              {{input}}
+
+              Output:
+              {{output}}
+
+              Return a score between 0 and 1.
+            outputDefinition:
+              dataType: NUMERIC
+              reasoning:
+                description: Explain why the score was assigned.
+              score:
+                description: Correctness score between 0 and 1.
+            modelConfig:
+              provider: openai
+              model: gpt-4.1-mini
+          response:
+            body:
+              id: evaltmpl_123
+              name: answer-correctness
+              version: 2
+              scope: project
+              type: llm_as_judge
+              prompt: |
+                You are grading an answer.
+
+                Input:
+                {{input}}
+
+                Output:
+                {{output}}
+
+                Return a score between 0 and 1.
+              variables:
+                - input
+                - output
+              outputDefinition:
+                dataType: NUMERIC
+                reasoning:
+                  description: Explain why the score was assigned.
+                score:
+                  description: Correctness score between 0 and 1.
+              modelConfig:
+                provider: openai
+                model: gpt-4.1-mini
+              evaluationRuleCount: 0
+              createdAt: "2026-03-30T09:00:00.000Z"
+              updatedAt: "2026-03-30T09:00:00.000Z"
+
+    list:
+      docs: |
+        List the evaluators available to the authenticated project.
+
+        Important behavior:
+        - This endpoint returns the latest version of each available evaluator.
+        - Results can include evaluators from your project and Langfuse-managed evaluators.
+        - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values.
+      method: GET
+      path: /evaluators
+      request:
+        name: ListEvaluatorsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: 1-based page number. Defaults to `1`.
+          limit:
+            type: optional<integer>
+            docs: Maximum number of items per page. Defaults to `50`.
+      response: Evaluators
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.MethodNotAllowedError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+    get:
+      docs: |
+        Get one evaluator by `id`.
+
+        Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule.
+      method: GET
+      path: /evaluators/{evaluatorId}
+      path-parameters:
+        evaluatorId:
+          type: string
+          docs: Evaluator identifier returned by the evaluator endpoints.
+      response: Evaluator
+      errors:
+        - errors.BadRequestError
+        - errors.UnauthorizedError
+        - errors.AccessDeniedError
+        - errors.NotFoundError
+        - errors.MethodNotAllowedError
+        - errors.TooManyRequestsError
+        - errors.InternalServerError
+
+types:
+  Evaluator:
+    docs: |
+      One evaluator that can be used for scoring.
+
+      An evaluator describes **how** to score data:
+      - prompt
+      - extracted prompt variables
+      - output schema
+      - optional explicit model configuration
+
+      It does not define **which** live objects are evaluated. That is the job of `evaluation-rules`.
+
+      For agent clients, the most important fields are:
+      - `variables`: use these exact names when building the evaluation-rule `mapping` array
+      - `outputDefinition`: tells you the expected score type and the evaluator's response instructions
+      - `modelConfig`: tells you whether the evaluator uses the project default model (`null`) or an explicit provider/model
+
+      Versioning behavior:
+      - `GET /evaluators` returns the latest version of each available evaluator.
+      - `GET /evaluators/{id}` can return an older version.
+      - Evaluation rules always run against the latest version for the selected evaluator name within the same source (`project` or `managed`).
+    properties:
+      id:
+        type: string
+        docs: Identifier of this evaluator.
+      name:
+        type: string
+        docs: Evaluator name.
+      version:
+        type: integer
+        docs: Version number of this evaluator.
+      scope:
+        type: commons.EvaluatorScope
+        docs: "Where this evaluator comes from: your project or Langfuse-managed defaults."
+      type:
+        type: commons.EvaluatorType
+        docs: Evaluator engine type. Currently always `llm_as_judge`.
+      prompt:
+        type: string
+        docs: Prompt template used during evaluation.
+      variables:
+        type: list<string>
+        docs: |
+          Variables extracted from the evaluator prompt.
+
+          Every variable in this list must be mapped exactly once when creating an evaluation rule.
+      outputDefinition:
+        type: commons.PublicEvaluatorOutputDefinition
+        docs: |
+          Structured output schema returned by this evaluator.
+
+          Responses always include `dataType` and omit the internal output-definition `version`.
+          Use `dataType` to decide how future scores should be interpreted.
+      modelConfig:
+        type: nullable<commons.EvaluatorModelConfig>
+        docs: Explicit model configuration, or `null` when the project default evaluation model is used.
+      evaluationRuleCount:
+        type: integer
+        docs: Number of evaluation rules in the project that currently use this evaluator version.
+      createdAt:
+        type: datetime
+        docs: Timestamp when this evaluator was created.
+      updatedAt:
+        type: datetime
+        docs: Timestamp when this evaluator was last updated.
+
+  Evaluators:
+    properties:
+      data:
+        type: list<Evaluator>
+      meta:
+        type: pagination.MetaResponse
+
+  CreateEvaluatorRequest:
+    docs: |
+      Request body for creating an evaluator.
+
+      If the same `name` already exists in your project, Langfuse creates the next version and returns it.
+      Existing evaluation rules in the same project are then moved to that new latest version automatically.
+    properties:
+      name:
+        type: string
+        docs: Evaluator name within the authenticated project.
+      prompt:
+        type: string
+        docs: Prompt template used by the evaluator.
+      outputDefinition:
+        type: commons.EvaluatorOutputDefinition
+        docs: |
+          Structured output schema the evaluator must return.
+
+          Always send `dataType`.
+          Do not send `version`; it is an internal storage detail and not part of the public request contract.
+      modelConfig:
+        type: optional<nullable<commons.EvaluatorModelConfig>>
+        docs: Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model.
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "3.167.3",
+  "version": "3.173.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -46,7 +46,7 @@
    "braces": "3.0.3",
    "dotenv-cli": "^7.4.4",
    "husky": "^9.1.7",
-    "prettier": "^3.8.1",
+    "prettier": "^3.8.3",
    "release-it": "^19.2.4",
    "turbo": "2.9.5"
  },
@@ -108,7 +108,9 @@
      "@types/node-fetch": "^2.6.13",
      "@types/react-dom": "19.2.3",
      "glob": "^10.5.0",
-      "qs": "6.14.1"
+      "qs": "6.14.1",
+      "path-to-regexp@0.1.12": "0.1.13",
+      "ip-address@10.1.0": "10.2.0"
    },
    "patchedDependencies": {
      "next-auth@4.24.13": "patches/next-auth@4.24.13.patch"
@@ -3,6 +3,7 @@ import globals from "globals";
 import tseslint from "typescript-eslint";
 import turboConfig from "eslint-config-turbo/flat";
 import eslintPluginPrettierRecommended from "eslint-plugin-prettier/recommended";
+import langfusePlugin from "@repo/eslint-plugin";
 import "eslint-plugin-only-warn";

 export default tseslint.config(
@@ -59,6 +60,9 @@ export default tseslint.config(
    },
    languageOptions: {
      parser: tseslint.parser,
+      parserOptions: {
+        projectService: true,
+      },
    },
    rules: {
      "no-undef": "off", // TypeScript handles this
@@ -82,6 +86,18 @@ export default tseslint.config(
          ignoreRestSiblings: true,
        },
      ],
+      "@typescript-eslint/no-deprecated": "warn",
+    },
+  },
+
+  // Vitest in-source testing should only be used while developing, not in committed code.
+  {
+    name: "langfuse/no-in-source-vitest",
+    plugins: {
+      "@repo": langfusePlugin,
+    },
+    rules: {
+      "@repo/no-in-source-vitest": "warn",
    },
  },
 );
@@ -2,6 +2,7 @@ import nextCoreWebVitals from "eslint-config-next/core-web-vitals";
 import eslintPluginPrettierRecommended from "eslint-plugin-prettier/recommended";
 import turboConfig from "eslint-config-turbo/flat";
 import "eslint-plugin-only-warn";
+import langfusePlugin from "@repo/eslint-plugin";

 export default [
  // Global ignores - include config files
@@ -62,6 +63,9 @@ export default [
    name: "langfuse/next/typescript",
    files: ["**/*.ts", "**/*.tsx"],
    languageOptions: {
+      parserOptions: {
+        projectService: true,
+      },
      globals: {
        React: "readonly",
        JSX: "readonly",
@@ -74,8 +78,12 @@ export default [
        },
      },
    },
+    plugins: {
+      "@repo": langfusePlugin,
+    },
    rules: {
      "no-unused-vars": "off", // Use @typescript-eslint/no-unused-vars instead
+      "@repo/no-tailwind-overflow-scroll": "warn",
      // Custom rules from old config
      "@typescript-eslint/consistent-type-imports": [
        "warn",
@@ -94,7 +102,20 @@ export default [
          ignoreRestSiblings: true,
        },
      ],
+      "@typescript-eslint/no-deprecated": "warn",
      "react/jsx-key": ["error", { warnOnDuplicates: true }],
+      "react/no-unused-prop-types": "warn",
+    },
+  },
+
+  // Vitest in-source testing should only be used while developing, not in committed code.
+  {
+    name: "langfuse/no-in-source-vitest",
+    plugins: {
+      "@repo": langfusePlugin,
+    },
+    rules: {
+      "@repo/no-in-source-vitest": "warn",
    },
  },
 ];
@@ -16,7 +16,8 @@
  ],
  "dependencies": {
    "@eslint/js": "^9.39.2",
-    "eslint-config-next": "16.2.3",
+    "@repo/eslint-plugin": "workspace:*",
+    "eslint-config-next": "16.2.6",
    "eslint-config-prettier": "^10.1.8",
    "eslint-config-turbo": "2.9.5",
    "eslint-plugin-only-warn": "^1.1.0",
@@ -27,7 +27,6 @@
    "isolatedModules": true,
    "jsx": "preserve",
    "types": [
-      "jest",
      "node"
    ],
  },
@@ -0,0 +1,21 @@
+{
+  "name": "@repo/eslint-plugin",
+  "version": "0.0.0",
+  "license": "MIT",
+  "private": true,
+  "type": "module",
+  "main": "dist/index.js",
+  "scripts": {
+    "test": "vitest run",
+    "build": "tsc"
+  },
+  "devDependencies": {
+    "@repo/typescript-config": "workspace:*",
+    "@types/node": "^24.3.0",
+    "@typescript-eslint/parser": "^8.59.0",
+    "@typescript-eslint/rule-tester": "^8.59.0",
+    "@typescript-eslint/utils": "^8.59.0",
+    "typescript": "^5.9.2",
+    "vitest": "^4.1.4"
+  }
+}
@@ -0,0 +1,11 @@
+import { default as noTailwindOverflowScroll } from "./rules/no-tailwind-overflow-scroll.js";
+import { default as noInSourceVitest } from "./rules/no-in-source-vitest.js";
+
+export const plugin = {
+  rules: {
+    "no-in-source-vitest": noInSourceVitest,
+    "no-tailwind-overflow-scroll": noTailwindOverflowScroll,
+  },
+};
+
+export default plugin;
@@ -0,0 +1,36 @@
+import { RuleTester } from "@typescript-eslint/rule-tester";
+import * as typescriptEslintParser from "@typescript-eslint/parser";
+import rule from "./no-in-source-vitest.js";
+
+const ruleTester = new RuleTester({
+  languageOptions: {
+    parser: typescriptEslintParser,
+    parserOptions: {
+      ecmaVersion: 2022,
+      sourceType: "module",
+      ecmaFeatures: {
+        jsx: true,
+      },
+    },
+  },
+});
+
+ruleTester.run("no-in-source-vitest", rule, {
+  valid: [
+    {
+      code: `export const value = 42;`,
+      filename: "/repo/web/src/features/example/value.ts",
+    },
+    {
+      code: `if (import.meta.env.DEV) console.log("dev");`,
+      filename: "/repo/web/src/features/example/value.ts",
+    },
+  ],
+  invalid: [
+    {
+      code: `if (import.meta.vitest) {}`,
+      filename: "/repo/web/src/features/example/value.ts",
+      errors: [{ messageId: "unexpected" }],
+    },
+  ],
+});
@@ -0,0 +1,29 @@
+import { createRule } from "../util.js";
+
+const rule = createRule({
+  name: "no-in-source-vitest",
+  meta: {
+    type: "suggestion",
+    docs: {
+      description:
+        "Vitest in-source testing should only be used while developing, not in committed code.",
+    },
+    messages: {
+      unexpected:
+        "Vitest in-source testing should only be used while developing, not in committed code.",
+    },
+    schema: [],
+  },
+  defaultOptions: [],
+  create(context) {
+    return {
+      'MemberExpression[computed=false][property.name="vitest"][object.type="MetaProperty"][object.meta.name="import"][object.property.name="meta"]'(
+        node,
+      ) {
+        context.report({ node, messageId: "unexpected" });
+      },
+    };
+  },
+});
+
+export default rule;
@@ -0,0 +1,93 @@
+import { RuleTester } from "@typescript-eslint/rule-tester";
+import * as typescriptEslintParser from "@typescript-eslint/parser";
+import { describe, expect, it, vi } from "vitest";
+import rule from "./no-tailwind-overflow-scroll.js";
+
+const ruleTester = new RuleTester({
+  languageOptions: {
+    parser: typescriptEslintParser,
+    parserOptions: {
+      ecmaFeatures: {
+        jsx: true,
+      },
+    },
+  },
+});
+
+ruleTester.run("no-tailwind-overflow-scroll", rule, {
+  valid: [
+    `<div className="button" />`,
+    `<div className="overflow-auto" />`,
+    `<div className="overflow-x-auto" />`,
+    `<div className="overflow-y-auto" />`,
+    `<div className="overflow-scrollbar" />`,
+    `const value = 42;`,
+  ],
+  invalid: [
+    {
+      code: `<div className="overflow-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="overflow-y-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="overflow-x-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="flex h-full overflow-x-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `const className = "flex h-full overflow-y-scroll";`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `const className = cn("text-muted-foreground flex h-full overflow-x-scroll", className);`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="md:overflow-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className={\`flex overflow-x-scroll\`} />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="!overflow-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="overflow-scroll!" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="md:!overflow-scroll" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+    {
+      code: `<div className="md:overflow-scroll!" />`,
+      errors: [{ messageId: "unexpected" }],
+    },
+  ],
+});
+
+describe("no-tailwind-overflow-scroll", () => {
+  it("ignores template elements with non-string raw values", () => {
+    const report = vi.fn();
+    const listeners = rule.create({
+      report,
+    } as never);
+
+    listeners.TemplateElement?.({
+      value: {
+        raw: null,
+      },
+    } as never);
+
+    expect(report).not.toHaveBeenCalled();
+  });
+});
@@ -0,0 +1,75 @@
+import { createRule } from "../util.js";
+
+const FORBIDDEN_TAILWIND_UTILITIES = new Set([
+  "overflow-scroll",
+  "overflow-x-scroll",
+  "overflow-y-scroll",
+]);
+
+// Tailwind important modifiers can wrap a utility token in either position,
+// for example `!overflow-scroll` or `overflow-scroll!`.
+function normalizeTailwindToken(token: string): string {
+  return token.replace(/^!|!$/g, "");
+}
+
+// Yield normalized utility tokens, for example `md:overflow-scroll!` ->
+// `overflow-scroll`.
+function* extractTailwindUtilityTokens(value: string): Generator<string> {
+  for (const match of value.matchAll(/\S+/g)) {
+    const normalizedToken = normalizeTailwindToken(match[0]);
+    const variantSeparatorIndex = normalizedToken.lastIndexOf(":");
+    yield variantSeparatorIndex === -1
+      ? normalizedToken
+      : normalizeTailwindToken(
+          normalizedToken.slice(variantSeparatorIndex + 1),
+        );
+  }
+}
+
+const rule = createRule({
+  name: "no-tailwind-overflow-scroll",
+  meta: {
+    type: "suggestion",
+    docs: {
+      description:
+        "Avoid Tailwind's forced scrollbars (`overflow-scroll`, `overflow-x-scroll`, `overflow-y-scroll`). Use `overflow-auto`, `overflow-x-auto`, `overflow-y-auto` instead to show scrollbars only when necessary.",
+    },
+    schema: [],
+    messages: {
+      unexpected:
+        "Avoid Tailwind's forced scrollbars (`overflow-scroll`, `overflow-x-scroll`, `overflow-y-scroll`). Use `overflow-auto`, `overflow-x-auto`, `overflow-y-auto` instead to show scrollbars only when necessary.",
+    },
+  },
+  defaultOptions: [],
+  create(context) {
+    return {
+      // Define AST listeners here
+      Literal(node) {
+        if (typeof node.value === "string") {
+          for (const candidateToken of extractTailwindUtilityTokens(
+            node.value,
+          )) {
+            if (FORBIDDEN_TAILWIND_UTILITIES.has(candidateToken)) {
+              context.report({ node, messageId: "unexpected" });
+              break;
+            }
+          }
+        }
+      },
+      TemplateElement(node) {
+        if (typeof node.value.raw === "string") {
+          for (const candidateToken of extractTailwindUtilityTokens(
+            node.value.raw,
+          )) {
+            if (FORBIDDEN_TAILWIND_UTILITIES.has(candidateToken)) {
+              context.report({ node, messageId: "unexpected" });
+              break;
+            }
+          }
+        }
+      },
+    };
+  },
+});
+
+export default rule;
@@ -0,0 +1,4 @@
+import { ESLintUtils } from "@typescript-eslint/utils";
+
+// There is no hosted documentation for our internal rules
+export const createRule = ESLintUtils.RuleCreator((name) => `#${name}`);
@@ -0,0 +1,14 @@
+{
+  "extends": "@repo/typescript-config/base",
+  "compilerOptions": {
+    "declaration": false,
+    "declarationMap": false,
+    "rootDir": "src",
+    "outDir": "dist",
+    "target": "ES2022",
+    "strictNullChecks": true,
+    "types": ["node", "vitest/globals"]
+  },
+  "exclude": ["node_modules"],
+  "include": ["src"]
+}
@@ -0,0 +1,19 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    setupFiles: ["./vitest.setup.ts"],
+    coverage: {
+      enabled: true,
+      provider: "v8",
+      include: ["src/**/*.ts"],
+      exclude: ["src/**/*.test.ts", "src/index.ts"],
+      thresholds: {
+        statements: 100,
+        branches: 100,
+        functions: 100,
+        lines: 100,
+      },
+    },
+  },
+});
@@ -0,0 +1,8 @@
+import { RuleTester } from "@typescript-eslint/rule-tester";
+import { afterAll, describe, it } from "vitest";
+
+// Required for RuleTester to work with Vitest
+RuleTester.afterAll = afterAll;
+RuleTester.it = it;
+RuleTester.itOnly = it.only;
+RuleTester.describe = describe;
@@ -25,11 +25,15 @@ Use root [AGENTS.md](../../AGENTS.md) for monorepo-level rules.
 - Main exports: `src/index.ts`
 - DB clients and types: `src/db.ts`
 - Server exports: `src/server/index.ts`
+- Server cache utilities: `src/server/cache/*`
 - Domain model types: `src/domain/*`
 - Repository layer: `src/server/repositories/*`
 - Queue payload schemas: `src/server/queues.ts`
 - Queue helpers: `src/server/redis/*`
 - Postgres schema: `prisma/schema.prisma`
+- For unstable public eval APIs, the public `evaluatorId` is currently the
+  exact `EvalTemplate.id`. Latest-version family grouping is derived from
+  `(projectId, name)` rather than stored on extra evaluator identity fields.
 - Prisma migrations: `prisma/migrations/*`
 - ClickHouse migrations: `clickhouse/migrations/{clustered,unclustered}/*`
 - Seeder and support scripts: `scripts/seeder/*`, `clickhouse/scripts/*`
@@ -1,2 +1,2 @@
-ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
-ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_session_id;
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1 SETTINGS alter_sync = 2;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_session_id SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
-ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1 SETTINGS alter_sync = 2;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id SETTINGS mutations_sync = 2;
@@ -1,3 +1,3 @@
-ALTER TABLE traces ON CLUSTER default DROP COLUMN IF EXISTS environment;
-ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS environment;
-ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS environment;
+ALTER TABLE traces ON CLUSTER default DROP COLUMN IF EXISTS environment SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS environment SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS environment SETTINGS alter_sync = 2;
@@ -1,3 +1,3 @@
-ALTER TABLE traces ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id;
-ALTER TABLE observations ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id;
-ALTER TABLE scores ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id;
+ALTER TABLE traces ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default ADD COLUMN environment LowCardinality(String) DEFAULT 'default' AFTER project_id SETTINGS alter_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS alter_sync = 2;
 ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS alter_sync = 2;
 ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS alter_sync = 2;
 ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_session SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_dataset_run (project_id, dataset_run_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_dataset_run (project_id, dataset_run_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS alter_sync = 2;
 ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_dataset_run SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_res_metadata_value;
-ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_res_metadata_key;
+ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_res_metadata_value SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_res_metadata_key SETTINGS alter_sync = 2;
@@ -1,4 +1,4 @@
-ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1;
-ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1;
-ALTER TABLE observations ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_res_metadata_key;
-ALTER TABLE observations ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_res_metadata_value;
+ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1 SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1 SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_res_metadata_key SETTINGS mutations_sync = 2;
+ALTER TABLE observations ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_res_metadata_value SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE dataset_run_items_rmt ON CLUSTER default ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1;
-ALTER TABLE dataset_run_items_rmt ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_trace_id;
+ALTER TABLE dataset_run_items_rmt ON CLUSTER default ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS alter_sync = 2;
+ALTER TABLE dataset_run_items_rmt ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_trace_id SETTINGS mutations_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS usage_pricing_tier_name;
-ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS usage_pricing_tier_id;
+ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS usage_pricing_tier_name SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default DROP COLUMN IF EXISTS usage_pricing_tier_id SETTINGS alter_sync = 2;
@@ -1,2 +1,2 @@
-ALTER TABLE observations ON CLUSTER default ADD COLUMN usage_pricing_tier_id Nullable(String);
-ALTER TABLE observations ON CLUSTER default ADD COLUMN usage_pricing_tier_name Nullable(String);
+ALTER TABLE observations ON CLUSTER default ADD COLUMN usage_pricing_tier_id Nullable(String) SETTINGS alter_sync = 2;
+ALTER TABLE observations ON CLUSTER default ADD COLUMN usage_pricing_tier_name Nullable(String) SETTINGS alter_sync = 2;
@@ -372,7 +372,9 @@ CREATE TABLE IF NOT EXISTS events_core
    INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1,
    INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,
    INDEX idx_updated_at updated_at TYPE minmax GRANULARITY 1,
-    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2
+    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2,
+    INDEX idx_experiment_id experiment_id TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_metadata_names metadata_names TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree(event_ts, is_deleted)
 PARTITION BY toYYYYMM(start_time)
@@ -456,6 +458,52 @@ SELECT
    is_deleted
 FROM events_full;

+-- Diagnostic table to track event size distributions across projects.
+-- Every insert (including updates) produces a row — no deduplication.
+-- See LFE-9402 for context.
+CREATE TABLE IF NOT EXISTS ingestion_size_stats (
+    project_id String,
+    trace_id String,
+    span_id String,
+    created_at DateTime64(3),
+    input_size UInt64,
+    output_size UInt64,
+    metadata_size UInt64,
+    total_size UInt64
+) ENGINE = MergeTree
+PRIMARY KEY (toStartOfHour(created_at), project_id)
+ORDER BY (toStartOfHour(created_at), project_id, trace_id, span_id, created_at);
+
+-- MV: observations -> ingestion_size_stats
+CREATE MATERIALIZED VIEW IF NOT EXISTS ingestion_size_stats_observations_mv
+TO ingestion_size_stats AS
+SELECT
+    project_id,
+    trace_id,
+    id AS span_id,
+    created_at,
+    length(coalesce(input, '')) AS input_size,
+    length(coalesce(output, '')) AS output_size,
+    arraySum(arrayMap(k -> length(k), mapKeys(metadata)))
+      + arraySum(arrayMap(v -> length(v), mapValues(metadata))) AS metadata_size,
+    byteSize(*) AS total_size
+FROM observations;
+
+-- MV: traces -> ingestion_size_stats
+CREATE MATERIALIZED VIEW IF NOT EXISTS ingestion_size_stats_traces_mv
+TO ingestion_size_stats AS
+SELECT
+    project_id,
+    id AS trace_id,
+    concat('t-', id) AS span_id,
+    created_at,
+    length(coalesce(input, '')) AS input_size,
+    length(coalesce(output, '')) AS output_size,
+    arraySum(arrayMap(k -> length(k), mapKeys(metadata)))
+      + arraySum(arrayMap(v -> length(v), mapValues(metadata))) AS metadata_size,
+    byteSize(*) AS total_size
+FROM traces;
+
 CREATE VIEW analytics_events_core AS
 SELECT
  project_id,
@@ -69,11 +69,11 @@
    "ch:drop": "bash clickhouse/scripts/drop.sh",
    "ch:dev-tables": "bash clickhouse/scripts/dev-tables.sh",
    "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed && pnpm run ch:dev-tables",
-    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' scripts/seeder/seed-clickhouse.ts",
+    "ch:seed": "dotenv -e ../../.env -- ts-node -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' scripts/seeder/seed-clickhouse.ts",
    "load:setup": "dotenv -e ../../.env -- tsx scripts/seeder/load-seed-clickhouse.ts"
  },
  "prisma": {
-    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} scripts/seeder/seed-postgres.ts"
+    "seed": "ts-node -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} scripts/seeder/seed-postgres.ts"
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
@@ -100,23 +100,26 @@
    "ajv": "^8.18.0",
    "ajv-formats": "^3.0.1",
    "bcryptjs": "^2.4.3",
-    "bullmq": "^5.34.10",
+    "bullmq": "^5.76.3",
    "date-fns": "^3.6.0",
-    "dd-trace": "^5.65.0",
+    "dd-trace": "5.97.0",
    "decimal.js": "^10.4.3",
    "exponential-backoff": "^3.1.2",
-    "ioredis": "^5.8.2",
+    "ioredis": "^5.10.1",
    "ipaddr.js": "^2.2.0",
    "jsonpath-plus": "10.3.0",
    "langchain": "^1.3.0",
    "langfuse-langchain": "3.38.20",
    "lodash": "^4.18.1",
    "lossless-json": "^4.1.1",
+    "lru-cache": "^11.2.7",
    "next-auth": "^4.24.13",
    "nodemailer": "^7.0.11",
+    "oci-common": "^2.125.0",
+    "oci-objectstorage": "^2.125.0",
    "safe-regex2": "^5.0.0",
    "undici": "^7.24.6",
-    "uuid": "^9.0.1",
+    "uuid": "^14.0.0",
    "winston": "^3.15.0",
    "zod": "^4.3.6"
  },
@@ -128,9 +131,8 @@
    "@types/nodemailer": "^7.0.11",
    "@types/pg": "^8.11.10",
    "@types/react": "19.2.14",
-    "@types/uuid": "^9.0.8",
    "eslint": "^9.39.2",
-    "prettier": "^3.8.1",
+    "prettier": "^3.8.3",
    "prisma": "^6.19.3",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_enabled" BOOLEAN NOT NULL DEFAULT true;
@@ -0,0 +1,25 @@
+-- CreateTable
+CREATE TABLE "verified_domains" (
+    "id" TEXT NOT NULL,
+    "organization_id" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "verification_token" TEXT NOT NULL,
+    "verified_at" TIMESTAMP(3),
+    "created_by_user_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "verified_domains_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "verified_domains_domain_key" ON "verified_domains"("domain");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "verified_domains_verification_token_key" ON "verified_domains"("verification_token");
+
+-- CreateIndex
+CREATE INDEX "verified_domains_organization_id_idx" ON "verified_domains"("organization_id");
+
+-- AddForeignKey
+ALTER TABLE "verified_domains" ADD CONSTRAINT "verified_domains_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,14 @@
+-- Pending claims are now shareable across orgs; verified claims remain
+-- exclusive. Drop the global uniqueness on `domain` and replace it with:
+--   1. a per-org uniqueness on (organization_id, domain) for idempotency
+--   2. a partial unique index on `domain` where verified_at IS NOT NULL,
+--      so at most one org can hold a verified claim for any given domain.
+
+DROP INDEX "verified_domains_domain_key";
+
+CREATE UNIQUE INDEX "verified_domains_organization_id_domain_key"
+  ON "verified_domains" ("organization_id", "domain");
+
+CREATE UNIQUE INDEX "verified_domains_domain_verified_key"
+  ON "verified_domains" ("domain")
+  WHERE "verified_at" IS NOT NULL;
@@ -106,6 +106,7 @@ model Organization {
  ApiKey                           ApiKey[]
  surveys                          Survey[]
  cloudSpendAlerts                 CloudSpendAlert[]
+  verifiedDomains                  VerifiedDomain[]

  @@map("organizations")
 }
@@ -585,6 +586,7 @@ model Dataset {
  metadata                Json?
  remoteExperimentUrl     String?       @map("remote_experiment_url")
  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  remoteExperimentEnabled Boolean       @default(true) @map("remote_experiment_enabled")
  inputSchema             Json?         @map("input_schema") @db.Json
  expectedOutputSchema    Json?         @map("expected_output_schema") @db.Json
  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -915,9 +917,9 @@ model EvalTemplate {
  projectId String?  @map("project_id")
  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)

-  name    String
-  version Int
-  prompt  String
+  name        String
+  version     Int
+  prompt      String

  partner String? // e.g. "ragas", describes the partner that created the template

@@ -1065,6 +1067,28 @@ model SsoConfig {
  @@map("sso_configs")
 }

+// Domain ownership claim for an organization, verified via DNS TXT record.
+// Required before an SsoConfig can be created for that domain.
+model VerifiedDomain {
+  id                String       @id @default(uuid())
+  organizationId    String       @map("organization_id")
+  organization      Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+  domain            String
+  verificationToken String       @unique @default(uuid()) @map("verification_token")
+  verifiedAt        DateTime?    @map("verified_at")
+  createdByUserId   String?      @map("created_by_user_id")
+  createdAt         DateTime     @default(now()) @map("created_at")
+  updatedAt         DateTime     @default(now()) @updatedAt @map("updated_at")
+
+  // A domain can have at most one pending claim per org (idempotency for
+  // create) and at most one verified claim across the whole instance
+  // (enforced via a partial unique index added in the migration, since
+  // Prisma cannot express partial uniqueness in the schema DSL).
+  @@unique([organizationId, domain])
+  @@index([organizationId])
+  @@map("verified_domains")
+}
+
 model PosthogIntegration {
  projectId              String                           @id @map("project_id")
  project                Project                          @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -49,13 +49,13 @@ const prepareProjectsAndApiKeys = async (
    });
    if (!apiKeyExists) {
      const sk = await hashSecretKey(
-        `sk-${Math.random().toString(36).substr(2, 9)}`,
+        `sk-${Math.random().toString(36).slice(2, 11)}`,
      );
      await prisma.apiKey.create({
        data: {
          id: apiKeyId,
          note: `API Key for ${projectId}`,
-          publicKey: `pk-${Math.random().toString(36).substr(2, 9)}`,
+          publicKey: `pk-${Math.random().toString(36).slice(2, 11)}`,
          hashedSecretKey: sk,
          displaySecretKey: getDisplaySecretKey(sk),
          scope: "PROJECT",
@@ -55,7 +55,6 @@ async function main() {
      name: "Demo User",
      email: "demo@langfuse.com",
      password: await hash("password", 12),
-      featureFlags: ["experimentsV4Enabled"],
    },
    create: {
      id: seedUserId1,
@@ -63,7 +62,6 @@ async function main() {
      email: "demo@langfuse.com",
      password: await hash("password", 12),
      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
-      featureFlags: ["experimentsV4Enabled"],
    },
  });
  const user2 = await prisma.user.upsert({
@@ -1 +1 @@
-export const VERSION = "v3.167.3";
+export const VERSION = "v3.173.0";
@@ -106,7 +106,7 @@ export type SafeGitHubDispatchActionConfig = z.infer<

 export const GitHubDispatchActionCreateSchema = z.object({
  type: z.literal("GITHUB_DISPATCH"),
-  url: z.string().url().optional(), // Optional for updates, validated in helper
+  url: z.url().optional(), // Optional for updates, validated in helper
  eventType: z.string().min(1).max(100).optional(), // Optional for updates, validated in helper
  githubToken: z.string().optional(), // Optional for updates, validated in helper
 });
@@ -116,6 +116,8 @@ export const EventsObservationSchema = ObservationSchema.extend({
  userId: z.string().nullable(),
  sessionId: z.string().nullable(),
  traceName: z.string().nullable(),
+  release: z.string().nullable().optional(),
+  tags: z.array(z.string()).optional(),
  bookmarked: z.boolean().optional(),
  public: z.boolean().optional(),
 });
@@ -7,6 +7,13 @@ export const ScoreConfigCategory = z.object({
  value: z.number(),
 });

+/** Input-only schema for score config names. Use at API/tRPC boundaries, not for DB reads. */
+export const ScoreConfigNameSchema = z
+  .string()
+  .min(1)
+  .max(35)
+  .regex(/^[\p{L}\p{N}_ .()-]+$/u, "Name contains invalid characters");
+
 // Numeric config fields
 export const NumericConfigFields = z.object({
  maxValue: z.number().nullish(),
@@ -47,7 +54,7 @@ export const validateCategories = (
  for (const category of categories) {
    if (uniqueNames.has(category.label)) {
      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
+        code: "custom",
        message: `Duplicate category label: ${category.label}, category labels must be unique`,
      });
      return;
@@ -56,7 +63,7 @@ export const validateCategories = (

    if (uniqueValues.has(category.value)) {
      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
+        code: "custom",
        message: `Duplicate category value: ${category.value}, category values must be unique`,
      });
      return;
@@ -100,7 +107,7 @@ export const validateNumericRangeFields = (
      data.maxValue <= data.minValue
    ) {
      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
+        code: "custom",
        message: "Maximum value must be greater than Minimum value",
      });
    }
@@ -10,6 +10,35 @@ export const ScoreSourceEnum = {
 export const ScoreSourceDomain = z.enum(ScoreSourceArray);
 export type ScoreSourceType = z.infer<typeof ScoreSourceDomain>;

+/**
+ * Source values external callers may set on the public create-score endpoint.
+ * EVAL is reserved for internal evaluator outputs. The `satisfies` clause
+ * keeps this a provable subset of {@link ScoreSourceArray} at compile time.
+ */
+export const PublicApiCreateScoreSourceDomain = z.enum([
+  ScoreSourceEnum.API,
+  ScoreSourceEnum.ANNOTATION,
+] as const satisfies readonly ScoreSourceType[]);
+
+/**
+ * Annotation scores need a matching score config so they can render in the
+ * annotation queue UI. CORRECTION scores are the one exception — by design
+ * they never carry a configId. This predicate + message are shared by the
+ * zod refine on {@link PostScoresBody} (sync 400 on REST) and the check in
+ * `validateAndInflateScore` (async drop for ingestion/SDK callers).
+ */
+export const ANNOTATION_SCORE_REQUIRES_CONFIG_ID_MESSAGE =
+  "configId is required when source is ANNOTATION (except for CORRECTION scores).";
+
+export const isAnnotationScoreMissingConfigId = (body: {
+  source?: ScoreSourceType | null;
+  configId?: string | null;
+  dataType?: ScoreDataTypeType;
+}): boolean =>
+  body.source === ScoreSourceEnum.ANNOTATION &&
+  !body.configId &&
+  body.dataType !== ScoreDataTypeEnum.CORRECTION;
+
 export const CORRECTION_NAME = "output" as const;

 export const TEXT_SCORE_MAX_LENGTH = 500 as const;
@@ -5,6 +5,7 @@ import z from "zod";
 export enum TableViewPresetTableName {
  Traces = "traces",
  Observations = "observations",
+  ObservationsEvents = "observations-events",
  Scores = "scores",
  Sessions = "sessions",
  SessionDetail = "session-detail",
@@ -13,8 +14,7 @@ export enum TableViewPresetTableName {
  ExperimentItems = "experiment-items",
 }

-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-const TableViewPresetDomainSchema = z.object({
+export const TableViewPresetDomainSchema = z.object({
  id: z.string(),
  projectId: z.string().nullable(),
  createdAt: z.date(),
@@ -25,7 +25,7 @@ const TableViewPresetDomainSchema = z.object({
  filters: z.array(singleFilter),
  columnOrder: z.array(z.string()),
  columnVisibility: z.record(z.string(), z.boolean()),
-  searchQuery: z.string().optional(),
+  searchQuery: z.string().nullable(),
  orderBy: orderBy,
 });

@@ -6,7 +6,7 @@ const EnvSchema = z.object({
  NODE_ENV: z
    .enum(["development", "test", "production"])
    .default("development"),
-  NEXTAUTH_URL: z.string().url().optional(),
+  NEXTAUTH_URL: z.url().optional(),
  EMAIL_FROM_ADDRESS: z.string().optional(),
  SMTP_CONNECTION_URL: z.string().optional(),
  CLOUD_CRM_EMAIL: z.string().optional(),
@@ -57,11 +57,23 @@ const EnvSchema = z.object({
    .optional(),
  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_TTL_MS: z.coerce
+    .number()
+    .positive()
+    .default(10_000),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_MAX: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(20_000),
  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(3600), // 1h
-  CLICKHOUSE_URL: z.string().url(),
-  CLICKHOUSE_READ_ONLY_URL: z.string().url().optional(),
-  CLICKHOUSE_EVENTS_READ_ONLY_URL: z.string().url().optional(),
+  CLICKHOUSE_URL: z.url(),
+  CLICKHOUSE_READ_ONLY_URL: z.url().optional(),
+  CLICKHOUSE_EVENTS_READ_ONLY_URL: z.url().optional(),
  CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
  CLICKHOUSE_DB: z.string().default("default"),
  CLICKHOUSE_USER: z.string(),
@@ -97,6 +109,10 @@ const EnvSchema = z.object({
    .number()
    .positive()
    .default(1),
+  LANGFUSE_OTEL_INGESTION_SECONDARY_QUEUE_SHARD_COUNT: z.coerce
+    .number()
+    .positive()
+    .default(1),
  LANGFUSE_OTEL_MAX_SPAN_BYTES: z.coerce.number().positive().default(9_500_000), // 9.5MB — just under ClickHouse's 10MB min_chunk_bytes_for_parallel_parsing default
  LANGFUSE_EVAL_EXECUTION_QUEUE_SHARD_COUNT: z.coerce
    .number()
@@ -179,6 +195,21 @@ const EnvSchema = z.object({
    .default("true"),
  LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
  LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
+  LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_OCI_AUTH_TYPE: z
+    .enum([
+      "workload_identity",
+      "instance_principal",
+      "resource_principal",
+      "oci_profile",
+      "session_token",
+    ])
+    .optional(),
+  LANGFUSE_OCI_CONFIG_FILE: z.string().optional(),
+  LANGFUSE_OCI_CONFIG_PROFILE: z.string().optional(),
+  NODE_EXTRA_CA_CERTS: z.string().optional(),
  STRIPE_SECRET_KEY: z.string().optional(),

  LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG: z
@@ -355,7 +386,7 @@ const EnvSchema = z.object({
  LANGFUSE_EE_LICENSE_KEY: z.string().optional(),

  // Ingestion Masking (EE feature)
-  LANGFUSE_INGESTION_MASKING_CALLBACK_URL: z.string().url().optional(),
+  LANGFUSE_INGESTION_MASKING_CALLBACK_URL: z.url().optional(),
  LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: z.coerce
    .number()
    .positive()
@@ -324,4 +324,10 @@ export const eventsTableCols: ColumnDefinition[] = [
    internal: "length(e.tool_calls)",
    nullable: true,
  },
+  {
+    name: "Is Experiment Item Root Span",
+    id: "isExperimentItemRootSpan",
+    type: "boolean",
+    internal: "e.experiment_item_root_span_id = e.span_id",
+  },
 ];
@@ -22,8 +22,22 @@ export type MappingError = {
  message: string;
 };

+export type JsonPathMissInfo = {
+  sourceField: SourceField;
+  jsonPath: string;
+  mappingKey: string | null;
+};
+
+export type JsonPathErrorInfo = JsonPathMissInfo & { message: string };
+
+export type FieldMappingResult = {
+  value: unknown;
+  misses: JsonPathMissInfo[];
+  errors: JsonPathErrorInfo[];
+};
+
 /**
- * Test if a JSON path is valid against the given data
+ * Test if a JSONPath is valid against the given data
 */
 export function testJsonPath(props: { jsonPath: string; data: unknown }): {
  success: boolean;
@@ -43,7 +57,7 @@ export function testJsonPath(props: { jsonPath: string; data: unknown }): {
 }

 /**
- * Evaluate a JSON path against the given data and return the result
+ * Evaluate a JSONPath against the given data and return the result
 */
 export function evaluateJsonPath(data: unknown, jsonPath: string): unknown {
  const parsed = typeof data === "string" ? parseJsonPrioritised(data) : data;
@@ -51,13 +65,14 @@ export function evaluateJsonPath(data: unknown, jsonPath: string): unknown {
    path: jsonPath,
    json: parsed as string | object,
    wrap: false,
+    eval: false,
  });

  return result;
 }

 /**
- * Check if a value is a JSON path (starts with $)
+ * Check if a value is a JSONPath (starts with $)
 */
 export function isJsonPath(value: string): boolean {
  return value.startsWith("$");
@@ -87,93 +102,116 @@ export function applyFieldMappingConfig(props: {
  observation: ObservationData;
  config: FieldMappingConfig;
  defaultSourceField: SourceField;
-  onJsonPathMiss?: (info: {
-    sourceField: SourceField;
-    jsonPath: string;
-    mappingKey: string | null;
-  }) => void;
-}): unknown {
-  const { observation, config, defaultSourceField, onJsonPathMiss } = props;
+}): FieldMappingResult {
+  const { observation, config, defaultSourceField } = props;
+  const misses: JsonPathMissInfo[] = [];
+  const errors: JsonPathErrorInfo[] = [];
+
+  const safeEvaluate = (
+    sourceField: SourceField,
+    jsonPath: string,
+    mappingKey: string | null,
+  ): { success: true; value: unknown } | { success: false } => {
+    try {
+      return {
+        success: true,
+        value: evaluateJsonPath(observation[sourceField], jsonPath),
+      };
+    } catch (error) {
+      errors.push({
+        sourceField,
+        jsonPath,
+        mappingKey,
+        message: error instanceof Error ? error.message : "Invalid JSONPath",
+      });
+      return { success: false };
+    }
+  };
+
+  const withValue = (value: unknown): FieldMappingResult => ({
+    value,
+    misses,
+    errors,
+  });

  switch (config.mode) {
    case "full":
-      // Return the full source field
-      return observation[defaultSourceField];
+      return withValue(observation[defaultSourceField]);

    case "none":
-      // Return null (will be written as Prisma.DbNull)
-      return null;
+      // null is written as Prisma.DbNull
+      return withValue(null);

-    case "custom":
-      if (!config.custom) {
-        return observation[defaultSourceField];
-      }
+    case "custom": {
+      if (!config.custom) return withValue(observation[defaultSourceField]);

      if (config.custom.type === "root") {
-        // Root mode: extract single value using JSON path
        const rootConfig = config.custom.rootConfig;
-        if (!rootConfig) {
-          return observation[defaultSourceField];
-        }
+        if (!rootConfig) return withValue(observation[defaultSourceField]);

-        const sourceData = observation[rootConfig.sourceField];
-        const result = evaluateJsonPath(sourceData, rootConfig.jsonPath);
-        if (result === undefined && onJsonPathMiss) {
-          onJsonPathMiss({
+        const evaluated = safeEvaluate(
+          rootConfig.sourceField,
+          rootConfig.jsonPath,
+          null,
+        );
+        if (!evaluated.success) return withValue(undefined);
+        if (evaluated.value === undefined) {
+          misses.push({
            sourceField: rootConfig.sourceField,
            jsonPath: rootConfig.jsonPath,
            mappingKey: null,
          });
        }
-        return result;
+        return withValue(evaluated.value);
      }

      if (config.custom.type === "keyValueMap") {
-        // Key-value map mode: build object from entries
-        // Supports dot notation for nested objects (e.g., "context.user_id")
        const keyValueMapConfig = config.custom.keyValueMapConfig;
        if (!keyValueMapConfig || keyValueMapConfig.entries.length === 0) {
-          return observation[defaultSourceField];
+          return withValue(observation[defaultSourceField]);
        }

        const result: Record<string, unknown> = {};
        for (const entry of keyValueMapConfig.entries) {
-          // Skip entries with empty values
-          if (!entry.value && entry.value !== "") {
-            continue;
-          }
+          if (!entry.value && entry.value !== "") continue;

          let resolvedValue: unknown;
          if (isJsonPath(entry.value)) {
-            // It's a JSON path - evaluate it
-            const sourceData = observation[entry.sourceField];
-            resolvedValue = evaluateJsonPath(sourceData, entry.value);
-            if (resolvedValue === undefined && onJsonPathMiss) {
-              onJsonPathMiss({
-                sourceField: entry.sourceField,
-                jsonPath: entry.value,
-                mappingKey: entry.key,
-              });
+            const evaluated = safeEvaluate(
+              entry.sourceField,
+              entry.value,
+              entry.key,
+            );
+            if (!evaluated.success) {
+              resolvedValue = undefined;
+            } else {
+              resolvedValue = evaluated.value;
+              if (resolvedValue === undefined) {
+                misses.push({
+                  sourceField: entry.sourceField,
+                  jsonPath: entry.value,
+                  mappingKey: entry.key,
+                });
+              }
            }
          } else {
-            // It's a literal string (including empty string)
            resolvedValue = entry.value;
          }

-          // Use dot notation path setter for nested objects
          if (entry.key.includes(".")) {
            setNestedValue(result, entry.key, resolvedValue);
          } else {
            result[entry.key] = resolvedValue;
          }
        }
-        return result;
+        return withValue(result);
      }

-      return observation[defaultSourceField];
+      return withValue(observation[defaultSourceField]);
+    }

    default:
-      return observation[defaultSourceField];
+      return withValue(observation[defaultSourceField]);
  }
 }

@@ -213,47 +251,45 @@ export function applyFullMapping(props: {
  const results: Record<string, unknown> = {};

  for (const field of fields) {
-    const onJsonPathMiss = (info: {
-      sourceField: SourceField;
-      jsonPath: string;
-      mappingKey: string | null;
-    }) => {
-      errors.push({
-        type: "json_path_miss",
-        targetField: field.key,
-        sourceField: info.sourceField,
-        jsonPath: info.jsonPath,
-        mappingKey: info.mappingKey,
-        message: `JSON path "${info.jsonPath}" did not match any data in "${info.sourceField}"${info.mappingKey ? ` (key: "${info.mappingKey}")` : ""}`,
-      });
-    };
-
    try {
-      results[field.key] = applyFieldMappingConfig({
+      const result = applyFieldMappingConfig({
        observation,
        config: field.config,
        defaultSourceField: field.defaultSourceField,
-        onJsonPathMiss,
      });
-    } catch (error) {
-      // Capture rare JSONPath evaluation errors (e.g. malformed filter expressions)
-      const sourceField =
-        field.config.mode === "custom" && field.config.custom?.type === "root"
-          ? (field.config.custom.rootConfig?.sourceField ??
-            field.defaultSourceField)
-          : field.defaultSourceField;
-      const jsonPath =
-        field.config.mode === "custom" && field.config.custom?.type === "root"
-          ? (field.config.custom.rootConfig?.jsonPath ?? "")
-          : "";
+      results[field.key] = result.value;

+      for (const miss of result.misses) {
+        errors.push({
+          type: "json_path_miss",
+          targetField: field.key,
+          sourceField: miss.sourceField,
+          jsonPath: miss.jsonPath,
+          mappingKey: miss.mappingKey,
+          message: `JSONPath "${miss.jsonPath}" did not match any data in "${miss.sourceField}"${miss.mappingKey ? ` (key: "${miss.mappingKey}")` : ""}`,
+        });
+      }
+
+      for (const err of result.errors) {
+        errors.push({
+          type: "json_path_error",
+          targetField: field.key,
+          sourceField: err.sourceField,
+          jsonPath: err.jsonPath,
+          mappingKey: err.mappingKey,
+          message: `JSONPath evaluation error for "${field.key}"${err.mappingKey ? ` (key: "${err.mappingKey}")` : ""}: ${err.message}`,
+        });
+      }
+    } catch (error) {
+      // Isolate per-field faults so a throw from one mapping doesn't abort
+      // the remaining fields on the same observation.
      errors.push({
        type: "json_path_error",
        targetField: field.key,
-        sourceField,
-        jsonPath,
+        sourceField: field.defaultSourceField,
+        jsonPath: "",
        mappingKey: null,
-        message: `JSON path evaluation error for "${field.key}": ${error instanceof Error ? error.message : "Unknown error"}`,
+        message: `JSONPath evaluation error for "${field.key}": ${error instanceof Error ? error.message : "Unknown error"}`,
      });
      results[field.key] = undefined;
    }
@@ -268,7 +304,7 @@ export function applyFullMapping(props: {
 }

 /**
- * Generate autocomplete suggestions for JSON paths based on the data structure
+ * Generate autocomplete suggestions for JSONPaths based on the data structure
 */
 export function generateJsonPathSuggestions(
  data: unknown,
--- a/Show More
+++ b/Show More