chore: release v3.92.0

* feat(llm-api-keys): add public api LIST and PATCH endpoints

* puhs

* push

* push

* add post endpoint

* add PUT endpoint

* push

* push

.devcontainer.json

@@ -1,6 +1,6 @@
 {
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+  "name": "langfuse-development",
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
 }

.env.dev.example

@@ -83,3 +83,8 @@ NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 # For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10
+
+# Slack credentials for development
+SLACK_CLIENT_ID=your_slack_client_id
+SLACK_CLIENT_SECRET=your_slack_client_secret
+SLACK_STATE_SECRET=your_slack_state_secret

.env.prod.example

@@ -172,6 +172,7 @@ OTEL_SERVICE_NAME="langfuse"
 # REDIS_HOST=
 # REDIS_PORT=
 # REDIS_AUTH=
+# REDIS_USERNAME=default
 # REDIS_CONNECTION_STRING=
 # REDIS_ENABLE_AUTO_PIPELINING=
 

.github/workflows/pipeline.yml

@@ -114,7 +114,9 @@ jobs:
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Build and run both images from compose
         run: |
-          docker compose -f docker-compose.build.yml up -d
+          docker compose --progress plain --verbose -f docker-compose.build.yml build --print > /tmp/bake.json
+          docker buildx bake -f /tmp/bake.json
+          docker compose --progress plain -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
       - name: Ensure no unhealthy status
         run: |
@@ -490,6 +492,14 @@ jobs:
         if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
+      - name: Notify Slack
+        uses: ravsamhq/notify-slack-action@v2
+        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   push-docker-image:
     needs: all-ci-passed

.github/workflows/trivy.yml

@@ -1,43 +0,0 @@
-name: Trivy Container Security
-on:
-  push:
-    branches: ["main"]
-
-jobs:
-  trivy-web:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Run Trivy vulnerability scanner (web)
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "langfuse/langfuse"
-          format: "sarif"
-          output: "trivy-web.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab (web)
-        if: always() && hashFiles('trivy-web.sarif') != ''
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "trivy-web.sarif"
-          category: trivy-web
-
-  trivy-worker:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Run Trivy vulnerability scanner (worker)
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "langfuse/langfuse-worker"
-          format: "sarif"
-          output: "trivy-worker.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab (worker)
-        if: always() && hashFiles('trivy-worker.sarif') != ''
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "trivy-worker.sarif"
-          category: trivy-worker

.gitignore

@@ -50,7 +50,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 
-/generated/typescript-server
+/generated
 
 # openapi spec that is copied during build
 /public/openapi*.yml

CLAUDE.md

@@ -192,3 +192,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## General Coding Guidelines
 - For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so
+
+## Development Tips
+- Before trying to build the package, try running the linter once first

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     ports:
       - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
       DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
       SALT: "mysalt" # CHANGEME
       ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
@@ -73,7 +74,6 @@ services:
       - 3000:3000
     environment:
       <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
       NEXTAUTH_SECRET: mysecret # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}

fern/apis/server/definition/llm-connections.yml

@@ -0,0 +1,102 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    list:
+      method: GET
+      docs: Get all LLM connections in a project
+      path: /llm-connections
+      request:
+        name: GetLlmConnectionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedLlmConnections
+    upsert:
+      method: PUT
+      docs: Create or update an LLM connection (upsert)
+      path: /llm-connections
+      request: UpsertLlmConnectionRequest
+      response: LlmConnection
+
+types:
+  LlmConnection:
+    docs: LLM API connection configuration (secrets excluded)
+    properties:
+      id: string
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'anthropic')
+      adapter:
+        type: string
+        docs: The adapter used to interface with the LLM
+      displaySecretKey:
+        type: string
+        docs: Masked version of the secret key for display purposes
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: list<string>
+        docs: List of custom model names available for this connection
+      withDefaultModels:
+        type: boolean
+        docs: Whether to include default models for this adapter
+      extraHeaderKeys:
+        type: list<string>
+        docs: Keys of extra headers sent with requests (values excluded for security)
+      createdAt: datetime
+      updatedAt: datetime
+
+  PaginatedLlmConnections:
+    properties:
+      data: list<LlmConnection>
+      meta: pagination.MetaResponse
+
+  UpsertLlmConnectionRequest:
+    docs: Request to create or update an LLM connection (upsert)
+    properties:
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'anthropic'). Must be unique in project.
+      adapter:
+        type: LlmAdapter
+        docs: The adapter used to interface with the LLM
+      secretKey:
+        type: string
+        docs: Secret key for the LLM API.
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: optional<list<string>>
+        docs: List of custom model names
+      withDefaultModels:
+        type: optional<boolean>
+        docs: Whether to include default models. Default is true.
+      extraHeaders:
+        type: optional<map<string, string>>
+        docs: Extra headers to send with requests
+
+  LlmAdapter:
+    enum:
+      - value: anthropic
+        name: Anthropic
+      - value: openai
+        name: OpenAI
+      - value: azure
+        name: Azure
+      - value: bedrock
+        name: Bedrock
+      - value: google-vertex-ai
+        name: GoogleVertexAI
+      - value: google-ai-studio
+        name: GoogleAIStudio

fern/apis/server/generators.yml

@@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://schema.buildwithfern.dev/generators-yml.json
 default-group: local
 groups:
   local:
@@ -7,6 +8,7 @@ groups:
         output:
           location: local-file-system
           path: ../../../web/public/generated/api
+
       - name: fernapi/fern-python-sdk
         version: 2.16.0
         output:
@@ -19,35 +21,32 @@ groups:
           pydantic_config:
             require_optional_fields: false
             use_str_enums: false
-#      - name: fernapi/fern-java-sdk
-#        version: 2.20.1
-#        output:
-#          location: local-file-system
-#          path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
-#        config:
-#          client-class-name: LangfuseClient
+
+      # - name: fernapi/fern-java-sdk
+      #   version: 2.20.1
+      #   output:
+      #     location: local-file-system
+      #     path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
+      #   config:
+      #     client-class-name: LangfuseClient
+
+      - name: fernapi/fern-typescript-node-sdk
+        version: 2.6.1
+        output:
+          location: local-file-system
+          path: ../../../generated/typescript
+        config:
+          namespaceExport: LangfuseAPI
+          outputSourceFiles: true
+          skipResponseValidation: true
+          fetchSupport: native
+          formDataSupport: Node18
+          fileResponseType: binary-response
+          streamType: web
+          omitFernHeaders: true
+
       - name: fernapi/fern-postman
         version: 0.0.45
         output:
           location: local-file-system
           path: ../../../web/public/generated/postman
-  # published:
-  #   generators:
-  #     - name: fernapi/fern-python-sdk
-  #       version: 0.3.7
-  #       output:
-  #         location: pypi
-  #         url: pypi.buildwithfern.com
-  #         package-name: finto-fern-langfuse
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true
-  #     - name: fernapi/fern-typescript-node-sdk
-  #       version: 0.7.1
-  #       output:
-  #         location: npm
-  #         url: npm.buildwithfern.com
-  #         package-name: "@finto-fern/langfuse-node"
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true

generated/.gitignore

@@ -1 +0,0 @@
-python/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.85.2",
+  "version": "3.92.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -40,7 +40,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items ON CLUSTER default (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/package.json

@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -73,6 +73,8 @@
     "@prisma/client": "^6.10.1",
     "@react-email/components": "^0.1.0",
     "@react-email/render": "^1.1.2",
+    "@slack/oauth": "^2.6.0",
+    "@slack/web-api": "^7.0.0",
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",

packages/shared/prisma/generated/types.ts

@@ -55,6 +55,7 @@ export type AnnotationQueueStatus =
 export const AnnotationQueueObjectType = {
   TRACE: "TRACE",
   OBSERVATION: "OBSERVATION",
+  SESSION: "SESSION",
 } as const;
 export type AnnotationQueueObjectType =
   (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
@@ -138,6 +139,7 @@ export type DashboardWidgetChartType =
   (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export const ActionType = {
   WEBHOOK: "WEBHOOK",
+  SLACK: "SLACK",
 } as const;
 export type ActionType = (typeof ActionType)[keyof typeof ActionType];
 export const ActionExecutionStatus = {
@@ -359,6 +361,8 @@ export type Dataset = {
   name: string;
   description: string | null;
   metadata: unknown | null;
+  remote_experiment_url: string | null;
+  remote_experiment_payload: unknown | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -624,6 +628,15 @@ export type OrganizationMembership = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type PendingDeletion = {
+  id: string;
+  project_id: string;
+  object: string;
+  object_id: string;
+  is_deleted: Generated<boolean>;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
   project_id: string;
   encrypted_posthog_api_key: string;
@@ -711,6 +724,16 @@ export type Session = {
   user_id: string;
   expires: Timestamp;
 };
+export type SlackIntegration = {
+  id: string;
+  project_id: string;
+  team_id: string;
+  team_name: string;
+  bot_token: string;
+  bot_user_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type SsoConfig = {
   domain: string;
   created_at: Generated<Timestamp>;
@@ -813,6 +836,7 @@ export type DB = {
   observations: LegacyPrismaObservation;
   organization_memberships: OrganizationMembership;
   organizations: Organization;
+  pending_deletions: PendingDeletion;
   posthog_integrations: PosthogIntegration;
   prices: Price;
   project_memberships: ProjectMembership;
@@ -823,6 +847,7 @@ export type DB = {
   score_configs: ScoreConfig;
   scores: LegacyPrismaScore;
   Session: Session;
+  slack_integrations: SlackIntegration;
   sso_configs: SsoConfig;
   table_view_presets: TableViewPreset;
   trace_media: TraceMedia;

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/migrations/20250730100100_add_slack_integration/migration.sql

@@ -0,0 +1,30 @@
+-- Migration: Add Slack Integration Support
+-- This migration adds support for Slack automation actions by:
+-- 1. Adding SLACK to the ActionType enum
+-- 2. Creating slack_integrations table for centralized token storage
+
+-- AlterEnum
+ALTER TYPE "ActionType" ADD VALUE 'SLACK';
+
+-- CreateTable
+CREATE TABLE "slack_integrations" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "team_name" TEXT NOT NULL,
+    "bot_token" TEXT NOT NULL,
+    "bot_user_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "slack_integrations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "slack_integrations_project_id_key" ON "slack_integrations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "slack_integrations_team_id_idx" ON "slack_integrations"("team_id");
+
+-- AddForeignKey
+ALTER TABLE "slack_integrations" ADD CONSTRAINT "slack_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; 

packages/shared/prisma/migrations/20250731100100_add_dataset_run_items_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('8d47f91b-3e5c-4a26-9f85-c12d6e4b9a3d', '20250731_1001_migrate_dataset_run_items_pg_to_ch', 'migrateDatasetRunItemsFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20250731202005_add_trace_deletion_table/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "pending_deletions" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "is_deleted" BOOLEAN NOT NULL DEFAULT false,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "pending_deletions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_project_id_object_is_deleted_idx" ON "pending_deletions"("project_id", "object", "is_deleted");
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_object_id_object_idx" ON "pending_deletions"("object_id", "object");
+
+-- AddForeignKey
+ALTER TABLE "pending_deletions" ADD CONSTRAINT "pending_deletions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -163,6 +163,8 @@ model Project {
   Automation             Automation[]
   DefaultLlmModel        DefaultLlmModel[]
   Price                  Price[]
+  SlackIntegration       SlackIntegration?
+  PendingDeletion        PendingDeletion[]
 
   @@index([orgId])
   @@map("projects")
@@ -539,6 +541,7 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
 }
 
 model CronJobs {
@@ -551,16 +554,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -1295,6 +1300,7 @@ model Automation {
 
 enum ActionType {
   WEBHOOK
+  SLACK
   // More action types can be added as needed
 }
 
@@ -1335,3 +1341,41 @@ model AutomationExecution {
   @@index([projectId])
   @@map("automation_executions")
 }
+
+// Slack Integration: Stores centralized Slack workspace connection for each project
+// One project can connect to one Slack workspace, supporting multiple channel automations
+model SlackIntegration {
+  id        String  @id @default(cuid())
+  projectId String  @unique @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // Installation details (encrypted using shared encryption utilities)
+  teamId    String @map("team_id") // Slack workspace ID
+  teamName  String @map("team_name") // Human-readable workspace name
+  botToken  String @map("bot_token") // Encrypted bot token for API calls
+  botUserId String @map("bot_user_id") // Bot user ID for workspace
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([teamId])
+  @@map("slack_integrations")
+}
+
+// Pending Deletions: Tracks objects (like traces) that are scheduled for batch deletion
+model PendingDeletion {
+  id        String  @id @default(cuid())
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  object    String  @map("object") // e.g., "trace", "observation", etc.
+  objectId  String  @map("object_id") // The ID of the object to be deleted
+  isDeleted Boolean @default(false) @map("is_deleted")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, object, isDeleted])
+  @@index([objectId, object])
+  @@map("pending_deletions")
+}

packages/shared/scripts/seeder/utils/markdown.txt

@@ -11,29 +11,29 @@
 ## 🎯 Getting Started
 
 ### Prerequisites
-\`\`\`bash
-npm install @langfuse/core
-pip install langfuse
+\`\`\`bash  
+npm install @langfuse/core  
+pip install langfuse  
 \`\`\`
 
 ### Quick Setup
 1. **Initialize your project**
-   \`\`\`typescript
-   import { Langfuse } from 'langfuse'
-   
-   const langfuse = new Langfuse({
-     secretKey: process.env.LANGFUSE_SECRET_KEY,
-     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-     baseUrl: 'https://cloud.langfuse.com'
-   })
+   \`\`\`typescript  
+   import { Langfuse } from 'langfuse'  
+
+   const langfuse = new Langfuse({  
+     secretKey: process.env.LANGFUSE_SECRET_KEY,  
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,  
+     baseUrl: 'https://cloud.langfuse.com'  
+   })  
    \`\`\`
 
 2. **Create your first trace**
-   \`\`\`python
-   from langfuse import Langfuse
-   
-   langfuse = Langfuse()
-   trace = langfuse.trace(name="chat-application")
+   \`\`\`python  
+   from langfuse import Langfuse  
+
+   langfuse = Langfuse()  
+   trace = langfuse.trace(name="chat-application")  
    \`\`\`
 
 ---
@@ -72,75 +72,75 @@ graph TD
 > **Note:** Traces are the foundation of observability in LLM applications.
 
 #### Creating Traces
-\`\`\`typescript
-// Basic trace creation
-const trace = langfuse.trace({
-  name: "user-query-processing",
-  userId: "user-123",
-  sessionId: "session-456",
-  metadata: {
-    environment: "production",
-    version: "2.1.0"
-  }
-})
+\`\`\`typescript  
+// Basic trace creation  
+const trace = langfuse.trace({  
+  name: "user-query-processing",  
+  userId: "user-123",  
+  sessionId: "session-456",  
+  metadata: {  
+    environment: "production",  
+    version: "2.1.0"  
+  }  
+})  
 
-// Nested observations
-const span = trace.span({
-  name: "document-retrieval",
-  input: { query: "What is machine learning?" },
-  metadata: { vectorStore: "pinecone" }
-})
+// Nested observations  
+const span = trace.span({  
+  name: "document-retrieval",  
+  input: { query: "What is machine learning?" },  
+  metadata: { vectorStore: "pinecone" }  
+})  
 
-const generation = span.generation({
-  name: "answer-generation",
-  model: "gpt-4",
-  input: retrievedDocs,
-  output: generatedAnswer,
-  usage: {
-    promptTokens: 1250,
-    completionTokens: 420,
-    totalTokens: 1670
-  }
-})
+const generation = span.generation({  
+  name: "answer-generation",  
+  model: "gpt-4",  
+  input: retrievedDocs,  
+  output: generatedAnswer,  
+  usage: {  
+    promptTokens: 1250,  
+    completionTokens: 420,  
+    totalTokens: 1670  
+  }  
+})  
 \`\`\`
 
 ### Advanced Features
 
 #### 🔄 Async Processing
-\`\`\`python
-import asyncio
-from langfuse import Langfuse
+\`\`\`python  
+import asyncio  
+from langfuse import Langfuse  
 
-async def process_batch():
-    langfuse = Langfuse()
-    
-    tasks = []
-    for item in batch_items:
-        task = asyncio.create_task(
-            process_item_with_tracing(langfuse, item)
-        )
-        tasks.append(task)
-    
-    results = await asyncio.gather(*tasks)
-    return results
+async def process_batch():  
+    langfuse = Langfuse()  
+
+    tasks = []  
+    for item in batch_items:  
+        task = asyncio.create_task(  
+            process_item_with_tracing(langfuse, item)  
+        )  
+        tasks.append(task)  
+
+    results = await asyncio.gather(*tasks)  
+    return results  
 \`\`\`
 
 #### 🎯 Custom Scoring
-\`\`\`typescript
-// Automated scoring
-trace.score({
-  name: "relevance",
-  value: 0.95,
-  comment: "Highly relevant response"
-})
+\`\`\`typescript  
+// Automated scoring  
+trace.score({  
+  name: "relevance",  
+  value: 0.95,  
+  comment: "Highly relevant response"  
+})  
 
-// Human feedback scoring
-trace.score({
-  name: "user-satisfaction", 
-  value: 1,
-  source: "user-feedback",
-  comment: "User rated 5/5 stars"
-})
+// Human feedback scoring  
+trace.score({  
+  name: "user-satisfaction",  
+  value: 1,  
+  source: "user-feedback",  
+  comment: "User rated 5/5 stars"  
+})  
 \`\`\`
 
 ---
@@ -156,20 +156,20 @@ trace.score({
 - **User Satisfaction**: Quality metrics
 
 #### Dashboard Setup
-\`\`\`yaml
-# monitoring-config.yml
-dashboards:
-  - name: "LLM Performance"
-    panels:
-      - type: "time-series"
-        title: "Response Latency"
-        query: "avg(response_time) by (model)"
-      - type: "stat"
-        title: "Daily Token Usage" 
-        query: "sum(tokens_used)"
-      - type: "table"
-        title: "Top Errors"
-        query: "topk(10, count by (error_type))"
+\`\`\`yaml  
+# monitoring-config.yml  
+dashboards:  
+  - name: "LLM Performance"  
+    panels:  
+      - type: "time-series"  
+        title: "Response Latency"  
+        query: "avg(response_time) by (model)"  
+      - type: "stat"  
+        title: "Daily Token Usage"  
+        query: "sum(tokens_used)"  
+      - type: "table"  
+        title: "Top Errors"  
+        query: "topk(10, count by (error_type))"  
 \`\`\`
 
 ### 🔐 Security Considerations
@@ -177,40 +177,40 @@ dashboards:
 > ⚠️ **Important**: Never log sensitive user data in traces
 
 #### Data Sanitization
-\`\`\`python
-def sanitize_input(data):
-    """Remove PII from trace data"""
-    sanitized = data.copy()
-    
-    # Remove email addresses
-    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
-                      '[EMAIL_REDACTED]', sanitized)
-    
-    # Remove phone numbers
-    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
-                      '[PHONE_REDACTED]', sanitized)
-    
-    return sanitized
+\`\`\`python  
+def sanitize_input(data):  
+    """Remove PII from trace data"""  
+    sanitized = data.copy()  
+
+    # Remove email addresses  
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b',  
+                      '[EMAIL_REDACTED]', sanitized)  
+
+    # Remove phone numbers  
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b',  
+                      '[PHONE_REDACTED]', sanitized)  
+
+    return sanitized  
 \`\`\`
 
 ### 🚀 Performance Optimization
 
 #### Batch Processing
-\`\`\`typescript
-// Efficient batch uploads
-const batchSize = 100
-const traces = []
+\`\`\`typescript  
+// Efficient batch uploads  
+const batchSize = 100  
+const traces = []  
 
-for (let i = 0; i < data.length; i += batchSize) {
-  const batch = data.slice(i, i + batchSize)
-  const processedBatch = await Promise.all(
-    batch.map(item => processWithLangfuse(item))
-  )
-  traces.push(...processedBatch)
-}
+for (let i = 0; i < data.length; i += batchSize) {  
+  const batch = data.slice(i, i + batchSize)  
+  const processedBatch = await Promise.all(  
+    batch.map(item => processWithLangfuse(item))  
+  )  
+  traces.push(...processedBatch)  
+}  
 
-// Flush all traces at once
-await langfuse.flushAsync()
+// Flush all traces at once  
+await langfuse.flushAsync()  
 \`\`\`
 
 ---
@@ -219,34 +219,34 @@ await langfuse.flushAsync()
 
 ### Multi-Agent System Tracing
 \`\`\`python
-class MultiAgentTracer:
-    def __init__(self):
-        self.langfuse = Langfuse()
-    
-    async def orchestrate_agents(self, task):
-        # Main orchestration trace
-        main_trace = self.langfuse.trace(
-            name="multi-agent-orchestration",
-            input={"task": task}
-        )
-        
-        # Agent 1: Research
-        research_span = main_trace.span(name="research-agent")
-        research_result = await self.research_agent.process(task)
-        research_span.end(output=research_result)
-        
+class MultiAgentTracer:  
+    def __init__(self):  
+        self.langfuse = Langfuse()  
+
+    async def orchestrate_agents(self, task):  
+        # Main orchestration trace  
+        main_trace = self.langfuse.trace(  
+            name="multi-agent-orchestration",  
+            input={"task": task}  
+        )  
+
+        # Agent 1: Research  
+        research_span = main_trace.span(name="research-agent")  
+        research_result = await self.research_agent.process(task)  
+        research_span.end(output=research_result)  
+
         # Agent 2: Analysis  
-        analysis_span = main_trace.span(name="analysis-agent")
-        analysis_result = await self.analysis_agent.process(research_result)
-        analysis_span.end(output=analysis_result)
-        
-        # Agent 3: Synthesis
-        synthesis_span = main_trace.span(name="synthesis-agent")
-        final_result = await self.synthesis_agent.process(analysis_result)
-        synthesis_span.end(output=final_result)
-        
-        main_trace.end(output=final_result)
-        return final_result
+        analysis_span = main_trace.span(name="analysis-agent")  
+        analysis_result = await self.analysis_agent.process(research_result)  
+        analysis_span.end(output=analysis_result)  
+
+        # Agent 3: Synthesis  
+        synthesis_span = main_trace.span(name="synthesis-agent")  
+        final_result = await self.synthesis_agent.process(analysis_result)  
+        synthesis_span.end(output=final_result)  
+
+        main_trace.end(output=final_result)  
+        return final_result  
 \`\`\`
 
 ---
@@ -256,11 +256,11 @@ class MultiAgentTracer:
 With proper implementation of Langfuse tracing, you can:
 
 - ✅ **Monitor** your LLM applications in real-time
-- ✅ **Debug** issues with detailed trace information  
+- ✅ **Debug** issues with detailed trace information
 - ✅ **Optimize** performance and costs
 - ✅ **Scale** your applications with confidence
 
 ### Next Steps
 1. Review the [official documentation](https://langfuse.com/docs)
 2. Join our [Discord community](https://discord.gg/langfuse)
-3. Check out [example projects](https://github.com/langfuse/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/src/domain/automations.ts

@@ -30,14 +30,14 @@ export type AutomationDomain = {
 };
 
 export type ActionDomain = Omit<Action, "config"> & {
-  config: SafeWebhookActionConfig;
+  config: SafeActionConfig;
 };
 
 export type ActionDomainWithSecrets = Omit<Action, "config"> & {
-  config: WebhookActionConfigWithSecrets;
+  config: ActionConfigWithSecrets;
 };
 
-export const ActionTypeSchema = z.enum(["WEBHOOK"]);
+export const ActionTypeSchema = z.enum(["WEBHOOK", "SLACK"]);
 
 export const AvailableWebhookApiSchema = z.record(
   z.enum(["prompt"]),
@@ -52,12 +52,13 @@ export const RequestHeaderSchema = z.object({
 export const WebhookActionConfigSchema = z.object({
   type: z.literal("WEBHOOK"),
   url: z.url(),
-  headers: z.record(z.string(), z.string()),
-  requestHeaders: z.record(z.string(), RequestHeaderSchema),
-  displayHeaders: z.record(z.string(), RequestHeaderSchema),
+  headers: z.record(z.string(), z.string()).optional(), // deprecated field, use requestHeaders instead
+  requestHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
+  displayHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
   apiVersion: AvailableWebhookApiSchema,
   secretKey: z.string(),
   displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
 });
 
 export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
@@ -77,18 +78,94 @@ export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
   displayHeaders: true,
 });
 
+export const SlackActionConfigSchema = z.object({
+  type: z.literal("SLACK"),
+  channelId: z.string(),
+  channelName: z.string(),
+  messageTemplate: z.string().optional(),
+});
+
+export type SlackActionConfig = z.infer<typeof SlackActionConfigSchema>;
+
 export const ActionConfigSchema = z.discriminatedUnion("type", [
   WebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export const ActionCreateSchema = z.discriminatedUnion("type", [
   WebhookActionCreateSchema,
+  SlackActionConfigSchema,
+]);
+
+export const SafeActionConfigSchema = z.discriminatedUnion("type", [
+  SafeWebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export type ActionTypes = z.infer<typeof ActionTypeSchema>;
 export type ActionConfig = z.infer<typeof ActionConfigSchema>;
 export type ActionCreate = z.infer<typeof ActionCreateSchema>;
+export type SafeActionConfig = z.infer<typeof SafeActionConfigSchema>;
 
+export type WebhookActionCreate = z.infer<typeof WebhookActionCreateSchema>;
 export type WebhookActionConfigWithSecrets = z.infer<
   typeof WebhookActionConfigSchema
 >;
+
+export type ActionConfigWithSecrets = z.infer<typeof ActionConfigSchema>;
+
+// Type Guards for Runtime Validation
+// Using existing Zod schemas to provide both compile-time and runtime type safety
+
+/**
+ * Type guard to check if a config is a valid webhook configuration with secrets
+ */
+export function isWebhookActionConfig(
+  config: unknown,
+): config is WebhookActionConfigWithSecrets {
+  return WebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if a config is a valid Slack configuration
+ */
+export function isSlackActionConfig(
+  config: unknown,
+): config is SlackActionConfig {
+  return SlackActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if an entire action has valid webhook configuration
+ */
+export function isWebhookAction(action: {
+  type: string;
+  config: unknown;
+}): action is { type: "WEBHOOK"; config: WebhookActionConfigWithSecrets } {
+  return action.type === "WEBHOOK" && isWebhookActionConfig(action.config);
+}
+
+/**
+ * Type guard for safe webhook config (without secrets)
+ */
+export function isSafeWebhookActionConfig(
+  config: unknown,
+): config is SafeWebhookActionConfig {
+  return SafeWebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Converts webhook config with secrets to safe config by only including allowed fields
+ */
+export function convertToSafeWebhookConfig(
+  webhookConfig: WebhookActionConfigWithSecrets,
+): SafeWebhookActionConfig {
+  return {
+    type: webhookConfig.type,
+    url: webhookConfig.url,
+    displayHeaders: webhookConfig.displayHeaders,
+    apiVersion: webhookConfig.apiVersion,
+    displaySecretKey: webhookConfig.displaySecretKey,
+    lastFailingExecutionId: webhookConfig.lastFailingExecutionId,
+  };
+}

packages/shared/src/env.ts

@@ -15,6 +15,7 @@ const EnvSchema = z.object({
     .default(6379)
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
+  REDIS_USERNAME: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
   REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
@@ -49,6 +50,10 @@ const EnvSchema = z.object({
     .nonnegative()
     .default(15_000),
   LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
+  LANGFUSE_TRACE_DELETE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(5_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -140,6 +145,45 @@ const EnvSchema = z.object({
   LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
+
+  SLACK_CLIENT_ID: z.string().optional(),
+  SLACK_CLIENT_SECRET: z.string().optional(),
+  SLACK_STATE_SECRET: z.string().optional(),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,8 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
+  "observation-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/index.ts

@@ -18,6 +18,7 @@ export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
 export * from "./domain/webhooks";
+export * from "./domain/dataset-run-items";
 
 // llm api
 export * from "./server/llm/types";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -18,6 +18,21 @@ export const CloudConfigSchema = z.object({
 
   // custom rate limits for an organization
   rateLimitOverrides: CloudConfigRateLimit.optional(),
+
+  // billing alert configuration
+  usageAlerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      type: z.enum(["STRIPE"]).default("STRIPE"),
+      threshold: z.number().int().positive(),
+      alertId: z.string(), // Alert ID for tracking
+      meterId: z.string(), // Meter ID for usage tracking
+      notifications: z.object({
+        email: z.boolean().default(true),
+        recipients: z.array(z.string().email()).default([]),
+      }),
+    })
+    .optional(),
 });
 
 export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/server/auth/types.ts

@@ -1,7 +1,7 @@
 import z from "zod/v4";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
-import { ApiKeyScope } from "../../";
+import { ApiKeyScope, MakeOptional } from "../../";
 
 const ApiKeyBaseSchema = z.object({
   id: z.string(),
@@ -48,12 +48,25 @@ export type AuthHeaderValidVerificationResult = {
   scope: ApiAccessScope;
 };
 
-export type ApiAccessScope = {
+export type AuthHeaderValidVerificationResultIngestion = {
+  validKey: true;
+  scope: ApiAccessScopeIngestion;
+};
+
+type BaseApiAccessScope = {
   projectId: string | null;
   accessLevel: "organization" | "project" | "scores";
+};
+
+type ApiAccessScopeMetadata = {
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
   apiKeyId: string;
   publicKey: string;
 };
+
+export type ApiAccessScopeIngestion = BaseApiAccessScope &
+  MakeOptional<ApiAccessScopeMetadata>;
+
+export type ApiAccessScope = BaseApiAccessScope & ApiAccessScopeMetadata;

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -1,4 +1,4 @@
-import { instrumentAsync } from "../instrumentation";
+import { instrumentAsync, recordDistribution } from "../instrumentation";
 import * as opentelemetry from "@opentelemetry/api";
 import { env } from "../../env";
 import { logger } from "../logger";
@@ -39,7 +39,9 @@ export const measureAndReturn = async <T, Y>(args: {
         "true"
       ) {
         currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
-        return existingExecution(input);
+        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
+          ? newExecution(input)
+          : existingExecution(input);
       }
 
       // If not whitelisted, apply sampling logic
@@ -68,6 +70,14 @@ export const measureAndReturn = async <T, Y>(args: {
           durationDifference,
         );
 
+        recordDistribution(
+          "langfuse.experiment.amts.duration_difference_distribution",
+          durationDifference,
+          {
+            operation: args.operationName,
+          },
+        );
+
         if (
           env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
             (p) => p === args.projectId,

packages/shared/src/server/clickhouse/schema.ts

@@ -2,6 +2,7 @@ export const ClickhouseTableNames = {
   traces: "traces",
   observations: "observations",
   scores: "scores",
+  dataset_run_items: "dataset_run_items",
 
   // Virtual tables for dashboards
   // TODO: Check if we can do this more elegantly

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/index.ts

@@ -2,6 +2,7 @@ export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
+export * from "./services/email/billingAlert/sendBillingAlertEmail";
 export * from "./services/PromptService";
 export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
@@ -32,6 +33,7 @@ export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -63,13 +65,17 @@ export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
 export * from "./clickhouse/measureAndReturn";
+export * from "./services/SlackService";
+
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
 // dataset run items
 export * from "./dataset-run-items/datasetExecution";
 export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
 
 // test utils
 export * from "./test-utils";
 export * from "./utils/headerUtils";
+export * from "./traceDeletionProcessor";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -83,7 +83,7 @@ const getModelFromRedis = async (
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -8,7 +8,7 @@ import {
   LangfuseNotFoundError,
   UnauthorizedError,
 } from "../../errors";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { AuthHeaderValidVerificationResultIngestion } from "../auth/types";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import {
   getCurrentSpan,
@@ -30,6 +30,7 @@ import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -61,7 +62,7 @@ export type TokenCountDelegate = (p: {
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -73,6 +74,10 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
@@ -94,12 +99,12 @@ type ProcessEventBatchOptions = {
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
- * @param authCheck - AuthHeaderValidVerificationResult
+ * @param authCheck - AuthHeaderValidVerificationResultIngestion
  * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
-  authCheck: AuthHeaderValidVerificationResult,
+  authCheck: AuthHeaderValidVerificationResultIngestion,
   options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
@@ -124,8 +129,10 @@ export const processEventBatch = async (
     "langfuse.project.id",
     authCheck.scope.projectId ?? "",
   );
-  currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
-  currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
+  if (authCheck.scope.orgId)
+    currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
+  if (authCheck.scope.plan)
+    currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
@@ -269,6 +276,27 @@ export const processEventBatch = async (
       const shouldSkipS3List =
         isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
 
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
+
       return queue
         ? queue.add(
             QueueJobs.IngestionJob,
@@ -292,7 +320,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -307,7 +335,7 @@ export const processEventBatch = async (
 
 const isAuthorized = (
   event: IngestionEventType,
-  authScope: AuthHeaderValidVerificationResult,
+  authScope: AuthHeaderValidVerificationResultIngestion,
 ): boolean => {
   if (event.type === eventTypes.SDK_LOG) {
     return true;

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -316,22 +316,6 @@ export async function fetchLLMCompletion(
       maxRetries,
       apiKey,
     });
-  } else if (modelParams.adapter === LLMAdapter.Atla) {
-    // Atla models do not support:
-    // - temperature
-    // - max_tokens
-    // - top_p
-    chatModel = new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      callbacks: finalCallbacks,
-      maxRetries,
-      configuration: {
-        baseURL: baseURL,
-        defaultHeaders: extraHeaders,
-      },
-      timeout: 1000 * 60, // 1 minute timeout
-    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;

packages/shared/src/server/llm/types.ts

@@ -113,6 +113,7 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
   Tool = "tool",
+  Model = "model", // Google Gemini assistant format
 }
 
 // Thought: should placeholder not semantically be part of this, because it can be
@@ -124,6 +125,7 @@ export enum ChatMessageType {
   AssistantText = "assistant-text",
   AssistantToolCall = "assistant-tool-call",
   ToolResult = "tool-result",
+  ModelText = "model-text",
   PublicAPICreated = "public-api-created",
   Placeholder = "placeholder",
 }
@@ -156,6 +158,13 @@ export const AssistantTextMessageSchema = z.object({
 });
 export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
 
+export const ModelMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ModelText),
+  role: z.literal(ChatMessageRole.Model),
+  content: z.string(),
+});
+export type ModelMessage = z.infer<typeof ModelMessageSchema>;
+
 export const AssistantToolCallMessageSchema = z.object({
   type: z.literal(ChatMessageType.AssistantToolCall),
   role: z.literal(ChatMessageRole.Assistant),
@@ -193,6 +202,7 @@ export const ChatMessageSchema = z.union([
   AssistantTextMessageSchema,
   AssistantToolCallMessageSchema,
   ToolResultMessageSchema,
+  ModelMessageSchema,
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
@@ -226,7 +236,6 @@ export type PromptVariable = { name: string; value: string; isUsed: boolean };
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
-  Atla = "atla",
   Azure = "azure",
   Bedrock = "bedrock",
   VertexAI = "google-vertex-ai",
@@ -368,8 +377,6 @@ export const googleAIStudioModels = [
   "gemini-1.5-flash-8b",
 ] as const;
 
-export const atlaModels = ["atla-selene", "atla-selene-20250214"] as const;
-
 export type AnthropicModel = (typeof anthropicModels)[number];
 export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
@@ -379,7 +386,6 @@ export const supportedModels = {
   [LLMAdapter.GoogleAIStudio]: googleAIStudioModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
-  [LLMAdapter.Atla]: atlaModels,
 } as const;
 
 export type LLMFunctionCall = {

packages/shared/src/server/queries/createSessionsAllQuery.ts

@@ -1,88 +0,0 @@
-import { z } from "zod/v4";
-
-import { Prisma } from "@prisma/client";
-import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
-import { singleFilter } from "../../interfaces/filters";
-import { orderBy } from "../../interfaces/orderBy";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { sessionsViewCols } from "../../tableDefinitions";
-
-const GetSessionTableSQLParamsSchema = z.object({
-  projectId: z.string(),
-  filter: z.array(singleFilter).nullable(),
-  orderBy: orderBy,
-  page: z.number(),
-  limit: z.number(),
-});
-type GetSessionTableSQLParams = z.infer<typeof GetSessionTableSQLParamsSchema>;
-
-export const createSessionsAllQuery = (
-  select: Prisma.Sql,
-  params: GetSessionTableSQLParams,
-  options?: {
-    ignoreOrderBy?: boolean; // used by session.metrics and session.all.totalCount
-    sessionIdList?: string[]; // used by session.metrics
-  },
-): Prisma.Sql => {
-  const { projectId, filter, orderBy, page, limit } =
-    GetSessionTableSQLParamsSchema.parse(params);
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filter ?? [],
-    sessionsViewCols,
-    "sessions",
-  );
-  const orderByCondition = orderByToPrismaSql(orderBy, sessionsViewCols);
-
-  const sessionIdFilter = options?.sessionIdList
-    ? Prisma.sql`AND s.id IN (${Prisma.join(options?.sessionIdList)})`
-    : Prisma.sql``;
-
-  const sql = Prisma.sql`
-      SELECT
-        ${select}
-      FROM
-        trace_sessions AS s
-        LEFT JOIN LATERAL (
-          SELECT
-            t.session_id,
-            MAX(t. "timestamp") AS "max_timestamp",
-            MIN(t. "timestamp") AS "min_timestamp",
-            array_agg(t.id) AS "traceIds",
-            array_agg(DISTINCT t.user_id) AS "userIds",
-            count(t.id)::int AS "countTraces",
-            array_agg(DISTINCT u.tag) AS "tags"
-          FROM
-            traces t
-          LEFT JOIN LATERAL (
-            SELECT DISTINCT UNNEST(t.tags) AS tag) AS u ON TRUE
-          WHERE
-            t.project_id = ${projectId}
-            AND t.session_id = s.id
-          GROUP BY
-            t.session_id) AS t ON TRUE
-        LEFT JOIN LATERAL (
-          SELECT
-            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
-            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
-            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
-            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
-            SUM(o.prompt_tokens) AS "promptTokens",
-            SUM(o.completion_tokens) AS "completionTokens",
-            SUM(o.total_tokens) AS "totalTokens"
-          FROM
-            observations_view o
-          WHERE
-            o.project_id = ${projectId}
-            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
-      WHERE
-        s. "project_id" = ${projectId}
-        ${filterCondition}
-        ${sessionIdFilter}
-      ${options?.ignoreOrderBy ? Prisma.sql`` : orderByCondition}
-      LIMIT ${limit}
-      OFFSET ${page * limit}
-    `;
-
-  return sql;
-};

packages/shared/src/server/queries/index.ts

@@ -1,4 +1,3 @@
-export { createSessionsAllQuery } from "./createSessionsAllQuery";
 export {
   type FullObservations,
   type FullObservationsWithScores,

packages/shared/src/server/queues.ts

@@ -40,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -101,6 +116,24 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
+    z.object({
+      actionId: z.literal("observation-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -144,6 +177,7 @@ export const WebhookInputSchema = z.object({
   payload: WebhookOutboundEnvelopeSchema,
 });
 
+export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   z.object({
     entityType: z.literal("prompt-version"),
@@ -154,8 +188,6 @@ export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   }),
   // Add other entity types here in the future
 ]);
-
-export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export type EntityChangeEventType = z.infer<typeof EntityChangeEventSchema>;
 
 export type CreateEvalQueueEventType = z.infer<
@@ -165,6 +197,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -192,6 +225,13 @@ export type DeadLetterRetryQueueEventType = z.infer<
 
 export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
 
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -214,6 +254,7 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
   WebhookQueue = "webhook-queue",
   EntityChangeQueue = "entity-change-queue",
@@ -241,6 +282,7 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
   WebhookJob = "webhook-job",
   EntityChangeJob = "entity-change-job",
@@ -265,6 +307,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -282,6 +330,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -306,6 +355,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -34,7 +34,7 @@ export class ExperimentCreateQueue {
               attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 1000,
+                delay: 10_000, // 10 seconds
               },
             },
           },

packages/shared/src/server/redis/getQueue.ts

@@ -23,6 +23,7 @@ import { ScoreDeleteQueue } from "./scoreDelete";
 import { DeadLetterRetryQueue } from "./dlqRetryQueue";
 import { WebhookQueue } from "./webhookQueue";
 import { EntityChangeQueue } from "./entityChangeQueue";
+import { DatasetDeleteQueue } from "./datasetDelete";
 
 // IngestionQueue is sharded and requires a sharding key
 // Use IngestionQueue.getInstance({ shardName: queueName }) directly instead
@@ -36,6 +37,8 @@ export function getQueue(
       return CloudUsageMeteringQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.DatasetDelete:
+      return DatasetDeleteQueue.getInstance();
     case QueueName.EvaluationExecution:
       return EvalExecutionQueue.getInstance();
     case QueueName.ExperimentCreate:

packages/shared/src/server/redis/redis.ts

@@ -77,6 +77,7 @@ const createRedisClusterInstance = (
       callback(null, address);
     },
     redisOptions: {
+      username: env.REDIS_USERNAME || undefined,
       password: env.REDIS_AUTH || undefined,
       ...defaultRedisOptions,
       ...additionalOptions,
@@ -129,6 +130,7 @@ export const createNewRedisInstance = (
       ? new Redis({
           host: String(env.REDIS_HOST),
           port: Number(env.REDIS_PORT),
+          username: env.REDIS_USERNAME || undefined,
           password: String(env.REDIS_AUTH),
           ...defaultRedisOptions,
           ...additionalOptions,

packages/shared/src/server/redis/webhookQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
 import { logger } from "../logger";
 
 export class WebhookQueue {
@@ -23,6 +27,7 @@ export class WebhookQueue {
           QueueName.WebhookQueue,
           {
             connection: newRedis,
+            prefix: getQueuePrefix(QueueName.WebhookQueue),
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,

packages/shared/src/server/repositories/automation-repository.ts

@@ -2,17 +2,22 @@ import {
   Action,
   ActionExecutionStatus,
   JobConfigState,
+  Prisma,
   prisma,
   Trigger,
 } from "../../db";
 import {
   TriggerEventSource,
-  WebhookActionConfigWithSecrets,
   TriggerDomain,
   TriggerEventAction,
   ActionDomain,
   AutomationDomain,
-  SafeWebhookActionConfig,
+  ActionDomainWithSecrets,
+  SafeActionConfig,
+  isWebhookActionConfig,
+  WebhookActionConfigWithSecrets,
+  isSafeWebhookActionConfig,
+  convertToSafeWebhookConfig,
 } from "../../domain/automations";
 import { FilterState } from "../../types";
 import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";
@@ -23,7 +28,7 @@ export const getActionByIdWithSecrets = async ({
 }: {
   projectId: string;
   actionId: string;
-}) => {
+}): Promise<ActionDomainWithSecrets | null> => {
   const actionConfig = await prisma.action.findFirst({
     where: {
       id: actionId,
@@ -35,31 +40,41 @@ export const getActionByIdWithSecrets = async ({
     return null;
   }
 
-  const config = actionConfig.config as WebhookActionConfigWithSecrets;
+  if (isWebhookActionConfig(actionConfig.config)) {
+    const config = actionConfig.config; // Type guard ensures this is WebhookActionConfigWithSecrets
 
-  // Decrypt secret headers for webhook execution using new structure
-  const decryptedHeaders = config.requestHeaders
-    ? decryptSecretHeaders(mergeHeaders(config.headers, config.requestHeaders))
-    : Object.entries(config.headers).reduce(
-        (acc, [key, value]) => {
-          acc[key] = { secret: false, value };
-          return acc;
-        },
-        {} as Record<string, { secret: boolean; value: string }>,
-      );
+    // Decrypt secret headers for webhook execution using new structure
+    const decryptedHeaders = config.requestHeaders
+      ? decryptSecretHeaders(
+          mergeHeaders(config.headers, config.requestHeaders),
+        )
+      : config.headers
+        ? Object.entries(config.headers).reduce(
+            (acc, [key, value]) => {
+              acc[key] = { secret: false, value };
+              return acc;
+            },
+            {} as Record<string, { secret: boolean; value: string }>,
+          )
+        : {};
 
-  return {
-    ...actionConfig,
-    config: {
-      type: config.type,
-      url: config.url,
-      requestHeaders: decryptedHeaders,
-      displayHeaders: config.displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-      secretKey: config.secretKey,
-    },
-  };
+    return {
+      ...actionConfig,
+      config: {
+        type: config.type,
+        url: config.url,
+        requestHeaders: decryptedHeaders,
+        displayHeaders: getDisplayHeaders(config),
+        apiVersion: config.apiVersion,
+        displaySecretKey: config.displaySecretKey,
+        secretKey: config.secretKey,
+        lastFailingExecutionId: config.lastFailingExecutionId,
+      },
+    };
+  }
+
+  // For SLACK and others, return as stored (already safe)
+  return actionConfig as ActionDomainWithSecrets;
 };
 
 export const getActionById = async ({
@@ -128,10 +143,7 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {
   };
 };
 
-const convertActionToDomain = (action: Action): ActionDomain => {
-  const config = action.config as WebhookActionConfigWithSecrets;
-
-  // Handle legacy headers - convert them to displayHeaders format if displayHeaders is undefined
+const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {
   let displayHeaders = config.displayHeaders;
   if (!displayHeaders && config.headers) {
     // Convert legacy headers to displayHeaders format
@@ -143,17 +155,25 @@ const convertActionToDomain = (action: Action): ActionDomain => {
       {} as Record<string, { secret: boolean; value: string }>,
     );
   }
+  return displayHeaders;
+};
 
+const convertActionToDomain = (action: Action): ActionDomain => {
+  if (isWebhookActionConfig(action.config)) {
+    const config = action.config;
+    config.displayHeaders = getDisplayHeaders(config);
+
+    return {
+      ...action,
+      config: convertToSafeWebhookConfig(config),
+    };
+  }
+
+  // For SLACK (or future types) return config as-is
   return {
     ...action,
-    config: {
-      type: config.type,
-      url: config.url,
-      displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-    } as SafeWebhookActionConfig,
-  };
+    config: action.config as SafeActionConfig,
+  } as ActionDomain;
 };
 
 export const getAutomationById = async ({
@@ -225,28 +245,49 @@ export const getConsecutiveAutomationFailures = async ({
   automationId: string;
   projectId: string;
 }): Promise<number> => {
-  // First get the automation to extract triggerId and actionId
-  const automation = await prisma.automation.findFirst({
-    where: {
-      id: automationId,
-      projectId,
-    },
+  const automation = await getAutomationById({
+    automationId,
+    projectId,
   });
 
   if (!automation) {
     return 0;
   }
 
-  const { triggerId, actionId } = automation;
-  const executions = await prisma.automationExecution.findMany({
-    where: {
-      triggerId,
-      actionId,
-      projectId,
-      status: {
-        in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
-      },
+  // Build where clause - if lastFailingExecutionId is set, only consider executions newer than it
+  const whereClause: Prisma.AutomationExecutionWhereInput = {
+    triggerId: automation.trigger.id,
+    actionId: automation.action.id,
+    projectId,
+    status: {
+      in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
     },
+  };
+
+  // If there's a lastFailingExecutionId, we need to get executions that are newer than that execution
+  if (
+    isSafeWebhookActionConfig(automation.action.config) &&
+    automation.action.config.lastFailingExecutionId
+  ) {
+    // First get the timestamp of the last failing execution
+    const lastFailingExecution = await prisma.automationExecution.findUnique({
+      where: {
+        id: automation.action.config.lastFailingExecutionId,
+      },
+      select: {
+        createdAt: true,
+      },
+    });
+
+    if (lastFailingExecution) {
+      whereClause.createdAt = {
+        gt: lastFailingExecution.createdAt,
+      };
+    }
+  }
+
+  const executions = await prisma.automationExecution.findMany({
+    where: whereClause,
     orderBy: {
       createdAt: "desc",
     },

packages/shared/src/server/repositories/dataset-run-items.ts

@@ -0,0 +1,458 @@
+import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
+import { type OrderByState } from "../../interfaces/orderBy";
+import { datasetRunItemsTableUiColumnDefinitions } from "../../tableDefinitions";
+import { FilterState } from "../../types";
+import {
+  createFilterFromFilterState,
+  FilterList,
+  orderByToClickhouseSql,
+  StringFilter,
+} from "../queries";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "./clickhouse";
+import { convertDatasetRunItemClickhouseToDomain } from "./dataset-run-items-converters";
+import { DatasetRunItemRecordReadType } from "./definitions";
+import { env } from "../../env";
+import { commandClickhouse } from "./clickhouse";
+import Decimal from "decimal.js";
+
+type DatasetRunItemsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  filter: FilterState;
+  orderBy?: OrderByState | OrderByState[];
+  limit?: number;
+  offset?: number;
+};
+
+type DatasetRunsMetricsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+};
+
+export type DatasetRunsMetrics = {
+  id: string;
+  projectId: string;
+  createdAt: Date;
+  datasetId: string;
+  countRunItems: number;
+  avgTotalCost: Decimal;
+  avgLatency: number;
+};
+
+type DatasetRunsMetricsRecordType = {
+  dataset_run_id: string;
+  project_id: string;
+  dataset_run_created_at: string;
+  dataset_id: string;
+  count_run_items: number;
+  avg_latency_seconds: number;
+  avg_total_cost: number;
+};
+
+const convertDatasetRunsMetricsRecord = (
+  record: DatasetRunsMetricsRecordType,
+): DatasetRunsMetrics => {
+  return {
+    id: record.dataset_run_id,
+    projectId: record.project_id,
+    createdAt: parseClickhouseUTCDateTimeFormat(record.dataset_run_created_at),
+    datasetId: record.dataset_id,
+    countRunItems: record.count_run_items,
+    avgTotalCost: record.avg_total_cost
+      ? new Decimal(record.avg_total_cost)
+      : new Decimal(0),
+    avgLatency: record.avg_latency_seconds ?? 0,
+  };
+};
+
+const getProjectDatasetIdDefaultFilter = (
+  projectId: string,
+  datasetId: string,
+) => {
+  return {
+    datasetRunItemsFilter: new FilterList([
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "project_id",
+        operator: "=",
+        value: projectId,
+      }),
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "dataset_id",
+        operator: "=",
+        value: datasetId,
+      }),
+    ]),
+  };
+};
+
+const getDatasetRunsTableInternal = async <T>(
+  opts: DatasetRunsMetricsTableQuery & {
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, orderBy, limit, offset } = opts;
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    orderByArray.push(orderBy);
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    WITH observations_filtered AS (
+      SELECT
+        o.id,
+        o.trace_id,
+        o.project_id,
+        o.start_time,
+        o.end_time,
+        o.total_cost
+      FROM observations o FINAL
+      WHERE o.project_id = {projectId: String}
+        AND o.start_time >= (
+          SELECT min(dri.dataset_run_created_at) - INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+        AND o.start_time <= (
+          SELECT max(dri.dataset_run_created_at) + INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+    ),
+    traces_aggregated AS (
+      SELECT
+        of.trace_id,
+        of.project_id,
+        dateDiff('millisecond', min(of.start_time), max(of.end_time)) as latency_ms,
+        sum(of.total_cost) as total_cost
+      FROM observations_filtered of
+      JOIN dataset_run_items dri ON dri.trace_id = of.trace_id 
+        AND dri.project_id = of.project_id
+        AND dri.observation_id IS NULL  -- Only for trace-level dataset run items
+      WHERE dri.dataset_id = {datasetId: String}
+      GROUP BY of.trace_id, of.project_id
+    ),
+    observations_direct AS (
+      SELECT
+        dri.observation_id,
+        dri.project_id,
+        dri.trace_id,
+        of.total_cost,
+        dateDiff('millisecond', of.start_time, of.end_time) as latency_ms
+      FROM dataset_run_items dri
+      JOIN observations_filtered of ON dri.observation_id = of.id
+        AND dri.project_id = of.project_id
+        AND dri.trace_id = of.trace_id
+      WHERE dri.dataset_id = {datasetId: String}
+        AND dri.observation_id IS NOT NULL  -- Only for observation-level dataset run items
+    )
+    SELECT DISTINCT
+      dri.dataset_run_id as dataset_run_id,
+      dri.project_id as project_id,
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count_run_items,
+      
+      -- Latency metrics (priority: observation > trace)
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.latency_ms IS NOT NULL
+        THEN od.latency_ms / 1000.0
+        ELSE COALESCE(ta.latency_ms / 1000.0, 0)
+      END) as avg_latency_seconds,
+      
+      -- Cost metrics (priority: observation > trace)  
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.total_cost IS NOT NULL
+        THEN od.total_cost
+        ELSE COALESCE(ta.total_cost, 0)
+      END) as avg_total_cost
+    FROM dataset_run_items dri 
+    LEFT JOIN traces_aggregated ta
+      ON dri.trace_id = ta.trace_id
+      AND dri.project_id = ta.project_id
+    LEFT JOIN observations_direct od
+      ON dri.observation_id = od.observation_id
+      AND dri.project_id = od.project_id
+      AND dri.trace_id = od.trace_id
+    WHERE ${appliedFilter.query}
+    GROUP BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_run_created_at
+    ORDER BY dri.dataset_run_created_at DESC
+    ${orderByClause}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      projectId,
+      datasetId,
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunsTableMetricsCh = async (
+  opts: DatasetRunsMetricsTableQuery,
+): Promise<DatasetRunsMetrics[]> => {
+  // First get the metrics (latency, cost, counts)
+  const rows = await getDatasetRunsTableInternal<DatasetRunsMetricsRecordType>({
+    ...opts,
+    tags: { kind: "list" },
+  });
+
+  return rows.map(convertDatasetRunsMetricsRecord);
+};
+
+const getDatasetRunItemsTableInternal = async <T>(
+  opts: DatasetRunItemsTableQuery & {
+    select: "count" | "rows";
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, filter, orderBy, limit, offset } = opts;
+
+  let selectString = "";
+
+  switch (opts.select) {
+    case "count":
+      selectString =
+        "count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count";
+      break;
+    case "rows":
+      selectString = `
+      dri.id as id, 
+      dri.project_id as project_id, 
+      dri.trace_id as trace_id, 
+      dri.observation_id as observation_id, 
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_id as dataset_run_id, 
+      dri.dataset_item_id as dataset_item_id, 
+      dri.error as error,
+      dri.created_at as created_at, 
+      dri.updated_at as updated_at,
+      dri.dataset_run_name as dataset_run_name,
+      dri.dataset_run_description as dataset_run_description,
+      dri.dataset_run_metadata as dataset_run_metadata,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      dri.dataset_item_input as dataset_item_input,
+      dri.dataset_item_expected_output as dataset_item_expected_output,
+      dri.dataset_item_metadata as dataset_item_metadata,
+      dri.is_deleted as is_deleted,
+      dri.event_ts as event_ts`;
+      break;
+    default:
+      throw new Error(`Unknown select type: ${opts.select}`);
+  }
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+
+  datasetRunItemsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      datasetRunItemsTableUiColumnDefinitions,
+    ),
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    if (Array.isArray(orderBy)) {
+      orderByArray.push(...orderBy);
+    } else {
+      orderByArray.push(orderBy);
+    }
+  }
+
+  // Add event_ts DESC for row queries (for deduplication)
+  if (opts.select === "rows") {
+    orderByArray.push({
+      column: "eventTs",
+      order: "DESC",
+    });
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    SELECT
+      ${selectString}
+    FROM dataset_run_items dri 
+    WHERE ${appliedFilter.query}
+    ${orderByClause}
+    ${opts.select === "rows" ? "LIMIT 1 BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id" : ""}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunItemsByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<DatasetRunItemDomain[]> => {
+  const rows =
+    await getDatasetRunItemsTableInternal<DatasetRunItemRecordReadType>({
+      ...opts,
+      select: "rows",
+      tags: { kind: "list" },
+    });
+
+  return rows.map(convertDatasetRunItemClickhouseToDomain);
+};
+
+export const getDatasetRunItemsCountByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<number> => {
+  const rows = await getDatasetRunItemsTableInternal<{ count: string }>({
+    ...opts,
+    select: "count",
+    tags: { kind: "list" },
+  });
+
+  return Number(rows[0]?.count);
+};
+
+export const deleteDatasetRunItemsByProjectId = async ({
+  projectId,
+}: {
+  projectId: string;
+}) => {
+  const query = `
+      DELETE FROM dataset_run_items
+      WHERE project_id = {projectId: String};
+    `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetId = async ({
+  projectId,
+  datasetId,
+}: {
+  projectId: string;
+  datasetId: string;
+}) => {
+  const query = `
+  DELETE FROM dataset_run_items
+  WHERE project_id = {projectId: String}
+  AND dataset_id = {datasetId: String}
+`;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetRunIds = async ({
+  projectId,
+  datasetRunIds,
+  datasetId,
+}: {
+  projectId: string;
+  datasetRunIds: string[];
+  datasetId: string;
+}) => {
+  const query = `
+    DELETE FROM dataset_run_items
+    WHERE project_id = {projectId: String}
+    AND dataset_id = {datasetId: String}
+    AND dataset_run_id IN ({datasetRunIds: Array(String)})
+  `;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};

packages/shared/src/server/repositories/definitions.ts

@@ -355,31 +355,37 @@ export const convertPostgresDatasetRunItemToInsert = (
     project_id: datasetRunItem.project_id,
     dataset_run_id: datasetRunItem.dataset_run_id,
     dataset_item_id: datasetRunItem.dataset_item_id,
+    dataset_id: datasetRunItem.dataset_id,
     trace_id: datasetRunItem.trace_id,
     observation_id: datasetRunItem.observation_id,
     error: datasetRunItem.error,
     created_at: datasetRunItem.created_at?.getTime(),
     updated_at: datasetRunItem.updated_at?.getTime(),
     // denormalized run data
-    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
-    dataset_id: datasetRunItem.dataset_id,
     dataset_run_name: datasetRunItem.dataset_run_name,
     dataset_run_description: datasetRunItem.dataset_run_description,
     dataset_run_metadata:
-      typeof datasetRunItem.dataset_run_metadata === "string"
-        ? { dataset_run_metadata: datasetRunItem.dataset_run_metadata }
+      typeof datasetRunItem.dataset_run_metadata === "string" ||
+      typeof datasetRunItem.dataset_run_metadata === "number" ||
+      typeof datasetRunItem.dataset_run_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_run_metadata }
         : Array.isArray(datasetRunItem.dataset_run_metadata)
-          ? { dataset_run_metadata: datasetRunItem.dataset_run_metadata }
-          : datasetRunItem.dataset_run_metadata,
+          ? { metadata: datasetRunItem.dataset_run_metadata }
+          : (datasetRunItem.dataset_run_metadata ?? {}),
+    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
     // denormalized item data
-    dataset_item_input: datasetRunItem.dataset_item_input,
-    dataset_item_expected_output: datasetRunItem.dataset_item_expected_output,
+    dataset_item_input: JSON.stringify(datasetRunItem.dataset_item_input),
+    dataset_item_expected_output: JSON.stringify(
+      datasetRunItem.dataset_item_expected_output,
+    ),
     dataset_item_metadata:
-      typeof datasetRunItem.dataset_item_metadata === "string"
-        ? { dataset_item_metadata: datasetRunItem.dataset_item_metadata }
+      typeof datasetRunItem.dataset_item_metadata === "string" ||
+      typeof datasetRunItem.dataset_item_metadata === "number" ||
+      typeof datasetRunItem.dataset_item_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_item_metadata }
         : Array.isArray(datasetRunItem.dataset_item_metadata)
-          ? { dataset_item_metadata: datasetRunItem.dataset_item_metadata }
-          : datasetRunItem.dataset_item_metadata,
+          ? { metadata: datasetRunItem.dataset_item_metadata }
+          : (datasetRunItem.dataset_item_metadata ?? {}),
     event_ts: datasetRunItem.created_at?.getTime(),
     is_deleted: 0,
   };

packages/shared/src/server/repositories/index.ts

@@ -14,3 +14,4 @@ export * from "./blobStorageLog";
 export * from "./environments";
 export * from "./automation-repository";
 export * from "./dataset-run-items-converters";
+export * from "./dataset-run-items";

packages/shared/src/server/repositories/observations.ts

@@ -23,7 +23,8 @@ import {
   observationsTableUiColumnDefinitions,
 } from "../../tableDefinitions";
 import { OrderByState } from "../../interfaces/orderBy";
-import { getTracesByIds } from "./traces";
+import { getTimeframesTracesAMT, getTracesByIds } from "./traces";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertObservation } from "./observations_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -612,12 +613,14 @@ const getObservationsTableInternal = async <T>(
   } = opts;
 
   const selectString = selectIOAndMetadata
-    ? `
-    ${select},
-    ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
-  `
+    ? `${select}, o.input, o.output, o.metadata`
     : select;
 
+  const timeFilter = filter.find(
+    (f) =>
+      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
+  );
+
   const scoresFilter = new FilterList([
     new StringFilter({
       clickhouseTable: "scores",
@@ -627,33 +630,22 @@ const getObservationsTableInternal = async <T>(
     }),
   ]);
 
-  const timeFilter = opts.filter.find(
-    (f) =>
-      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
-  );
-
-  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
-  const traceTableFilter = opts.filter.filter(
-    (f) =>
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(f.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(f.column),
-  );
-
   const hasScoresFilter = filter.some((f) =>
     f.column.toLowerCase().includes("scores"),
   );
 
-  const orderByTraces = opts.orderBy
-    ? observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(opts.orderBy.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(opts.orderBy.column)
+  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
+  const traceTableFilter = filter.filter((f) =>
+    observationsTableTraceUiColumnDefinitions.some(
+      (c) => c.uiTableId === f.column || c.uiTableName === f.column,
+    ),
+  );
+
+  const orderByTraces = orderBy
+    ? observationsTableTraceUiColumnDefinitions.some(
+        (c) =>
+          c.uiTableId === orderBy.column || c.uiTableName === orderBy.column,
+      )
     : undefined;
 
   timeFilter
@@ -759,7 +751,7 @@ const getObservationsTableInternal = async <T>(
       SELECT
        ${selectString}
       FROM observations o 
-        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN __TRACE_TABLE__ t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
         ${hasScoresFilter ? `LEFT JOIN scores_agg AS s ON s.trace_id = o.trace_id and s.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
         
@@ -769,30 +761,51 @@ const getObservationsTableInternal = async <T>(
       ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-  const res = await queryClickhouse<T>({
-    query,
-    params: {
-      ...appliedScoresFilter.params,
-      ...appliedObservationsFilter.params,
-      ...(timeFilter
-        ? {
-            tracesTimestampFilter: convertDateToClickhouseDateTime(
-              timeFilter.value as Date,
-            ),
-          }
-        : {}),
-      ...search.params,
+  return measureAndReturn({
+    operationName: "getObservationsTableInternal",
+    projectId,
+    input: {
+      params: {
+        ...appliedScoresFilter.params,
+        ...appliedObservationsFilter.params,
+        ...(timeFilter
+          ? {
+              tracesTimestampFilter: convertDateToClickhouseDateTime(
+                timeFilter.value as Date,
+              ),
+            }
+          : {}),
+        ...search.params,
+      },
+      tags: {
+        ...(opts.tags ?? {}),
+        feature: "tracing",
+        type: "observation",
+        projectId,
+        kind: opts.select,
+        operation_name: "getObservationsTableInternal",
+      },
     },
-    tags: {
-      ...(opts.tags ?? {}),
-      feature: "tracing",
-      type: "observation",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(
+        (timeFilter?.value as Date) || undefined,
+      );
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };
 
 export const getObservationsGroupedByModel = async (

packages/shared/src/server/repositories/scores.ts

@@ -32,6 +32,7 @@ import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { recordDistribution } from "../instrumentation";
 import { prisma } from "../../db";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -284,6 +285,53 @@ export const getScoresForDatasetRuns = async <
   return rows.map(convertToScore);
 };
 
+export const getTraceScoresForDatasetRuns = async (
+  projectId: string,
+  datasetRunIds: string[],
+): Promise<Array<{ dataset_run_id: string } & any>> => {
+  if (datasetRunIds.length === 0) return [];
+
+  const query = `
+    SELECT 
+      s.* EXCEPT (metadata),
+      length(mapKeys(s.metadata)) > 0 AS has_metadata,
+      dri.dataset_run_id as run_id
+    FROM dataset_run_items dri 
+    JOIN scores s FINAL ON dri.trace_id = s.trace_id 
+      AND dri.project_id = s.project_id
+    WHERE dri.project_id = {projectId: String}
+      AND dri.dataset_run_id IN {datasetRunIds: Array(String)}
+      AND s.project_id = {projectId: String}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id, dri.dataset_run_id
+  `;
+
+  const rows = await queryClickhouse<
+    Omit<ScoreRecordReadType, "metadata"> & {
+      has_metadata: 0 | 1;
+      run_id: string;
+    }
+  >({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+    },
+    tags: {
+      feature: "dataset-run-items",
+      type: "trace-scores",
+      kind: "list",
+      projectId,
+    },
+  });
+
+  return rows.map((row) => ({
+    ...convertToScore({ ...row, metadata: {} }),
+    datasetRunId: row.run_id,
+    hasMetadata: !!row.has_metadata,
+  }));
+};
+
 // Used in multiple places, including the public API, hence the non-default exclusion of metadata via excludeMetadata flag
 export const getScoresForTraces = async <
   ExcludeMetadata extends boolean,
@@ -895,31 +943,49 @@ const getScoresUiGeneric = async <T>(props: {
       SELECT 
           ${select}
       FROM scores s final
-      ${performTracesJoin ? "LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
+      ${performTracesJoin ? "LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId: projectId,
-      ...(scoresFilterRes ? scoresFilterRes.params : {}),
-      limit: limit,
-      offset: offset,
+  return measureAndReturn({
+    operationName: "getScoresUiGeneric",
+    projectId,
+    input: {
+      params: {
+        projectId: projectId,
+        ...(scoresFilterRes ? scoresFilterRes.params : {}),
+        limit: limit,
+        offset: offset,
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "score",
+        projectId,
+        select: props.select,
+        operation_name: "getScoresUiGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "score",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces_all_amt"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return rows;
 };
 
 export const getScoreNames = async (

packages/shared/src/server/repositories/traces.ts

@@ -15,10 +15,13 @@ import {
   FilterList,
   StringFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
-import { TraceRecordReadType } from "./definitions";
+import { TraceRecordReadType, convertTraceToTraceMt } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
 import { UiColumnMappings } from "../../tableDefinitions";
-import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+  convertDateToClickhouseDateTime,
+  clickhouseClient,
+} from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import {
@@ -163,6 +166,7 @@ export const checkTraceExists = async ({
         type: "trace",
         kind: "exists",
         projectId,
+        operation_name: "checkTraceExists",
       },
       timestamp: timestamp ?? exactTimestamp,
     },
@@ -186,7 +190,7 @@ export const checkTraceExists = async ({
       const rows = await queryClickhouse<{ id: string; project_id: string }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
 
       return rows.length > 0;
@@ -209,7 +213,7 @@ export const checkTraceExists = async ({
       const rows = await queryClickhouse<{ id: string; project_id: string }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
 
       return rows.length > 0;
@@ -236,6 +240,38 @@ export const upsertTrace = async (trace: Partial<TraceRecordReadType>) => {
       projectId: trace.project_id ?? "",
     },
   });
+
+  // Also insert into traces_mt if experiment flag is enabled
+  if (env.LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES === "true") {
+    // Convert trace to insert format first (since we have read format)
+    const traceRecord = trace as TraceRecordReadType;
+    const traceInsert = {
+      ...traceRecord,
+      timestamp: new Date(traceRecord.timestamp).getTime(),
+      created_at: new Date(traceRecord.created_at).getTime(),
+      updated_at: new Date(traceRecord.updated_at).getTime(),
+      event_ts: new Date(traceRecord.event_ts).getTime(),
+      is_deleted: 0,
+    };
+
+    // Convert to traces_mt format
+    const traceMt = convertTraceToTraceMt(traceInsert);
+
+    // Insert directly into traces_mt using clickhouse client
+    await clickhouseClient().insert({
+      table: "traces_mt",
+      format: "JSONEachRow",
+      values: [traceMt],
+      clickhouse_settings: {
+        log_comment: JSON.stringify({
+          feature: "tracing",
+          type: "trace_mt",
+          kind: "upsert",
+          experiment: "insert_into_aggregating_merge_trees",
+        }),
+      },
+    });
+  }
 };
 
 export const getTracesByIds = async (
@@ -260,6 +296,7 @@ export const getTracesByIds = async (
         type: "trace",
         kind: "byId",
         projectId,
+        operation_name: "getTracesByIds",
       },
       clickhouseConfigs,
     },
@@ -276,7 +313,7 @@ export const getTracesByIds = async (
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
         clickhouseConfigs: input.clickhouseConfigs,
       });
     },
@@ -311,7 +348,7 @@ export const getTracesByIds = async (
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
         clickhouseConfigs: input.clickhouseConfigs,
       });
     },
@@ -341,6 +378,7 @@ export const getTracesBySessionId = async (
         type: "trace",
         kind: "list",
         projectId,
+        operation_name: "getTracesBySessionId",
       },
       timestamp,
     },
@@ -357,7 +395,7 @@ export const getTracesBySessionId = async (
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: (input) => {
@@ -388,7 +426,7 @@ export const getTracesBySessionId = async (
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -416,6 +454,7 @@ export const hasAnyTrace = async (projectId: string) => {
         type: "trace",
         kind: "hasAny",
         projectId,
+        operation_name: "hasAnyTrace",
       },
     },
     existingExecution: async (input) => {
@@ -431,7 +470,7 @@ export const hasAnyTrace = async (projectId: string) => {
         params: {
           projectId: input.projectId,
         },
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
 
       return rows.length > 0;
@@ -449,7 +488,7 @@ export const hasAnyTrace = async (projectId: string) => {
         params: {
           projectId: input.projectId,
         },
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
 
       return rows.length > 0;
@@ -464,33 +503,72 @@ export const getTraceCountsByProjectInCreationInterval = async ({
   start: Date;
   end: Date;
 }) => {
-  const query = `
-    SELECT
-      project_id,
-      count(*) as count
-    FROM traces
-    WHERE created_at >= {start: DateTime64(3)}
-    AND created_at < {end: DateTime64(3)}
-    GROUP BY project_id
-  `;
-
-  const rows = await queryClickhouse<{ project_id: string; count: string }>({
-    query,
-    params: {
-      start: convertDateToClickhouseDateTime(start),
-      end: convertDateToClickhouseDateTime(end),
+  return measureAndReturn({
+    operationName: "getTraceCountsByProjectInCreationInterval",
+    projectId: "__CROSS_PROJECT__",
+    input: {
+      params: {
+        start: convertDateToClickhouseDateTime(start),
+        end: convertDateToClickhouseDateTime(end),
+      },
+      tags: {
+        feature: "tracing",
+        type: "trace",
+        kind: "analytic",
+        operation_name: "getTraceCountsByProjectInCreationInterval",
+      },
+      timestamp: start,
     },
-    tags: {
-      feature: "tracing",
-      type: "trace",
-      kind: "analytic",
+    existingExecution: async (input) => {
+      const query = `
+        SELECT
+          project_id,
+          count(*) as count
+        FROM traces
+        WHERE created_at >= {start: DateTime64(3)}
+        AND created_at < {end: DateTime64(3)}
+        GROUP BY project_id
+      `;
+
+      const rows = await queryClickhouse<{ project_id: string; count: string }>(
+        {
+          query,
+          params: input.params,
+          tags: { ...input.tags, experiment_amt: "original" },
+        },
+      );
+
+      return rows.map((row) => ({
+        projectId: row.project_id,
+        count: Number(row.count),
+      }));
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(input.timestamp);
+      const query = `
+        SELECT
+          project_id,
+          count(*) as count
+        FROM ${traceAmt}
+        WHERE created_at >= {start: DateTime64(3)}
+        AND created_at < {end: DateTime64(3)}
+        GROUP BY project_id
+      `;
+
+      const rows = await queryClickhouse<{ project_id: string; count: string }>(
+        {
+          query,
+          params: input.params,
+          tags: { ...input.tags, experiment_amt: "new" },
+        },
+      );
+
+      return rows.map((row) => ({
+        projectId: row.project_id,
+        count: Number(row.count),
+      }));
     },
   });
-
-  return rows.map((row) => ({
-    projectId: row.project_id,
-    count: Number(row.count),
-  }));
 };
 
 export const getTraceCountOfProjectsSinceCreationDate = async ({
@@ -500,28 +578,58 @@ export const getTraceCountOfProjectsSinceCreationDate = async ({
   projectIds: string[];
   start: Date;
 }) => {
-  const query = `
-    SELECT
-      count(*) as count
-    FROM traces
-    WHERE project_id IN ({projectIds: Array(String)})
-    AND created_at >= {start: DateTime64(3)}
-  `;
-
-  const rows = await queryClickhouse<{ count: string }>({
-    query,
-    params: {
-      projectIds,
-      start: convertDateToClickhouseDateTime(start),
+  return measureAndReturn({
+    operationName: "getTraceCountOfProjectsSinceCreationDate",
+    projectId: "__CROSS_PROJECT__",
+    input: {
+      params: {
+        projectIds,
+        start: convertDateToClickhouseDateTime(start),
+      },
+      tags: {
+        feature: "tracing",
+        type: "trace",
+        kind: "analytic",
+        operation_name: "getTraceCountOfProjectsSinceCreationDate",
+      },
+      timestamp: start,
     },
-    tags: {
-      feature: "tracing",
-      type: "trace",
-      kind: "analytic",
+    existingExecution: async (input) => {
+      const query = `
+        SELECT
+          count(*) as count
+        FROM traces
+        WHERE project_id IN ({projectIds: Array(String)})
+        AND created_at >= {start: DateTime64(3)}
+      `;
+
+      const rows = await queryClickhouse<{ count: string }>({
+        query,
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+      });
+
+      return Number(rows[0]?.count ?? 0);
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(input.timestamp);
+      const query = `
+        SELECT
+          count(*) as count
+        FROM ${traceAmt}
+        WHERE project_id IN ({projectIds: Array(String)})
+        AND created_at >= {start: DateTime64(3)}
+      `;
+
+      const rows = await queryClickhouse<{ count: string }>({
+        query,
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+      });
+
+      return Number(rows[0]?.count ?? 0);
     },
   });
-
-  return Number(rows[0]?.count ?? 0);
 };
 
 /**
@@ -561,6 +669,7 @@ export const getTraceById = async ({
         type: "trace",
         kind: "byId",
         projectId,
+        operation_name: "getTraceById",
       },
     },
     existingExecution: (input) => {
@@ -578,7 +687,7 @@ export const getTraceById = async ({
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: (input) => {
@@ -612,7 +721,7 @@ export const getTraceById = async ({
       return queryClickhouse<TraceRecordReadType>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -658,6 +767,7 @@ export const getTracesGroupedByName = async (
         type: "trace",
         kind: "analytic",
         projectId,
+        operation_name: "getTracesGroupedByName",
       },
     },
     existingExecution: async (input) => {
@@ -682,7 +792,7 @@ export const getTracesGroupedByName = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: async (input) => {
@@ -711,7 +821,7 @@ export const getTracesGroupedByName = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -755,6 +865,7 @@ export const getTracesGroupedByUsers = async (
         type: "trace",
         kind: "analytic",
         projectId,
+        operation_name: "getTracesGroupedByUsers",
       },
     },
     existingExecution: async (input) => {
@@ -781,7 +892,7 @@ export const getTracesGroupedByUsers = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: async (input) => {
@@ -813,7 +924,7 @@ export const getTracesGroupedByUsers = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -848,6 +959,7 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
         type: "trace",
         kind: "analytic",
         projectId,
+        operation_name: "getTracesGroupedByTags",
       },
     },
     existingExecution: async (input) => {
@@ -864,7 +976,7 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: async (input) => {
@@ -888,7 +1000,7 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -911,6 +1023,7 @@ export const getTracesIdentifierForSession = async (
         type: "trace",
         kind: "list",
         projectId,
+        operation_name: "getTracesIdentifierForSession",
       },
     },
     existingExecution: (input) => {
@@ -938,7 +1051,7 @@ export const getTracesIdentifierForSession = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: (input) => {
@@ -950,10 +1063,11 @@ export const getTracesIdentifierForSession = async (
           start_time as timestamp,
           project_id,
           environment
-        FROM traces_all_amt FINAL
+        FROM traces_all_amt
         WHERE (project_id = {projectId: String})
         AND (session_id = {sessionId: String})
-        ORDER BY start_time ASC;
+        ORDER BY start_time ASC
+        LIMIT 1 BY id, project_id;
       `;
 
       return queryClickhouse<{
@@ -965,7 +1079,7 @@ export const getTracesIdentifierForSession = async (
       }>({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -1063,6 +1177,7 @@ export const hasAnyUser = async (projectId: string) => {
         type: "user",
         kind: "hasAny",
         projectId,
+        operation_name: "hasAnyUser",
       },
     },
     existingExecution: async (input) => {
@@ -1080,7 +1195,7 @@ export const hasAnyUser = async (projectId: string) => {
         params: {
           projectId: input.projectId,
         },
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
 
       return rows.length > 0;
@@ -1100,7 +1215,7 @@ export const hasAnyUser = async (projectId: string) => {
         params: {
           projectId: input.projectId,
         },
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
 
       return rows.length > 0;
@@ -1137,6 +1252,7 @@ export const getTotalUserCount = async (
         type: "trace",
         kind: "analytic",
         projectId,
+        operation_name: "getTotalUserCount",
       },
     },
     existingExecution: async (input) => {
@@ -1152,7 +1268,7 @@ export const getTotalUserCount = async (
       return queryClickhouse({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "original" },
       });
     },
     newExecution: async (input) => {
@@ -1175,7 +1291,7 @@ export const getTotalUserCount = async (
       return queryClickhouse({
         query,
         params: input.params,
-        tags: input.tags,
+        tags: { ...input.tags, experiment_amt: "new" },
       });
     },
   });
@@ -1232,10 +1348,8 @@ export const getUserMetrics = async (
                     o.project_id = {projectId: String }
                     ${timestampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
                     AND o.trace_id in (
-                        SELECT
-                            distinct id
-                        from
-                            traces
+                        SELECT distinct id
+                        from __TRACE_TABLE__
                         where
                             user_id IN ({userIds: Array(String) })
                             AND project_id = {projectId: String }
@@ -1249,25 +1363,17 @@ export const getUserMetrics = async (
                     t.user_id,
                     t.project_id,
                     t.timestamp,
-                    t.environment,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY id
-                        ORDER BY
-                            event_ts DESC
-                    ) AS rn
+                    t.environment
                 FROM
-                    traces t
+                    __TRACE_TABLE__ t FINAL
                 WHERE
                     t.user_id IN ({userIds: Array(String) })
                     AND t.project_id = {projectId: String }
                     ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
             ) as t on t.id = o.trace_id
             and t.project_id = o.project_id
-        WHERE
-            o.rn = 1
-            and t.rn = 1
-        group by
-            t.user_id
+        WHERE o.rn = 1
+        group by t.user_id
     )
     SELECT
         arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sum_usage_details))) as input_usage,
@@ -1280,56 +1386,102 @@ export const getUserMetrics = async (
         sum_total_cost,
         max_timestamp,
         min_timestamp
-    FROM
-        stats
+    FROM stats`;
 
-  `;
-
-  const rows = await queryClickhouse<{
-    user_id: string;
-    environment: string;
-    max_timestamp: string;
-    min_timestamp: string;
-    input_usage: string;
-    output_usage: string;
-    total_usage: string;
-    obs_count: string;
-    trace_count: string;
-    sum_total_cost: string;
-  }>({
-    query,
-    params: {
-      projectId,
-      userIds,
-      ...chFilterRes.params,
-      ...(timestampFilter
-        ? {
-            traceTimestamp: convertDateToClickhouseDateTime(
-              (timestampFilter as DateTimeFilter).value,
-            ),
-          }
-        : {}),
+  return measureAndReturn({
+    operationName: "getUserMetrics",
+    projectId,
+    input: {
+      params: {
+        projectId,
+        userIds,
+        ...chFilterRes.params,
+        ...(timestampFilter
+          ? {
+              traceTimestamp: convertDateToClickhouseDateTime(
+                (timestampFilter as DateTimeFilter).value,
+              ),
+            }
+          : {}),
+      },
+      tags: {
+        feature: "tracing",
+        type: "trace",
+        kind: "analytic",
+        projectId,
+        operation_name: "getUserMetrics",
+      },
     },
-    tags: {
-      feature: "tracing",
-      type: "trace",
-      kind: "analytic",
-      projectId,
+    existingExecution: async (input) => {
+      const rows = await queryClickhouse<{
+        user_id: string;
+        environment: string;
+        max_timestamp: string;
+        min_timestamp: string;
+        input_usage: string;
+        output_usage: string;
+        total_usage: string;
+        obs_count: string;
+        trace_count: string;
+        sum_total_cost: string;
+      }>({
+        query: query.replaceAll("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+      });
+
+      return rows.map((row) => ({
+        userId: row.user_id,
+        environment: row.environment,
+        maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
+        minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
+        inputUsage: Number(row.input_usage),
+        outputUsage: Number(row.output_usage),
+        totalUsage: Number(row.total_usage),
+        observationCount: Number(row.obs_count),
+        traceCount: Number(row.trace_count),
+        totalCost: Number(row.sum_total_cost),
+      }));
+    },
+    newExecution: async (input) => {
+      // Extract the timestamp from filter for AMT table selection
+      const fromTimestamp = filter?.find(
+        (f) =>
+          f.column === "timestamp" &&
+          (f.operator === ">=" || f.operator === ">"),
+      )?.value as Date | undefined;
+      const traceAmt = getTimeframesTracesAMT(fromTimestamp);
+      const rows = await queryClickhouse<{
+        user_id: string;
+        environment: string;
+        max_timestamp: string;
+        min_timestamp: string;
+        input_usage: string;
+        output_usage: string;
+        total_usage: string;
+        obs_count: string;
+        trace_count: string;
+        sum_total_cost: string;
+      }>({
+        query: query.replaceAll("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+      });
+
+      return rows.map((row) => ({
+        userId: row.user_id,
+        environment: row.environment,
+        maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
+        minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
+        inputUsage: Number(row.input_usage),
+        outputUsage: Number(row.output_usage),
+        totalUsage: Number(row.total_usage),
+        observationCount: Number(row.obs_count),
+        traceCount: Number(row.trace_count),
+        totalCost: Number(row.sum_total_cost),
+      }));
     },
   });
-
-  return rows.map((row) => ({
-    userId: row.user_id,
-    environment: row.environment,
-    maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
-    minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
-    inputUsage: Number(row.input_usage),
-    outputUsage: Number(row.output_usage),
-    totalUsage: Number(row.total_usage),
-    observationCount: Number(row.obs_count),
-    traceCount: Number(row.trace_count),
-    totalCost: Number(row.sum_total_cost),
-  }));
 };
 
 export const getTracesForBlobStorageExport = function (
@@ -1360,7 +1512,7 @@ export const getTracesForBlobStorageExport = function (
     AND timestamp <= {maxTimestamp: DateTime64(3)}
   `;
 
-  const records = queryClickhouseStream<Record<string, unknown>>({
+  return queryClickhouseStream<Record<string, unknown>>({
     query,
     params: {
       projectId,
@@ -1374,8 +1526,6 @@ export const getTracesForBlobStorageExport = function (
       projectId,
     },
   });
-
-  return records;
 };
 
 export const getTracesForPostHog = async function* (
@@ -1440,6 +1590,7 @@ export const getTracesForPostHog = async function* (
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
+
   for await (const record of records) {
     yield {
       timestamp: record.timestamp,
@@ -1467,32 +1618,69 @@ export const getTracesForPostHog = async function* (
   }
 };
 
+/**
+ * This query is used only for legacy support of redirects without a projectId.
+ * We don't have an index on the traceId so it will be a full table scan.
+ * We expect at most 10s of calls per day, so this is acceptable.
+ */
 export const getTracesByIdsForAnyProject = async (traceIds: string[]) => {
-  const query = `
-      SELECT id, project_id
-      FROM traces
-      WHERE id IN ({traceIds: Array(String)})
-      ORDER BY event_ts DESC
-      LIMIT 1 by id, project_id;`;
-  const records = await queryClickhouse<{
-    id: string;
-    project_id: string;
-  }>({
-    query,
-    params: {
-      traceIds,
+  return measureAndReturn({
+    operationName: "getTracesByIdsForAnyProject",
+    projectId: "__CROSS_PROJECT__",
+    input: {
+      params: {
+        traceIds,
+      },
+      tags: {
+        feature: "tracing",
+        type: "trace",
+        kind: "list",
+        operation_name: "getTracesByIdsForAnyProject",
+      },
     },
-    tags: {
-      feature: "tracing",
-      type: "trace",
-      kind: "list",
+    existingExecution: async (input) => {
+      const query = `
+          SELECT id, project_id
+          FROM traces
+          WHERE id IN ({traceIds: Array(String)})
+          ORDER BY event_ts DESC
+          LIMIT 1 by id, project_id;`;
+      const records = await queryClickhouse<{
+        id: string;
+        project_id: string;
+      }>({
+        query,
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+      });
+
+      return records.map((record) => ({
+        id: record.id,
+        projectId: record.project_id,
+      }));
+    },
+    newExecution: async (input) => {
+      // For this query, we need to query all AMT tables as we don't have a specific timestamp
+      // We'll use the all AMT table as it contains all data
+      const query = `
+          SELECT DISTINCT id, project_id
+          FROM traces_all_amt
+          WHERE id IN ({traceIds: Array(String)})`;
+      const records = await queryClickhouse<{
+        id: string;
+        project_id: string;
+      }>({
+        query,
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+      });
+
+      return records.map((record) => ({
+        id: record.id,
+        projectId: record.project_id,
+      }));
     },
   });
-
-  return records.map((record) => ({
-    id: record.id,
-    projectId: record.project_id,
-  }));
 };
 
 export async function getAgentGraphData(params: {

packages/shared/src/server/services/SlackService.ts

@@ -0,0 +1,386 @@
+/**
+ * Slack Integration Service
+ *
+ * Simplified service that properly uses the official Slack SDK libraries:
+ * - @slack/oauth InstallProvider for OAuth flow management
+ * - @slack/web-api WebClient for Slack API operations
+ * - Metadata-based project-to-team mapping
+ */
+
+import { WebClient } from "@slack/web-api";
+import { InstallProvider } from "@slack/oauth";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { prisma } from "../../db";
+import { encrypt, decrypt } from "../../encryption";
+
+// Types for Slack integration
+export interface SlackChannel {
+  id: string;
+  name: string;
+  isPrivate: boolean;
+  isMember: boolean;
+}
+
+export interface SlackMessageParams {
+  client: WebClient;
+  channelId: string;
+  blocks: any[];
+  text?: string;
+}
+
+export interface SlackMessageResponse {
+  messageTs: string;
+  channel: string;
+}
+
+// Interface for Slack installation metadata
+export interface SlackInstallationMetadata {
+  projectId: string;
+}
+
+/**
+ * Type guard to validate Slack installation metadata
+ */
+function isSlackInstallationMetadata(
+  metadata: unknown,
+): metadata is SlackInstallationMetadata {
+  return (
+    typeof metadata === "object" &&
+    metadata !== null &&
+    "projectId" in metadata &&
+    typeof metadata.projectId === "string" &&
+    metadata.projectId.length > 0
+  );
+}
+
+/**
+ * Helper function to safely parse and validate Slack installation metadata
+ */
+export function parseSlackInstallationMetadata(
+  metadata: unknown,
+): SlackInstallationMetadata {
+  if (typeof metadata !== "string") {
+    throw new Error("Installation metadata must be a string");
+  }
+
+  let parsedMetadata: unknown;
+  try {
+    parsedMetadata = JSON.parse(metadata);
+  } catch {
+    throw new Error("Failed to parse installation metadata as JSON");
+  }
+
+  if (!isSlackInstallationMetadata(parsedMetadata)) {
+    throw new Error(
+      "Invalid installation metadata: missing or invalid projectId",
+    );
+  }
+
+  return parsedMetadata;
+}
+
+/**
+ * Slack Service Class
+ *
+ * Uses InstallProvider for OAuth flow and metadata-based project mapping.
+ * Much simpler than the previous implementation while maintaining all functionality.
+ */
+export class SlackService {
+  private static instance: SlackService | null = null;
+  private installer: InstallProvider;
+
+  private constructor() {
+    this.installer = new InstallProvider({
+      clientId: env.SLACK_CLIENT_ID!,
+      clientSecret: env.SLACK_CLIENT_SECRET!,
+      stateSecret: env.SLACK_STATE_SECRET!,
+      installUrlOptions: {
+        scopes: ["channels:read", "chat:write", "chat:write.public"],
+      },
+      installationStore: {
+        storeInstallation: async (installation) => {
+          try {
+            const metadata = parseSlackInstallationMetadata(
+              installation.metadata,
+            );
+            const projectId = metadata.projectId;
+
+            logger.info("Storing Slack installation for project", {
+              projectId,
+              teamId: installation.team?.id,
+              teamName: installation.team?.name,
+            });
+
+            // Store by projectId (one integration per project)
+            await prisma.slackIntegration.upsert({
+              where: { projectId },
+              create: {
+                projectId,
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+              update: {
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+            });
+
+            logger.info("Slack installation stored successfully", {
+              projectId,
+              teamId: installation.team?.id,
+            });
+          } catch (error) {
+            logger.error("Failed to store Slack installation", { error });
+            throw error;
+          }
+        },
+
+        fetchInstallation: async (installQuery) => {
+          try {
+            // Handle both teamId and projectId lookups
+            // When SDK calls with teamId, we treat it as projectId
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided");
+            }
+
+            const integration = await prisma.slackIntegration.findFirst({
+              where: {
+                OR: [
+                  { teamId: lookupId }, // Actual team ID lookup
+                  { projectId: lookupId }, // Project ID lookup (our custom usage)
+                ],
+              },
+            });
+
+            if (!integration) {
+              throw new Error("Slack integration not found");
+            }
+
+            // Return full Installation interface as expected by SDK
+            return {
+              team: {
+                id: integration.teamId,
+                name: integration.teamName,
+              },
+              bot: {
+                id: integration.botUserId,
+                token: decrypt(integration.botToken),
+                userId: integration.botUserId,
+                scopes: [],
+              },
+              enterprise: undefined,
+              user: {
+                token: undefined,
+                refreshToken: undefined,
+                expiresAt: undefined,
+                scopes: undefined,
+                id: integration.botUserId,
+              },
+            };
+          } catch (error) {
+            logger.error("Failed to fetch Slack installation", { error });
+            throw error;
+          }
+        },
+
+        deleteInstallation: async (installQuery) => {
+          try {
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided for deletion");
+            }
+
+            await prisma.slackIntegration.deleteMany({
+              where: {
+                OR: [{ teamId: lookupId }, { projectId: lookupId }],
+              },
+            });
+
+            logger.info("Slack installation deleted successfully", {
+              lookupId,
+            });
+          } catch (error) {
+            logger.error("Failed to delete Slack installation", { error });
+            throw error;
+          }
+        },
+      },
+    });
+  }
+
+  /**
+   * Get singleton instance of SlackService
+   */
+  static getInstance(): SlackService {
+    if (!SlackService.instance) {
+      SlackService.instance = new SlackService();
+    }
+    return SlackService.instance;
+  }
+
+  /**
+   * Get the configured InstallProvider instance for OAuth handling
+   */
+  getInstaller(): InstallProvider {
+    return this.installer;
+  }
+
+  /**
+   * Reset the singleton instance (useful for testing)
+   */
+  static resetInstance(): void {
+    SlackService.instance = null;
+  }
+
+  /**
+   * Delete Slack integration for a project
+   */
+  async deleteIntegration(projectId: string): Promise<void> {
+    try {
+      if (!this.installer.installationStore?.deleteInstallation) {
+        throw new Error("Installation store not configured");
+      }
+
+      await this.installer.installationStore.deleteInstallation({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      logger.info("Slack integration deleted for project", { projectId });
+    } catch (error) {
+      logger.error("Failed to delete Slack integration", { error, projectId });
+      throw new Error(
+        `Failed to delete integration: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get WebClient for a specific project
+   */
+  async getWebClientForProject(projectId: string): Promise<WebClient> {
+    try {
+      // Use projectId as the teamId parameter (handled by our fetchInstallation)
+      const auth = await this.installer.authorize({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      if (!auth.botToken) {
+        throw new Error("No bot token found for project");
+      }
+
+      const client = new WebClient(auth.botToken);
+      logger.debug("Created WebClient for project", { projectId });
+
+      return client;
+    } catch (error) {
+      logger.error("Failed to create WebClient for project", {
+        error,
+        projectId,
+      });
+      throw new Error(
+        `Failed to create WebClient: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get channels accessible to the bot
+   */
+  async getChannels(client: WebClient): Promise<SlackChannel[]> {
+    try {
+      const result = await client.conversations.list({
+        exclude_archived: true,
+        types: "public_channel",
+        limit: 200,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Slack API error: ${result.error}`);
+      }
+
+      const channels: SlackChannel[] = (result.channels || []).map(
+        (channel) => ({
+          id: channel.id!,
+          name: channel.name!,
+          isPrivate: channel.is_private || false,
+          isMember: channel.is_member || false,
+        }),
+      );
+
+      logger.debug("Retrieved channels from Slack", {
+        channelCount: channels.length,
+      });
+
+      return channels;
+    } catch (error) {
+      logger.error("Failed to fetch channels", { error });
+      throw new Error(
+        `Failed to fetch channels: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Send a message to a Slack channel
+   */
+  async sendMessage(params: SlackMessageParams): Promise<SlackMessageResponse> {
+    try {
+      const result = await params.client.chat.postMessage({
+        channel: params.channelId,
+        blocks: params.blocks,
+        text: params.text || "Langfuse Notification",
+        unfurl_links: false,
+        unfurl_media: false,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Failed to send message: ${result.error}`);
+      }
+
+      const response = {
+        messageTs: result.ts!,
+        channel: result.channel!,
+      };
+
+      logger.info("Message sent successfully to Slack", {
+        channel: params.channelId,
+        messageTs: response.messageTs,
+      });
+
+      return response;
+    } catch (error) {
+      logger.error("Failed to send message", {
+        error,
+        channelId: params.channelId,
+      });
+      throw new Error(
+        `Failed to send message: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Validate a WebClient instance
+   */
+  async validateClient(client: WebClient): Promise<boolean> {
+    try {
+      const result = await client.auth.test();
+      return result.ok || false;
+    } catch (error) {
+      logger.warn("Client validation failed", { error });
+      return false;
+    }
+  }
+}

packages/shared/src/server/services/email/billingAlert/BillingAlertEmailTemplate.tsx

@@ -0,0 +1,134 @@
+import React from "react";
+import {
+  Body,
+  Button,
+  Container,
+  Head,
+  Heading,
+  Hr,
+  Html,
+  Img,
+  Preview,
+  Section,
+  Text,
+  Tailwind,
+  Row,
+  Column,
+} from "@react-email/components";
+
+interface BillingAlertEmailProps {
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const BillingAlertEmailTemplate = ({
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  return (
+    <Html>
+      <Head />
+      <Preview>
+        Your Langfuse Cloud usage is {`${currentUsage}`} events for the current
+        billing period
+      </Preview>
+      <Tailwind>
+        <Body className="bg-background my-auto mx-auto font-sans">
+          <Container className="mx-auto my-10 w-[465px] rounded border border-solid border-[#eaeaea] p-5">
+            <Section className="mt-8">
+              <Img
+                src="https://static.langfuse.com/langfuse_logo_transactional_email.png"
+                width="40"
+                height="40"
+                alt="Langfuse"
+                className="mx-auto my-0"
+              />
+            </Section>
+
+            <Section>
+              <Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
+                Usage Threshold Exceeded
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                Your organization &quot;{organizationName}&quot; has exceeded
+                the configured billing threshold
+              </Text>
+            </Section>
+
+            <Section className="mt-8">
+              <div className="bg-gray-50 border border-gray-200 rounded-lg p-4">
+                <Row>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Current Usage (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {currentUsage}
+                    </Text>
+                  </Column>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Alert Threshold (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {threshold}
+                    </Text>
+                  </Column>
+                </Row>
+              </div>
+            </Section>
+
+            <Section className="mt-8 text-center">
+              <Button
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
+                href={billingUrl}
+              >
+                View Billing Page and Manage Alerts
+              </Button>
+            </Section>
+
+            <Section className="mt-8">
+              <Heading className="text-black text-[18px] font-semibold">
+                What happens next?
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                • Your current billing cycle continues normally
+                <br />
+                • Charges will appear on your next invoice
+                <br />
+                • You can adjust usage or modify alert thresholds
+                <br />• Contact support if you have questions about your bill
+              </Text>
+            </Section>
+
+            <Hr className="border border-solid border-[#eaeaea] my-[26px] mx-0 w-full" />
+
+            <Section>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                This email was sent to {receiverEmail} regarding billing alerts
+                for &quot;{organizationName}&quot;.
+              </Text>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                Questions? Contact us at{" "}
+                <a
+                  href="mailto:support@langfuse.com"
+                  className="text-blue-600 no-underline"
+                >
+                  support@langfuse.com
+                </a>
+              </Text>
+            </Section>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  );
+};
+
+export default BillingAlertEmailTemplate;

packages/shared/src/server/services/email/billingAlert/sendBillingAlertEmail.ts

@@ -0,0 +1,59 @@
+import { createTransport } from "nodemailer";
+import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
+import { render } from "@react-email/render";
+import { BillingAlertEmailTemplate } from "./BillingAlertEmailTemplate";
+import { logger } from "../../../logger";
+
+export interface BillingAlertEmailProps {
+  env: Partial<
+    Record<"EMAIL_FROM_ADDRESS" | "SMTP_CONNECTION_URL", string | undefined>
+  >;
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const sendBillingAlertEmail = async ({
+  env,
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
+    logger.error(
+      "Missing environment variables for sending billing alert email.",
+    );
+    return;
+  }
+
+  try {
+    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
+
+    const emailSubject = `Langfuse Cloud Billing Alert: ${organizationName} usage exceeded ${threshold} events`;
+    const emailHtml = await render(
+      BillingAlertEmailTemplate({
+        organizationName,
+        currentUsage,
+        threshold,
+        billingUrl,
+        receiverEmail,
+      }),
+    );
+
+    await mailer.sendMail({
+      to: receiverEmail,
+      from: {
+        address: env.EMAIL_FROM_ADDRESS,
+        name: "Langfuse",
+      },
+      subject: emailSubject,
+      html: emailHtml,
+    });
+  } catch (error) {
+    logger.error(`Failed to send billing alert email`, error);
+  }
+};

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -3,6 +3,7 @@ import { OrderByState } from "../../interfaces/orderBy";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { FilterState } from "../../types";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
 import {
   getProjectIdDefaultFilter,
@@ -11,6 +12,7 @@ import {
 import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
   queryClickhouse,
+  getTimeframesTracesAMT,
 } from "../repositories";
 
 export type SessionDataReturnType = {
@@ -22,6 +24,8 @@ export type SessionDataReturnType = {
   trace_count: number;
   trace_tags: string[];
   trace_environment?: string;
+  scores_avg?: Array<Array<[string, number]>>;
+  score_categories?: Array<Array<string>>;
 };
 
 export type SessionWithMetricsReturnType = SessionDataReturnType & {
@@ -157,7 +161,9 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         session_total_cost,
         session_input_usage,
         session_output_usage,
-        session_total_usage`;
+        session_total_usage,
+        scores_avg,
+        score_categories`;
       break;
     default: {
       const exhaustiveCheckDefault: never = select;
@@ -165,7 +171,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     }
   }
 
-  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+  const { tracesFilter, scoresFilter } = getProjectIdDefaultFilter(projectId, {
     tracesPrefix: "s",
   });
 
@@ -174,6 +180,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
   const tracesFilterRes = tracesFilter
     .filter((f) => f.field !== "environment")
     .apply();
+  const scoresFilterRes = scoresFilter.apply();
 
   const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
     (f) =>
@@ -193,20 +200,25 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     );
   }
 
-  const additionalSingleTraceFilter = tracesFilter.find(
-    (f) =>
-      f.field === "bookmarked" ||
-      f.field === "session_id" ||
-      f.field === "environment",
-  );
-
-  if (additionalSingleTraceFilter) {
-    filters.push(additionalSingleTraceFilter);
-  }
+  tracesFilter
+    .filter(
+      (f) =>
+        f.field === "bookmarked" ||
+        f.field === "session_id" ||
+        f.field === "environment",
+    )
+    .forEach((f) => filters.push(f));
 
   const singleTraceFilter =
     filters.length > 0 ? new FilterList(filters).apply() : undefined;
 
+  const requiresScoresJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "scores") !== undefined ||
+    sessionCols.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "scores";
+
   const hasMetricsFilter =
     tracesFilter.find((f) =>
       [
@@ -217,6 +229,8 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         "session_total_usage",
         "session_output_usage",
         "session_input_usage",
+        "scores_avg",
+        "score_categories",
       ].includes(f.field),
     ) ||
     (orderBy &&
@@ -233,115 +247,183 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
 
   const selectMetrics = select === "metrics" || hasMetricsFilter;
 
+  const scoresCte = `scores_agg AS (
+    SELECT
+      project_id,
+      session_id AS score_session_id,
+      -- For numeric scores, use tuples of (name, avg_value)
+      groupArrayIf(
+        tuple(name, avg_value),
+        data_type IN ('NUMERIC', 'BOOLEAN')
+      ) AS scores_avg,
+      -- For categorical scores, use name:value format for improved query performance
+      groupArrayIf(
+        concat(name, ':', string_value),
+        data_type = 'CATEGORICAL' AND notEmpty(string_value)
+      ) AS score_categories
+    FROM (
+      SELECT
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value,
+        avg(value) avg_value
+      FROM scores s FINAL
+      WHERE 
+        project_id = {projectId: String}
+        ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
+      GROUP BY
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value
+      ) tmp
+    GROUP BY
+      project_id, session_id
+  )`;
+
   // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
   const query = `
-    WITH deduplicated_traces AS (
-      SELECT * EXCEPT input, output, metadata
-      FROM traces t
-      WHERE t.session_id IS NOT NULL 
-        AND t.project_id = {projectId: String}
-        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    deduplicated_observations AS (
-        SELECT * 
-        FROM observations o
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        AND o.trace_id IN (
-          SELECT id
-          FROM deduplicated_traces
-        )
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    observations_agg AS (
-      SELECT o.trace_id,
-            count(*) as obs_count,
-            min(o.start_time) as min_start_time,
-            max(o.end_time) as max_end_time,
-            sumMap(usage_details) as sum_usage_details,
-            sumMap(cost_details) as sum_cost_details,
-            anyLast(project_id) as project_id
-      FROM deduplicated_observations o
-      WHERE o.project_id = {projectId: String}
-      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-      GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            anyLast(t.environment) as trace_environment
-            -- Aggregate observations data at session level
-            ${
-              selectMetrics
-                ? `
-            ,
-            sum(o.obs_count) as total_observations,
-            -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
-            -- LLM session started on that date so this behaviour should yield better results.
-            date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage`
-                : ""
-            }
-        FROM deduplicated_traces t
-        ${
-          selectMetrics
-            ? `LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id`
-            : ""
-        }
-        WHERE t.session_id IS NOT NULL
+        WITH ${select === "metrics" || requiresScoresJoin ? `${scoresCte},` : ""}
+        deduplicated_traces AS (
+          SELECT * EXCEPT input, output, metadata
+          FROM __TRACE_TABLE__ t
+          WHERE t.session_id IS NOT NULL 
             AND t.project_id = {projectId: String}
             ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${sqlSelect}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
+            LIMIT 1 BY id, project_id
+        ),
+        deduplicated_observations AS (
+            SELECT * 
+            FROM observations o
+            WHERE o.project_id = {projectId: String}
+            ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+            AND o.trace_id IN (
+              SELECT id
+              FROM deduplicated_traces
+            )
+            ORDER BY event_ts DESC
+            LIMIT 1 BY id, project_id
+        ),
+        observations_agg AS (
+          SELECT o.trace_id,
+                count(*) as obs_count,
+                min(o.start_time) as min_start_time,
+                max(o.end_time) as max_end_time,
+                sumMap(usage_details) as sum_usage_details,
+                sumMap(cost_details) as sum_cost_details,
+                anyLast(project_id) as project_id
+          FROM deduplicated_observations o
+          WHERE o.project_id = {projectId: String}
+          ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+          GROUP BY o.trace_id
+        ),
+        session_data AS (
+            SELECT
+                t.session_id,
+                anyLast(t.project_id) as project_id,
+                max(t.timestamp) as max_timestamp,
+                min(t.timestamp) as min_timestamp,
+                groupArray(t.id) AS trace_ids,
+                groupUniqArray(t.user_id) AS user_ids,
+                count(*) as trace_count,
+                groupUniqArrayArray(t.tags) as trace_tags,
+                anyLast(t.environment) as trace_environment
+                -- Aggregate observations data at session level
+                ${
+                  selectMetrics
+                    ? `,
+                      sum(o.obs_count) as total_observations,
+                      -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
+                      -- LLM session started on that date so this behaviour should yield better results.
+                      date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
+                      sumMap(o.sum_usage_details) as session_usage_details,
+                      sumMap(o.sum_cost_details) as session_cost_details,
+                      ${
+                        select === "metrics" || requiresScoresJoin
+                          ? `groupUniqArrayArray(s.scores_avg) as scores_avg,
+                      groupUniqArrayArray(s.score_categories) as score_categories,`
+                          : ""
+                      }
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+                      sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+                      sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                    : ""
+                }
+            FROM deduplicated_traces t
+            ${
+              selectMetrics
+                ? `LEFT JOIN observations_agg o
+                   ON t.id = o.trace_id AND t.project_id = o.project_id`
+                : ""
+            }
+           ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_agg s on s.project_id = t.project_id and t.session_id = s.score_session_id` : ""}
+            WHERE t.session_id IS NOT NULL
+                AND t.project_id = {projectId: String}
+                ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+            GROUP BY t.session_id
+        )
+        SELECT ${sqlSelect}
+        FROM session_data s
+        WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+        ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+        `;
 
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
+  return measureAndReturn({
+    operationName: "getSessionsTableGeneric",
+    projectId,
+    input: {
+      params: {
+        projectId,
+        limit: limit,
+        offset: limit && page ? limit * page : 0,
+        ...tracesFilterRes.params,
+        ...singleTraceFilter?.params,
+        ...scoresFilterRes.params,
+        ...(traceTimestampFilter
+          ? {
+              observationsStartTime: convertDateToClickhouseDateTime(
+                traceTimestampFilter.value,
+              ),
+            }
+          : {}),
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "sessions-table",
+        projectId,
+        operation_name: "getSessionsTableGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "sessions-table",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      // Extract the timestamp from filter for AMT table selection
+      const fromTimestamp = filter?.find(
+        (f) =>
+          f.column === "min_timestamp" &&
+          (f.operator === ">=" || f.operator === ">"),
+      )?.value as Date | undefined;
+      const traceAmt = getTimeframesTracesAMT(fromTimestamp);
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -479,6 +479,8 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
           feature: "tracing",
           type: "traces-table",
           projectId,
+          experiment_amt: "original",
+          operation_name: "getTracesTableGeneric",
         },
         clickhouseConfigs,
       });
@@ -588,6 +590,8 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
           feature: "tracing",
           type: "traces-table",
           projectId,
+          experiment_amt: "new",
+          operation_name: "getTracesTableGeneric",
         },
         clickhouseConfigs,
       });

packages/shared/src/server/traceDeletionProcessor.ts

@@ -0,0 +1,90 @@
+import { randomUUID } from "crypto";
+import { prisma } from "../db";
+import { TraceDeleteQueue } from "./redis/traceDelete";
+import { QueueJobs } from "./queues";
+import { logger } from "./logger";
+import { env } from "../env";
+
+export interface TraceDeletionProcessorOptions {
+  delayMs?: number; // Default from LANGFUSE_TRACE_DELETE_DELAY_MS env var
+}
+
+/**
+ * Efficient trace deletion processor that batches deletions for better performance.
+ *
+ * This function:
+ * 1. Creates a record in the pending_deletions table for each trace
+ * 2. Sends a deletion event to the queue with a configurable delay
+ * 3. The worker will batch delete all pending traces from ClickHouse
+ * 4. Sets the is_deleted flag to true after successful deletion
+ *
+ * @param projectId - The project ID
+ * @param traceIds - Array of trace IDs to delete
+ * @param options - Configuration options including delay
+ */
+export async function traceDeletionProcessor(
+  projectId: string,
+  traceIds: string[],
+  options: TraceDeletionProcessorOptions = {},
+): Promise<void> {
+  const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS } = options;
+
+  if (traceIds.length === 0) {
+    logger.warn("traceDeletionProcessor called with empty traceIds array", {
+      projectId,
+    });
+    return;
+  }
+
+  logger.info(
+    `Processing trace deletion for ${traceIds.length} traces in project ${projectId}`,
+    {
+      projectId,
+      traceIds,
+      delayMs,
+    },
+  );
+
+  try {
+    // Create pending deletion records for all traces
+    await prisma.pendingDeletion.createMany({
+      data: traceIds.map((traceId) => ({
+        projectId,
+        object: "trace",
+        objectId: traceId,
+        isDeleted: false,
+      })),
+      skipDuplicates: true, // Avoid conflicts if trace is already pending deletion
+    });
+
+    // Get the trace delete queue
+    const traceDeleteQueue = TraceDeleteQueue.getInstance();
+    if (!traceDeleteQueue) {
+      throw new Error("TraceDeleteQueue not available");
+    }
+
+    // Send deletion event with delay
+    await traceDeleteQueue.add(
+      QueueJobs.TraceDelete,
+      {
+        timestamp: new Date(),
+        id: randomUUID(),
+        name: QueueJobs.TraceDelete,
+        payload: {
+          projectId,
+          traceIds,
+        },
+      },
+      {
+        delay: delayMs,
+      },
+    );
+  } catch (error) {
+    logger.error(`Failed to process trace deletion for project ${projectId}`, {
+      projectId,
+      traceIds,
+      error,
+    });
+    throw error;
+  }
+}

packages/shared/src/tableDefinitions/index.ts

@@ -5,3 +5,4 @@ export * from "./mapTracesTable";
 export * from "./mapDashboards";
 export * from "./mapScoresTable";
 export * from "./promptsTable";
+export * from "./mapDatasetRunItemsTable";

packages/shared/src/tableDefinitions/mapDatasetRunItemsTable.ts

@@ -0,0 +1,28 @@
+import { UiColumnMappings } from "./types";
+
+export const datasetRunItemsTableUiColumnDefinitions: UiColumnMappings = [
+  {
+    uiTableName: "Dataset Run ID",
+    uiTableId: "datasetRunId",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."dataset_run_id"',
+  },
+  {
+    uiTableName: "Created At",
+    uiTableId: "createdAt",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."created_at"',
+  },
+  {
+    uiTableName: "Event Timestamp",
+    uiTableId: "eventTs",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."event_ts"',
+  },
+  {
+    uiTableName: "Dataset Item ID",
+    uiTableId: "datasetItemId",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."dataset_item_id"',
+  },
+];

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -138,4 +138,16 @@ export const sessionCols: UiColumnMappings = [
     clickhouseTableName: "traces",
     clickhouseSelect: "session_id",
   },
+  {
+    uiTableName: "Scores (numeric)",
+    uiTableId: "scores",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "scores_avg",
+  },
+  {
+    uiTableName: "Scores (categorical)",
+    uiTableId: "scores",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "score_categories",
+  },
 ];

packages/shared/src/tableDefinitions/sessionsView.ts

@@ -1,6 +1,7 @@
 import {
   type ColumnDefinition,
   type SingleValueOption,
+  type MultiValueOption,
 } from "../tableDefinitions/types";
 import { formatColumnOptions } from "./typeHelpers";
 
@@ -87,11 +88,27 @@ export const sessionsViewCols: ColumnDefinition[] = [
     internal: 't."tags"',
     options: [], // to be filled in at runtime
   },
+  {
+    name: "Scores (numeric)",
+    id: "scores_avg",
+    type: "numberObject",
+    internal: "scores",
+  },
+  {
+    name: "Scores (categorical)",
+    id: "score_categories",
+    type: "categoryOptions",
+    internal: "score_categories",
+    options: [], // to be added at runtime
+    nullable: true,
+  },
 ];
 
 export type SessionOptions = {
   userIds: Array<SingleValueOption>;
   tags: Array<SingleValueOption>;
+  scores_avg?: Array<string>;
+  score_categories?: Array<MultiValueOption>;
 };
 
 export function sessionsTableColsWithOptions(
@@ -104,6 +121,12 @@ export function sessionsTableColsWithOptions(
     if (col.id === "tags") {
       return formatColumnOptions(col, options?.tags ?? []);
     }
+    if (col.id === "scores_avg") {
+      return formatColumnOptions(col, options?.scores_avg ?? []);
+    }
+    if (col.id === "score_categories") {
+      return formatColumnOptions(col, options?.score_categories ?? []);
+    }
     return col;
   });
 }

packages/shared/src/types.ts

@@ -7,7 +7,7 @@ export type FilterCondition = z.infer<typeof singleFilter>;
 export type FilterState = FilterCondition[];
 
 // to be used in the client during editing
-type MakeOptional<T> = {
+export type MakeOptional<T> = {
   [K in keyof T]?: T[K];
 };
 // if key is value, add string as value

packages/shared/src/utils/zod.ts

@@ -41,11 +41,11 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().nonnegative().default(1),
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().positive().lte(100).default(50),
   ),
 };
 
@@ -80,7 +80,7 @@ export const paginationMetaResponseZod = z.object({
   totalPages: z.number().int().nonnegative(),
 });
 
-const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
+export const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
 export const noUrlCheck = (value: string) => !urlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);

web/Dockerfile

@@ -5,7 +5,7 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
-RUN npm install turbo@^2.5.4 --global
+RUN npm install turbo@^2.5.5 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/next.config.mjs

@@ -15,7 +15,7 @@ const cspHeader = `
   default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io;
   script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com https://chat.cdn-plain.com;
   style-src 'self' 'unsafe-inline' https://uptime.betterstack.com https://fonts.googleapis.com;
-  img-src 'self' https: blob: data: http://localhost:*;
+  img-src 'self' https: blob: data: http://localhost:* https://prod-uk-services-workspac-workspacefilespublicbuck-vs4gjqpqjkh6.s3.amazonaws.com https://prod-uk-services-attachm-attachmentsbucket28b3ccf-uwfssb4vt2us.s3.eu-west-2.amazonaws.com https://i0.wp.com;
   font-src 'self';
   frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com;
   worker-src 'self' blob:;
@@ -23,7 +23,7 @@ const cspHeader = `
   base-uri 'self';
   form-action 'self';
   frame-ancestors 'none';
-  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.ingest.us.sentry.io https://*.sentry.io https://uptime.betterstack.com https://chat.uk.plain.com https://*.s3.amazonaws.com;
+  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.ingest.us.sentry.io https://*.sentry.io https://uptime.betterstack.com https://chat.uk.plain.com https://*.s3.amazonaws.com https://prod-uk-services-attachm-attachmentsuploadbucket2-1l2e4906o2asm.s3.eu-west-2.amazonaws.com;
   media-src 'self' https: http://localhost:*;
   ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.85.2",
+  "version": "3.92.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -9,6 +9,7 @@
   "scripts": {
     "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
+    "dev:https": "dotenv -e ../.env -- next dev --experimental-https --experimental-https-key ./localhost+1-key.pem --experimental-https-cert ./localhost+1.pem",
     "lint": "dotenv -e ../.env -- next lint --max-warnings 0",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
     "clean": "rm -rf node_modules",
@@ -83,6 +84,8 @@
     "@radix-ui/react-tooltip": "^1.2.7",
     "@remixicon/react": "^4.2.0",
     "@sentry/nextjs": "^8.52.0",
+    "@slack/oauth": "^2.6.0",
+    "@slack/web-api": "^7.0.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -125,6 +128,7 @@
     "posthog-js": "^1.176.0",
     "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
+    "prism-react-renderer": "^2.4.1",
     "prisma": "^6.10.1",
     "protobufjs": "^7.4.0",
     "rate-limiter-flexible": "^5.0.3",
@@ -137,7 +141,6 @@
     "react-markdown": "^9.0.1",
     "react-resizable-panels": "^2.1.1",
     "react-responsive": "^10.0.0",
-    "react-syntax-highlighter": "^15.6.1",
     "react18-json-view": "^0.2.8-canary.6",
     "recharts": "^2.15.2",
     "remark-gfm": "^4.0.0",
@@ -178,7 +181,6 @@
     "@types/react": "~18.2.79",
     "@types/react-dom": "~18.2.25",
     "@types/react-grid-layout": "^1.3.5",
-    "@types/react-syntax-highlighter": "^15.5.13",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",

web/public/generated/api/openapi.yml

@@ -1421,6 +1421,107 @@ paths:
                         name: My Score
                         value: 0.9
                         environment: default
+  /api/public/llm-connections:
+    get:
+      description: Get all LLM connections in a project
+      operationId: llmConnections_list
+      tags:
+        - LlmConnections
+      parameters:
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedLlmConnections'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+    put:
+      description: Create or update an LLM connection (upsert)
+      operationId: llmConnections_upsert
+      tags:
+        - LlmConnections
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/LlmConnection'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UpsertLlmConnectionRequest'
   /api/public/media/{mediaId}:
     get:
       description: Get a media record
@@ -6225,6 +6326,121 @@ components:
             type: integer
         - $ref: '#/components/schemas/OpenAICompletionUsageSchema'
         - $ref: '#/components/schemas/OpenAIResponseUsageSchema'
+    LlmConnection:
+      title: LlmConnection
+      type: object
+      description: LLM API connection configuration (secrets excluded)
+      properties:
+        id:
+          type: string
+        provider:
+          type: string
+          description: Provider name (e.g., 'openai', 'anthropic')
+        adapter:
+          type: string
+          description: The adapter used to interface with the LLM
+        displaySecretKey:
+          type: string
+          description: Masked version of the secret key for display purposes
+        baseURL:
+          type: string
+          nullable: true
+          description: Custom base URL for the LLM API
+        customModels:
+          type: array
+          items:
+            type: string
+          description: List of custom model names available for this connection
+        withDefaultModels:
+          type: boolean
+          description: Whether to include default models for this adapter
+        extraHeaderKeys:
+          type: array
+          items:
+            type: string
+          description: >-
+            Keys of extra headers sent with requests (values excluded for
+            security)
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+      required:
+        - id
+        - provider
+        - adapter
+        - displaySecretKey
+        - customModels
+        - withDefaultModels
+        - extraHeaderKeys
+        - createdAt
+        - updatedAt
+    PaginatedLlmConnections:
+      title: PaginatedLlmConnections
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/LlmConnection'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    UpsertLlmConnectionRequest:
+      title: UpsertLlmConnectionRequest
+      type: object
+      description: Request to create or update an LLM connection (upsert)
+      properties:
+        provider:
+          type: string
+          description: Provider name (e.g., 'openai', 'anthropic')
+        adapter:
+          $ref: '#/components/schemas/LlmAdapter'
+          description: The adapter used to interface with the LLM
+        secretKey:
+          type: string
+          description: Secret key for the LLM API
+        baseURL:
+          type: string
+          nullable: true
+          description: Custom base URL for the LLM API
+        customModels:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of custom model names
+        withDefaultModels:
+          type: boolean
+          nullable: true
+          description: Whether to include default models
+        extraHeaders:
+          type: object
+          additionalProperties:
+            type: string
+          nullable: true
+          description: Extra headers to send with requests
+        config:
+          nullable: true
+          description: Additional configuration specific to the adapter
+      required:
+        - provider
+        - adapter
+        - secretKey
+    LlmAdapter:
+      title: LlmAdapter
+      type: string
+      enum:
+        - anthropic
+        - openai
+        - azure
+        - bedrock
+        - google-vertex-ai
+        - google-ai-studio
     GetMediaResponse:
       title: GetMediaResponse
       type: object

web/public/generated/postman/collection.json

@@ -1129,6 +1129,82 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Llm Connections",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all LLM connections in a project",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/llm-connections?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "llm-connections"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Upsert",
+          "request": {
+            "description": "Create or update an LLM connection (upsert)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/llm-connections",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "llm-connections"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"provider\": \"example\",\n    \"adapter\": \"anthropic\",\n    \"secretKey\": \"example\",\n    \"baseURL\": \"example\",\n    \"customModels\": [\n        \"example\"\n    ],\n    \"withDefaultModels\": true,\n    \"extraHeaders\": {\n        \"example\": \"example\"\n    },\n    \"config\": \"UNKNOWN\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,

web/src/__tests__/async/automations-trpc.servertest.ts

@@ -12,6 +12,7 @@ import {
   JobConfigState,
   type SafeWebhookActionConfig,
   type WebhookActionConfigWithSecrets,
+  isWebhookAction,
 } from "@langfuse/shared";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { generateWebhookSecret } from "@langfuse/shared/encryption";
@@ -204,11 +205,10 @@ describe("automations trpc", () => {
               "x-api-key": encrypt("secret-api-key-123"),
               authorization: encrypt("Bearer secret-token-456"),
             },
-            secretHeaderKeys: ["x-api-key", "authorization"],
             displayHeaders: {
-              "content-type": "application/json",
-              "x-api-key": "secr***123",
-              authorization: "Bear***456",
+              "content-type": { secret: false, value: "application/json" },
+              "x-api-key": { secret: true, value: "secr***123" },
+              authorization: { secret: true, value: "Bear***456" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -232,13 +232,14 @@ describe("automations trpc", () => {
       });
 
       expect(response).toHaveLength(1);
-      const automationConfig = response[0].action.config;
+      const automationConfig = response[0].action
+        .config as SafeWebhookActionConfig;
 
       // Should have display values, not encrypted values
       expect(automationConfig.displayHeaders).toEqual({
-        "content-type": "application/json",
-        "x-api-key": "secr***123",
-        authorization: "Bear***456",
+        "content-type": { secret: false, value: "application/json" },
+        "x-api-key": { secret: true, value: "secr***123" },
+        authorization: { secret: true, value: "Bear***456" },
       });
 
       // Should NOT have the raw headers with encrypted values
@@ -321,7 +322,9 @@ describe("automations trpc", () => {
             type: "WEBHOOK",
             url: "https://example.com/webhook",
             headers: { "Content-Type": "application/json" },
-            displayHeaders: { "Content-Type": "application/json" },
+            displayHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
             displaySecretKey,
@@ -353,20 +356,22 @@ describe("automations trpc", () => {
         }),
       });
 
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
+
       // check that the action does not have a secret key in the config
-      expect(response.action.config).not.toHaveProperty("secretKey");
-      expect(response.action.config).toHaveProperty("displaySecretKey");
-      expect(response.action.config.url).toBe("https://example.com/webhook");
-      expect(response.action.config).not.toHaveProperty("headers");
-      expect(response.action.config.displayHeaders).toEqual({
-        "Content-Type": "application/json",
+      expect(actionConfig).not.toHaveProperty("secretKey");
+      expect(actionConfig).toHaveProperty("displaySecretKey");
+      expect(actionConfig.url).toBe("https://example.com/webhook");
+      expect(actionConfig).not.toHaveProperty("headers");
+      expect(actionConfig.displayHeaders).toEqual({
+        "Content-Type": { secret: false, value: "application/json" },
       });
-      expect(response.action.config.apiVersion).toEqual({ prompt: "v1" });
-      expect(response.action.config.type).toBe("WEBHOOK");
-      expect(response.action.config.displaySecretKey).toBe(displaySecretKey);
-      expect(response.action.config).not.toHaveProperty("headers");
-      expect(response.action.config).not.toHaveProperty("decryptedHeaders");
-      expect(response.action.config).not.toHaveProperty("requestHeaders");
+      expect(actionConfig.apiVersion).toEqual({ prompt: "v1" });
+      expect(actionConfig.type).toBe("WEBHOOK");
+      expect(actionConfig.displaySecretKey).toBe(displaySecretKey);
+      expect(actionConfig).not.toHaveProperty("headers");
+      expect(actionConfig).not.toHaveProperty("decryptedHeaders");
+      expect(actionConfig).not.toHaveProperty("requestHeaders");
     });
 
     it("should not expose secret headers in single automation response", async () => {
@@ -398,11 +403,10 @@ describe("automations trpc", () => {
               "x-custom": "public-value",
               "x-secret": encrypt("secret-value-789"),
             },
-            secretHeaderKeys: ["x-secret"],
             displayHeaders: {
-              "content-type": "application/json",
-              "x-custom": "public-value",
-              "x-secret": "secr***789",
+              "content-type": { secret: false, value: "application/json" },
+              "x-custom": { secret: false, value: "public-value" },
+              "x-secret": { secret: true, value: "secr***789" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -425,13 +429,13 @@ describe("automations trpc", () => {
         automationId: automation.id,
       });
 
-      const config = response.action.config;
+      const config = response.action.config as SafeWebhookActionConfig;
 
       // Should have display values
       expect(config.displayHeaders).toEqual({
-        "content-type": "application/json",
-        "x-custom": "public-value",
-        "x-secret": "secr***789",
+        "content-type": { secret: false, value: "application/json" },
+        "x-custom": { secret: false, value: "public-value" },
+        "x-secret": { secret: true, value: "secr***789" },
       });
 
       // Should NOT have raw encrypted headers
@@ -488,7 +492,7 @@ describe("automations trpc", () => {
         automationId: automation.id,
       });
 
-      const config = response.action.config;
+      const config = response.action.config as SafeWebhookActionConfig;
 
       // Should have display values with all legacy header values returned
       // Legacy headers are converted to the new format with secret: false
@@ -629,6 +633,7 @@ describe("automations trpc", () => {
         type: "WEBHOOK",
         url: "https://example.com/mixed-headers",
       });
+      expect(isWebhookAction(createdAction as any)).toBe(true);
 
       // Headers should be encrypted for secret ones, plain for others
       const config = createdAction?.config as WebhookActionConfigWithSecrets;
@@ -795,6 +800,85 @@ describe("automations trpc", () => {
         }),
       ).rejects.toThrow("Name is required");
     });
+
+    it("should create a new Slack automation", async () => {
+      const { project, caller } = await prepare();
+
+      // Create Slack integration first
+      await prisma.slackIntegration.create({
+        data: {
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botToken: encrypt("xoxb-test-token"),
+          botUserId: "U123456",
+        },
+      });
+
+      const response = await caller.automations.createAutomation({
+        projectId: project.id,
+        name: "New Slack Automation",
+        eventSource: "prompt",
+        eventAction: ["created"],
+        filter: [],
+        status: JobConfigState.ACTIVE,
+        actionType: "SLACK",
+        actionConfig: {
+          type: "SLACK",
+          channelId: "C123456",
+          channelName: "general",
+          messageTemplate: JSON.stringify([
+            {
+              type: "section",
+              text: { type: "mrkdwn", text: "Custom template" },
+            },
+          ]),
+        },
+      });
+
+      expect(response.trigger).toMatchObject({
+        projectId: project.id,
+        eventSource: "prompt",
+        eventActions: ["created"],
+        status: JobConfigState.ACTIVE,
+      });
+
+      expect(response.action).toMatchObject({
+        projectId: project.id,
+        type: "SLACK",
+        config: expect.objectContaining({
+          type: "SLACK",
+          channelId: "C123456",
+          channelName: "general",
+        }),
+      });
+
+      // Ensure no bot token is exposed in response
+      expect(JSON.stringify(response)).not.toContain("xoxb-");
+    });
+
+    it("should fail to create Slack automation without integration", async () => {
+      const { project, caller } = await prepare();
+
+      await expect(
+        caller.automations.createAutomation({
+          projectId: project.id,
+          name: "Invalid Slack Automation",
+          eventSource: "prompt",
+          eventAction: ["created"],
+          filter: [],
+          status: JobConfigState.ACTIVE,
+          actionType: "SLACK",
+          actionConfig: {
+            type: "SLACK",
+            channelId: "C123456",
+            channelName: "general",
+          },
+        }),
+      ).rejects.toThrow(
+        "Slack integration not found. Please connect your Slack workspace first.",
+      );
+    });
   });
 
   describe("automations.updateAutomation", () => {
@@ -922,9 +1006,9 @@ describe("automations trpc", () => {
               "x-case-key": { secret: false, value: "some-value" },
             },
             displayHeaders: {
-              "content-type": "application/json",
-              "x-old-header": "old-value",
-              "x-case-key": "some-value",
+              "content-type": { secret: false, value: "application/json" },
+              "x-old-header": { secret: false, value: "old-value" },
+              "x-case-key": { secret: false, value: "some-value" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -963,19 +1047,20 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the API response contains safe display values
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" }, // header preserved
         "x-public": { secret: false, value: "new-public-value" }, // new public header
         "x-secret-key": { secret: true, value: "new-...-123" }, // new secret header
         "x-Case-KEY": { secret: false, value: "new-value" }, // matched existing key, but new value
       });
 
-      expect(response.action.config.displayHeaders).not.toHaveProperty(
+      expect(actionConfig.displayHeaders).not.toHaveProperty(
         "x-old-header", // header deleted
       );
-      expect(response.action.config.displayHeaders).not.toHaveProperty(
+      expect(actionConfig.displayHeaders).not.toHaveProperty(
         "x-case-key", // new case replaced the old header name
       );
 
@@ -1083,9 +1168,10 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the API response contains safe display values reflecting the switch
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "x-currently-public": { secret: true, value: "now-...alue" },
         "x-currently-secret": { secret: false, value: "now-public-value" },
       });
@@ -1097,7 +1183,7 @@ describe("automations trpc", () => {
         where: { id: action.id },
       });
 
-      const config = updatedAction?.config as any;
+      const config = updatedAction?.config as WebhookActionConfigWithSecrets;
 
       // x-currently-public should now be encrypted (was plain, now secret)
       expect(config.requestHeaders["x-currently-public"].value).not.toBe(
@@ -1283,14 +1369,13 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the URL was updated
-      expect(response.action.config.url).toBe(
-        "https://example.com/new-webhook-url",
-      );
+      expect(actionConfig.url).toBe("https://example.com/new-webhook-url");
 
       // Verify secret headers were preserved
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" },
         "x-api-key": { secret: true, value: "secr...-123" },
         authorization: { secret: true, value: "Bear...-456" },
@@ -1388,8 +1473,10 @@ describe("automations trpc", () => {
         },
       });
 
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
+
       // Verify the API response
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" },
         "x-api-key": { secret: true, value: "new-...-key" },
       });
@@ -1780,6 +1867,199 @@ describe("automations trpc", () => {
 
       expect(response.count).toBe(3);
     });
+
+    it("should return 0 consecutive failures when lastFailingExecutionId is set", async () => {
+      const { project, caller } = await prepare();
+
+      // Create automation
+      const trigger = await prisma.trigger.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          eventSource: "prompt",
+          eventActions: ["created"],
+          filter: [],
+          status: JobConfigState.INACTIVE, // Disabled due to failures
+        },
+      });
+
+      const { secretKey, displaySecretKey } = generateWebhookSecret();
+      const action = await prisma.action.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          type: "WEBHOOK",
+          config: {
+            type: "WEBHOOK",
+            url: "https://example.com/webhook",
+            requestHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
+            apiVersion: { prompt: "v1" },
+            secretKey: encrypt(secretKey),
+            displaySecretKey,
+            lastFailingExecutionId: "some-failing-execution-id", // This simulates a webhook that was disabled
+          },
+        },
+      });
+
+      const automation = await prisma.automation.create({
+        data: {
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          name: "Test Automation",
+        },
+      });
+
+      // Create failed executions that occurred BEFORE the lastFailingExecutionId
+      await prisma.automationExecution.create({
+        data: {
+          id: "some-failing-execution-id",
+          automationId: automation.id,
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          status: ActionExecutionStatus.ERROR,
+          sourceId: v4(),
+          input: { iteration: 0 },
+          error: "Old failure",
+          createdAt: new Date(Date.now() - 60000), // 1 minute ago
+        },
+      });
+
+      // Create more failed executions that occurred BEFORE the lastFailingExecutionId
+      for (let i = 1; i < 5; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `Old failure ${i}`,
+            createdAt: new Date(Date.now() - (60000 + i * 1000)), // Before the lastFailingExecutionId
+          },
+        });
+      }
+
+      const response = await caller.automations.getCountOfConsecutiveFailures({
+        projectId: project.id,
+        automationId: automation.id,
+      });
+
+      // Should return 0 because all failures occurred before the lastFailingExecutionId
+      expect(response.count).toBe(0);
+    });
+
+    it("should count failures after lastFailingExecutionId correctly", async () => {
+      const { project, caller } = await prepare();
+
+      // Create automation
+      const trigger = await prisma.trigger.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          eventSource: "prompt",
+          eventActions: ["created"],
+          filter: [],
+          status: JobConfigState.ACTIVE,
+        },
+      });
+
+      const { secretKey, displaySecretKey } = generateWebhookSecret();
+      const lastFailingExecutionId = v4();
+      const action = await prisma.action.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          type: "WEBHOOK",
+          config: {
+            type: "WEBHOOK",
+            url: "https://example.com/webhook",
+            requestHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
+            apiVersion: { prompt: "v1" },
+            secretKey: encrypt(secretKey),
+            displaySecretKey,
+            lastFailingExecutionId,
+          },
+        },
+      });
+
+      const automation = await prisma.automation.create({
+        data: {
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          name: "Test Automation",
+        },
+      });
+
+      // Create the lastFailingExecution
+      await prisma.automationExecution.create({
+        data: {
+          id: lastFailingExecutionId,
+          automationId: automation.id,
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          status: ActionExecutionStatus.ERROR,
+          sourceId: v4(),
+          input: { iteration: 0 },
+          error: "Last failing execution",
+          createdAt: new Date(Date.now() - 60000), // 1 minute ago
+        },
+      });
+
+      // Create old failures that should be ignored
+      for (let i = 0; i < 3; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `Old failure ${i}`,
+            createdAt: new Date(Date.now() - (120000 + i * 1000)), // Before the lastFailingExecutionId
+          },
+        });
+      }
+
+      // Create new failures AFTER the lastFailingExecutionId
+      for (let i = 0; i < 2; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `New failure ${i}`,
+            createdAt: new Date(Date.now() - (30000 - i * 1000)), // After the lastFailingExecutionId
+          },
+        });
+      }
+
+      const response = await caller.automations.getCountOfConsecutiveFailures({
+        projectId: project.id,
+        automationId: automation.id,
+      });
+
+      // Should return 2 because only the 2 new failures after lastFailingExecutionId should be counted
+      expect(response.count).toBe(2);
+    });
   });
 
   describe("automations.regenerateWebhookSecret", () => {

web/src/__tests__/async/scim-api.servertest.ts

@@ -18,16 +18,20 @@ const ScimUserSchema = z.object({
   schemas: z.array(z.string()),
   id: z.string(),
   userName: z.string(),
-  name: z.object({
-    formatted: z.string().nullable(),
-  }),
-  emails: z.array(
-    z.object({
-      primary: z.boolean(),
-      value: z.string(),
-      type: z.string(),
-    }),
-  ),
+  name: z
+    .object({
+      formatted: z.string().nullable(),
+    })
+    .optional(),
+  emails: z
+    .array(
+      z.object({
+        primary: z.boolean(),
+        value: z.string(),
+        type: z.string(),
+      }),
+    )
+    .optional(),
   meta: z.object({
     resourceType: z.string(),
     created: z.string().optional(),
@@ -533,6 +537,45 @@ describe("SCIM API", () => {
         expect(result.body.detail).toContain("userName is required");
       });
 
+      it("should create a new user with specified role", async () => {
+        const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "POST",
+          "/api/public/scim/Users",
+          {
+            userName: uniqueEmail,
+            name: {
+              formatted: "Test User With Role",
+            },
+            emails: [
+              {
+                primary: true,
+                value: uniqueEmail,
+                type: "work",
+              },
+            ],
+            active: true,
+            roles: ["ADMIN"],
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          201,
+        );
+
+        expect(response.status).toBe(201);
+        expect(response.body.userName).toBe(uniqueEmail);
+        expect(response.body.name.formatted).toBe("Test User With Role");
+
+        testUserId = response.body.id;
+
+        // Verify the user was created with the specified role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("ADMIN");
+      });
+
       it("should return 409 when user with the same userName already exists", async () => {
         const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
 
@@ -617,6 +660,147 @@ describe("SCIM API", () => {
       });
     });
 
+    describe("PUT /api/public/scim/Users/{id}", () => {
+      beforeEach(async () => {
+        // Create a test user
+        const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
+        const user = await prisma.user.create({
+          data: {
+            email: uniqueEmail,
+            name: "Test User",
+          },
+        });
+        await prisma.organizationMembership.create({
+          data: {
+            userId: user.id,
+            orgId: orgId,
+            role: "NONE",
+          },
+        });
+        testUserId = user.id;
+      });
+
+      it("should deactivate a user when active is false", async () => {
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was removed from the organization
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(0);
+      });
+
+      it("should reactivate a user when active is true", async () => {
+        // First deactivate the user
+        await prisma.organizationMembership.deleteMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: true,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was re-added to the organization with default role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("NONE");
+      });
+
+      it("should reactivate a user with specified role", async () => {
+        // First deactivate the user
+        await prisma.organizationMembership.deleteMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: true,
+            roles: ["MEMBER"],
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was re-added to the organization with specified role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("MEMBER");
+      });
+
+      it("should return 400 when SCIM schema is missing", async () => {
+        const result = await makeAPICall(
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+        );
+        expect(result.status).toBe(400);
+        expect(result.body.detail).toContain("schemas");
+      });
+
+      it("should return 404 when user does not exist", async () => {
+        const nonExistentUserId = randomUUID();
+        const result = await makeAPICall(
+          "PUT",
+          `/api/public/scim/Users/${nonExistentUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: nonExistentUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+        );
+        expect(result.status).toBe(404);
+        expect(result.body.detail).toContain("User not found");
+      });
+    });
+
     describe("DELETE /api/public/scim/Users/{id}", () => {
       beforeEach(async () => {
         // Create a test user

web/src/__tests__/async/sessions-trpc.servertest.ts

@@ -137,6 +137,8 @@ describe("traces trpc", () => {
       // When
       const sessions = await caller.sessions.all({
         projectId,
+        limit: 50,
+        page: 0,
         orderBy: {
           column: "createdAt",
           order: "DESC",
@@ -154,6 +156,55 @@ describe("traces trpc", () => {
       // Then
       expect(sessions.sessions).toBeDefined();
     });
+
+    it("should return empty when filtering by session_id and mismatched environment", async () => {
+      // Setup - create a session with non-default environment
+      const sessionId = randomUUID();
+      const testEnvironment = "staging";
+
+      await prisma.traceSession.create({
+        data: {
+          id: sessionId,
+          projectId,
+          environment: testEnvironment,
+        },
+      });
+
+      const trace = createTrace({
+        project_id: projectId,
+        session_id: sessionId,
+        environment: testEnvironment,
+      });
+      await createTracesCh([trace]);
+
+      // When - filter by correct session_id but wrong environment
+      const sessions = await caller.sessions.all({
+        projectId,
+        limit: 50,
+        page: 0,
+        orderBy: {
+          column: "createdAt",
+          order: "DESC",
+        },
+        filter: [
+          {
+            column: "ID",
+            operator: "=",
+            value: sessionId,
+            type: "string",
+          },
+          {
+            column: "environment",
+            operator: "=",
+            value: "production",
+            type: "string",
+          },
+        ],
+      });
+
+      // Then - should return empty result
+      expect(sessions.sessions).toEqual([]);
+    });
   });
 
   describe("sessions.countAll", () => {
@@ -177,6 +228,8 @@ describe("traces trpc", () => {
       // When
       const sessions = await caller.sessions.countAll({
         projectId,
+        limit: 50,
+        page: 0,
         filter: null,
         orderBy: null,
       });

web/src/__tests__/async/sessions-ui-table.servertest.ts

@@ -4,10 +4,13 @@ import {
   createObservation,
   createObservationsCh,
   createOrgProjectAndApiKey,
+  createScoresCh,
+  createSessionScore,
   createTracesCh,
   getSessionsWithMetrics,
 } from "@langfuse/shared/src/server";
 import { createTrace, getSessionsTable } from "@langfuse/shared/src/server";
+import { type FilterState } from "@langfuse/shared";
 
 describe("trpc.sessions", () => {
   describe("GET sessions.all", () => {
@@ -385,4 +388,52 @@ describe("trpc.sessions", () => {
     expect(sessions[0]?.trace_count).toBe(2);
     expect(parseInt(sessions[0]?.duration as any)).toBe(1);
   });
+  it("should GET correct session data with filters", async () => {
+    const project_id = v4();
+    const trace_id_with_score = v4();
+    const session_id_with_score = v4();
+    const trace_id_without_score = v4();
+    const session_id_without_score = v4();
+
+    const filterState: FilterState = [
+      {
+        type: "numberObject",
+        column: "Scores (numeric)",
+        key: "test",
+        operator: ">",
+        value: 0,
+      },
+    ];
+
+    const trace_with_score = createTrace({
+      id: trace_id_with_score,
+      project_id,
+      session_id: session_id_with_score,
+    });
+    const trace_without_score = createTrace({
+      id: trace_id_without_score,
+      project_id,
+      session_id: session_id_without_score,
+    });
+    await createTracesCh([trace_with_score, trace_without_score]);
+
+    const score = createSessionScore({
+      project_id,
+      session_id: session_id_with_score,
+      name: "test",
+      value: 1,
+      data_type: "NUMERIC",
+    });
+    await createScoresCh([score]);
+
+    const tableRows = await getSessionsTable({
+      projectId: project_id,
+      filter: filterState,
+      limit: 10,
+      page: 0,
+    });
+
+    expect(tableRows).toHaveLength(1);
+    expect(tableRows[0].session_id).toEqual(session_id_with_score);
+  });
 });

web/src/__tests__/async/slack-integration.servertest.ts

@@ -0,0 +1,554 @@
+import { prisma } from "@langfuse/shared/src/db";
+import type { Session } from "next-auth";
+import { encrypt } from "@langfuse/shared/encryption";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import { appRouter } from "@/src/server/api/root";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import { TRPCError } from "@trpc/server";
+
+// Mock SlackService
+jest.mock("@langfuse/shared/src/server", () => {
+  const actual = jest.requireActual("@langfuse/shared/src/server");
+  return {
+    ...actual,
+    SlackService: {
+      getInstance: jest.fn(),
+    },
+  };
+});
+
+const __orgIds: string[] = [];
+let mockSlackService: any;
+
+const prepare = async () => {
+  const { project, org } = await createOrgProjectAndApiKey();
+
+  const session: Session = {
+    expires: "1",
+    user: {
+      id: "user-1",
+      canCreateOrganizations: true,
+      name: "Demo User",
+      organizations: [
+        {
+          id: org.id,
+          name: org.name,
+          role: "OWNER",
+          plan: "cloud:hobby",
+          cloudConfig: undefined,
+          metadata: {},
+          projects: [
+            {
+              id: project.id,
+              role: "ADMIN",
+              retentionDays: 30,
+              deletedAt: null,
+              name: project.name,
+              metadata: {},
+            },
+          ],
+        },
+      ],
+      featureFlags: {
+        excludeClickhouseRead: false,
+        templateFlag: true,
+      },
+      admin: true,
+    },
+    environment: {
+      enableExperimentalFeatures: false,
+      selfHostedInstancePlan: "cloud:hobby",
+    },
+  };
+
+  const ctx = createInnerTRPCContext({ session, headers: {} });
+  const caller = appRouter.createCaller({ ...ctx, prisma });
+
+  __orgIds.push(org.id);
+
+  return { project, org, session, ctx, caller };
+};
+
+describe("Slack Integration", () => {
+  beforeAll(async () => {
+    // Import mocked SlackService
+    const { SlackService } = await import("@langfuse/shared/src/server");
+
+    // Create mock service instance
+    mockSlackService = {
+      getWebClientForProject: jest.fn(),
+      sendMessage: jest.fn(),
+      getChannels: jest.fn(),
+      validateClient: jest.fn(),
+      deleteIntegration: jest.fn(),
+    };
+
+    // Setup the getInstance mock to return our mock service
+    (SlackService.getInstance as jest.Mock).mockReturnValue(mockSlackService);
+  });
+
+  afterAll(async () => {
+    await prisma.organization.deleteMany({
+      where: {
+        id: { in: __orgIds },
+      },
+    });
+  });
+
+  describe("Slack tRPC Router", () => {
+    describe("getIntegrationStatus", () => {
+      it("should return connected status for valid integration", async () => {
+        mockSlackService.validateClient.mockResolvedValue(true);
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-test-token-secret"),
+            botUserId: "U123456",
+          },
+        });
+
+        const result = await caller.slack.getIntegrationStatus({
+          projectId: project.id,
+        });
+
+        expect(result).toMatchObject({
+          isConnected: true,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botUserId: "U123456",
+          installUrl: null,
+        });
+
+        // 🔒 CRITICAL: Ensure no bot token is exposed
+        expect(JSON.stringify(result)).not.toContain("xoxb-test-token-secret");
+        expect(result).not.toHaveProperty("botToken");
+      });
+
+      it("should return disconnected status when no integration exists", async () => {
+        const { caller, project } = await prepare();
+
+        const result = await caller.slack.getIntegrationStatus({
+          projectId: project.id,
+        });
+
+        expect(result).toMatchObject({
+          isConnected: false,
+          teamId: null,
+          teamName: null,
+          installUrl: expect.stringContaining(
+            `/api/public/slack/install?projectId=${project.id}`,
+          ),
+        });
+
+        // 🔒 Ensure no sensitive data is present
+        expect(JSON.stringify(result)).not.toContain("xoxb-");
+      });
+
+      it("should return disconnected status for invalid integration", async () => {
+        mockSlackService.validateClient.mockResolvedValue(false);
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration with invalid token
+        await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-invalid-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        const result = await caller.slack.getIntegrationStatus({
+          projectId: project.id,
+        });
+
+        expect(result).toMatchObject({
+          isConnected: false,
+          teamId: "T123456",
+          teamName: "Test Team",
+          error:
+            "Integration is invalid. Please reconnect your Slack workspace.",
+        });
+
+        // 🔒 CRITICAL: Ensure no bot token is exposed even for invalid integrations
+        expect(JSON.stringify(result)).not.toContain("xoxb-invalid-token");
+      });
+    });
+
+    describe("getChannels", () => {
+      it("should fetch channels for valid integration", async () => {
+        const mockChannels = [
+          { id: "C123456", name: "general", isPrivate: false, isMember: true },
+          { id: "C789012", name: "random", isPrivate: false, isMember: true },
+          {
+            id: "C345678",
+            name: "private-channel",
+            isPrivate: true,
+            isMember: true,
+          },
+        ];
+
+        mockSlackService.getChannels.mockResolvedValue(mockChannels);
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-valid-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        const result = await caller.slack.getChannels({
+          projectId: project.id,
+        });
+
+        expect(result).toMatchObject({
+          channels: mockChannels,
+          teamId: "T123456",
+          teamName: "Test Team",
+        });
+
+        // 🔒 CRITICAL: Ensure no bot token is exposed in channel data
+        expect(JSON.stringify(result)).not.toContain("xoxb-valid-token");
+        expect(result).not.toHaveProperty("botToken");
+      });
+
+      it("should throw NOT_FOUND for missing integration", async () => {
+        const { caller, project } = await prepare();
+
+        await expect(
+          caller.slack.getChannels({ projectId: project.id }),
+        ).rejects.toThrow("Slack integration not found");
+      });
+
+      it("should handle Slack API failures gracefully", async () => {
+        mockSlackService.getChannels.mockRejectedValue(
+          new Error("Slack API error"),
+        );
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-test-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        await expect(
+          caller.slack.getChannels({ projectId: project.id }),
+        ).rejects.toThrow(
+          "Failed to fetch channels. Please check your Slack connection and try again.",
+        );
+      });
+    });
+
+    describe("sendTestMessage", () => {
+      it("should send test message successfully", async () => {
+        const mockClient = { auth: { test: jest.fn() } };
+        mockSlackService.getWebClientForProject.mockResolvedValue(mockClient);
+        mockSlackService.sendMessage.mockResolvedValue({
+          messageTs: "1234567890.123456",
+          channel: "C123456",
+        });
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-test-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        const result = await caller.slack.sendTestMessage({
+          projectId: project.id,
+          channelId: "C123456",
+          channelName: "general",
+        });
+
+        expect(result).toMatchObject({
+          success: true,
+          messageTs: "1234567890.123456",
+          channel: "C123456",
+        });
+
+        // Verify SlackService was called with proper parameters
+        expect(mockSlackService.sendMessage).toHaveBeenCalledWith({
+          client: expect.any(Object),
+          channelId: "C123456",
+          blocks: expect.any(Array),
+          text: "Test message from Langfuse",
+        });
+
+        // 🔒 CRITICAL: Ensure no bot token is exposed in test results
+        expect(JSON.stringify(result)).not.toContain("xoxb-test-token");
+      });
+
+      it("should create audit log entry", async () => {
+        const mockClient = { auth: { test: jest.fn() } };
+        mockSlackService.getWebClientForProject.mockResolvedValue(mockClient);
+        mockSlackService.sendMessage.mockResolvedValue({
+          messageTs: "1234567890.123456",
+          channel: "C123456",
+        });
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        const integration = await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-test-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        await caller.slack.sendTestMessage({
+          projectId: project.id,
+          channelId: "C123456",
+          channelName: "general",
+        });
+
+        // Verify audit log was created
+        const auditLog = await prisma.auditLog.findFirst({
+          where: {
+            projectId: project.id,
+            resourceType: "slackIntegration",
+            resourceId: integration.id,
+            action: "create",
+          },
+        });
+
+        expect(auditLog).toBeDefined();
+        const afterData = auditLog?.after ? JSON.parse(auditLog.after) : null;
+        expect(afterData).toMatchObject({
+          action: "test_message_sent",
+          channelId: "C123456",
+          channelName: "general",
+          messageTs: "1234567890.123456",
+        });
+
+        // 🔒 Ensure audit log doesn't contain tokens
+        expect(JSON.stringify(afterData)).not.toContain("xoxb-");
+      });
+    });
+
+    describe("disconnect", () => {
+      it("should remove integration and audit log it", async () => {
+        mockSlackService.deleteIntegration.mockResolvedValue(undefined);
+
+        const { caller, project } = await prepare();
+
+        // Create Slack integration
+        const integration = await prisma.slackIntegration.create({
+          data: {
+            projectId: project.id,
+            teamId: "T123456",
+            teamName: "Test Team",
+            botToken: encrypt("xoxb-test-token"),
+            botUserId: "U123456",
+          },
+        });
+
+        const result = await caller.slack.disconnect({
+          projectId: project.id,
+        });
+
+        expect(result).toMatchObject({
+          success: true,
+        });
+
+        // Verify SlackService was called
+        expect(mockSlackService.deleteIntegration).toHaveBeenCalledWith(
+          project.id,
+        );
+
+        // Verify audit log was created
+        const auditLog = await prisma.auditLog.findFirst({
+          where: {
+            projectId: project.id,
+            resourceType: "slackIntegration",
+            resourceId: integration.id,
+            action: "delete",
+          },
+        });
+
+        expect(auditLog).toBeDefined();
+        const beforeData = auditLog?.before
+          ? JSON.parse(auditLog.before)
+          : null;
+        expect(beforeData).toMatchObject({
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+        });
+
+        // 🔒 CRITICAL: Ensure audit log doesn't expose encrypted bot token
+        expect(JSON.stringify(beforeData)).not.toContain("xoxb-");
+        // But encrypted token should be in the audit log for recovery purposes
+        expect(beforeData).toHaveProperty("botToken");
+      });
+
+      it("should handle missing integration gracefully", async () => {
+        const { caller, project } = await prepare();
+
+        await expect(
+          caller.slack.disconnect({ projectId: project.id }),
+        ).rejects.toThrow("Slack integration not found");
+      });
+    });
+  });
+
+  describe("Slack Security", () => {
+    it("should encrypt bot tokens in database", async () => {
+      const { project } = await prepare();
+
+      const originalToken = "xoxb-secret-bot-token-12345";
+
+      // Create Slack integration
+      await prisma.slackIntegration.create({
+        data: {
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botToken: encrypt(originalToken),
+          botUserId: "U123456",
+        },
+      });
+
+      // Verify token is encrypted in database
+      const rawIntegration = await prisma.slackIntegration.findUnique({
+        where: { projectId: project.id },
+      });
+
+      expect(rawIntegration?.botToken).toBeDefined();
+      expect(rawIntegration?.botToken).not.toBe(originalToken);
+      expect(rawIntegration?.botToken).not.toContain("xoxb-secret-bot-token");
+
+      // Verify the encrypted token can be decrypted back to original
+      const { decrypt } = await import("@langfuse/shared/encryption");
+      const decryptedToken = decrypt(rawIntegration!.botToken);
+      expect(decryptedToken).toBe(originalToken);
+    });
+
+    it("should NEVER expose raw bot tokens in any API response", async () => {
+      mockSlackService.validateClient.mockResolvedValue(true);
+      mockSlackService.getChannels.mockResolvedValue([
+        { id: "C123456", name: "general", isPrivate: false, isMember: true },
+      ]);
+      mockSlackService.sendMessage.mockResolvedValue({
+        messageTs: "1234567890.123456",
+        channel: "C123456",
+      });
+
+      const { caller, project } = await prepare();
+
+      const secretToken = "xoxb-extremely-secret-token-abcdef123456";
+
+      // Create Slack integration
+      await prisma.slackIntegration.create({
+        data: {
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botToken: encrypt(secretToken),
+          botUserId: "U123456",
+        },
+      });
+
+      // Test all tRPC endpoints
+      const results = await Promise.all([
+        caller.slack.getIntegrationStatus({ projectId: project.id }),
+        caller.slack.getChannels({ projectId: project.id }),
+        caller.slack.sendTestMessage({
+          projectId: project.id,
+          channelId: "C123456",
+          channelName: "general",
+        }),
+      ]);
+
+      results.forEach((result) => {
+        const resultText = JSON.stringify(result);
+
+        // 🔒 CRITICAL: Ensure no raw token appears anywhere in response
+        expect(resultText).not.toContain(secretToken);
+        expect(resultText).not.toContain("xoxb-extremely-secret-token");
+      });
+    });
+
+    it("should NEVER set unencrypted tokens in cookies", async () => {
+      const { caller, project } = await prepare();
+
+      // Test tRPC calls (cookies would be set at HTTP layer, not tRPC layer)
+      const result = await caller.slack.getIntegrationStatus({
+        projectId: project.id,
+      });
+
+      // 🔒 Ensure tRPC result doesn't contain any token fields that could leak to cookies
+      expect(JSON.stringify(result)).not.toContain("xoxb-");
+      expect(result).not.toHaveProperty("botToken");
+      expect(result).not.toHaveProperty("token");
+    });
+
+    it("should sanitize tokens from error messages", async () => {
+      const secretToken = "xoxb-secret-error-token-999";
+
+      // Mock SlackService to throw error containing token
+      mockSlackService.getChannels.mockRejectedValue(
+        new Error(`Authentication failed for token ${secretToken}`),
+      );
+
+      const { caller, project } = await prepare();
+
+      // Create Slack integration
+      await prisma.slackIntegration.create({
+        data: {
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botToken: encrypt(secretToken),
+          botUserId: "U123456",
+        },
+      });
+
+      try {
+        await caller.slack.getChannels({ projectId: project.id });
+        throw new Error("Expected error to be thrown");
+      } catch (error) {
+        const errorMessage =
+          error instanceof TRPCError ? error.message : String(error);
+
+        // 🔒 CRITICAL: Error messages should not contain raw tokens
+        expect(errorMessage).not.toContain(secretToken);
+        expect(errorMessage).not.toContain("xoxb-secret-error-token");
+
+        // Should still indicate there was an error, just sanitized
+        expect(errorMessage).toContain("Failed to fetch channels.");
+      }
+    });
+  });
+});

web/src/components/ChatMessages/ChatMessageComponent.tsx

@@ -156,6 +156,12 @@ export const ChatMessageComponent: React.FC<ChatMessageProps> = ({
             role: nextRole,
             type: ChatMessageType.System,
           });
+        } else if (nextRole === ChatMessageRole.Model) {
+          replaceMessage(message.id, {
+            content: message.content,
+            role: nextRole,
+            type: ChatMessageType.ModelText,
+          });
         } else {
           const exhaustiveCheck: never = nextRole;
           console.error(`Unhandled role: ${exhaustiveCheck}`);

web/src/components/ItemBadge.tsx

@@ -99,12 +99,16 @@ export function ItemBadge({
       variant="outline"
       title={label}
       className={cn(
-        "flex max-w-fit items-center gap-1 border-2 bg-background px-1",
+        "flex max-w-fit items-center gap-1 overflow-hidden whitespace-nowrap border-2 bg-background px-1",
         isSmall && "h-4",
       )}
     >
       <Icon className={iconClass} />
-      {showLabel && <span>{label.replace(/_/g, " ")}</span>}
+      {showLabel && (
+        <span className="truncate" title={label.replace(/_/g, " ")}>
+          {label.replace(/_/g, " ")}
+        </span>
+      )}
     </Badge>
   );
 }

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -46,7 +46,7 @@ export const LLMApiKeyComponent = (p: {
       <Label className="text-xs font-semibold">API key</Label>
       <div>
         {apiKey ? (
-          <Link href={`/project/${p.projectId}/settings/api-keys`}>
+          <Link href={`/project/${p.projectId}/settings/llm-connections`}>
             <span className="mr-2 rounded-sm bg-input p-1 text-xs">
               {apiKey.displaySecretKey}
             </span>

web/src/components/ModelParameters/index.tsx

@@ -33,6 +33,7 @@ export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableProviders: string[];
   availableModels: string[];
+  providerModelCombinations: string[];
   updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
     value: UIModelParams[Key]["value"],
@@ -41,13 +42,14 @@ export type ModelParamsContext = {
   formDisabled?: boolean;
   modelParamsDescription?: string;
   customHeader?: React.ReactNode;
-  layout?: "vertical" | "compact";
+  layout?: "compact" | "vertical";
 };
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
   availableProviders,
   availableModels,
+  providerModelCombinations,
   updateModelParamValue,
   setModelParamEnabled,
   formDisabled = false,
@@ -168,12 +170,6 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   if (layout === "compact") {
     // Create combined options in "Provider: model" format
     // We create combinations of all available providers with all available models
-    const combinedOptions: string[] = [];
-    availableProviders.forEach((provider) => {
-      availableModels.forEach((model) => {
-        combinedOptions.push(`${provider}: ${model}`);
-      });
-    });
 
     // Current combined value in "Provider: model" format
     const currentCombinedValue = `${modelParams.provider.value}: ${modelParams.model.value}`;
@@ -202,7 +198,7 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
                 <SelectValue />
               </SelectTrigger>
               <SelectContent>
-                {combinedOptions.map((option) => (
+                {(providerModelCombinations ?? []).map((option) => (
                   <SelectItem value={option} key={option}>
                     {option}
                   </SelectItem>

web/src/components/layouts/layout.tsx

@@ -119,6 +119,19 @@ export default function Layout(props: PropsWithChildren) {
   // project info based on projectId in the URL
   const { project, organization } = useQueryProjectOrOrganization();
 
+  // Helper function for precise path matching
+  const isPathActive = (routePath: string, currentPath: string): boolean => {
+    // Exact match
+    if (currentPath === routePath) return true;
+
+    // Only allow prefix matching if the route ends with a specific page (not just project root)
+    // This prevents /project/123 from matching /project/123/datasets
+    const isRoot = routePath.split("/").length <= 3;
+    if (isRoot) return false;
+
+    return currentPath.startsWith(routePath + "/");
+  };
+
   const mapNavigation = (route: Route): NavigationItem | null => {
     // Project-level routes
     if (!routerProjectId && route.pathname.includes("[projectId]")) return null;
@@ -196,7 +209,7 @@ export default function Layout(props: PropsWithChildren) {
     return {
       ...route,
       url: url,
-      isActive: router.pathname === route.pathname,
+      isActive: isPathActive(route.pathname, router.pathname),
       items:
         items.length > 0
           ? (items as NavigationItem[]) // does not include null due to filter

web/src/components/session/index.tsx

@@ -25,7 +25,8 @@ import {
 } from "@/src/components/ui/popover";
 import { ScrollArea } from "@/src/components/ui/scroll-area";
 import { Label } from "@/src/components/ui/label";
-import { type APIScoreV2 } from "@langfuse/shared";
+import { AnnotationQueueObjectType, type APIScoreV2 } from "@langfuse/shared";
+import { CreateNewAnnotationQueueItem } from "@/src/features/annotation-queues/components/CreateNewAnnotationQueueItem";
 
 // some projects have thousands of traces in a sessions, paginate to avoid rendering all at once
 const PAGE_SIZE = 50;
@@ -247,17 +248,26 @@ export const SessionPage: React.FC<{
               objectType="SESSION"
               count={sessionCommentCounts.data?.get(sessionId)}
             />
-            <AnnotateDrawer
-              projectId={projectId}
-              scoreTarget={{
-                type: "session",
-                sessionId,
-              }}
-              scores={session.data?.scores ?? []}
-              emptySelectedConfigIds={emptySelectedConfigIds}
-              setEmptySelectedConfigIds={setEmptySelectedConfigIds}
-              buttonVariant="outline"
-            />
+            <div className="flex items-start">
+              <AnnotateDrawer
+                projectId={projectId}
+                scoreTarget={{
+                  type: "session",
+                  sessionId,
+                }}
+                scores={session.data?.scores ?? []}
+                emptySelectedConfigIds={emptySelectedConfigIds}
+                setEmptySelectedConfigIds={setEmptySelectedConfigIds}
+                buttonVariant="outline"
+                hasGroupedButton={true}
+              />
+              <CreateNewAnnotationQueueItem
+                projectId={projectId}
+                objectId={sessionId}
+                objectType={AnnotationQueueObjectType.SESSION}
+                variant="outline"
+              />
+            </div>
           </>
         ),
       }}
@@ -341,7 +351,7 @@ export const SessionPage: React.FC<{
   );
 };
 
-const SessionIO = ({
+export const SessionIO = ({
   traceId,
   projectId,
   timestamp,

web/src/components/table/ValueCell.tsx

@@ -0,0 +1,271 @@
+import { memo } from "react";
+import { type Row } from "@tanstack/react-table";
+import { urlRegex } from "@langfuse/shared";
+import { type JsonTableRow } from "@/src/components/table/utils/jsonExpansionUtils";
+
+const MAX_STRING_LENGTH_FOR_LINK_DETECTION = 1500;
+const MAX_CELL_DISPLAY_CHARS = 2000;
+const SMALL_ARRAY_THRESHOLD = 5;
+const ARRAY_PREVIEW_ITEMS = 3;
+const OBJECT_PREVIEW_KEYS = 2;
+const MONO_TEXT_CLASSES = "font-mono text-xs break-words";
+const PREVIEW_TEXT_CLASSES = "italic text-gray-500 dark:text-gray-400";
+
+function renderStringWithLinks(text: string): React.ReactNode {
+  if (text.length >= MAX_STRING_LENGTH_FOR_LINK_DETECTION) {
+    return text;
+  }
+
+  const localUrlRegex = new RegExp(urlRegex.source, "gi");
+  const parts = text.split(localUrlRegex);
+  const matches = text.match(localUrlRegex) || [];
+
+  const result: React.ReactNode[] = [];
+  let matchIndex = 0;
+
+  for (let i = 0; i < parts.length; i++) {
+    if (parts[i]) {
+      result.push(parts[i]);
+    }
+
+    if (matchIndex < matches.length) {
+      const url = matches[matchIndex];
+      result.push(
+        <a
+          key={`link-${matchIndex}`}
+          href={url}
+          target="_blank"
+          rel="noopener noreferrer"
+          className="hover:opacity-80"
+          onClick={(e) => e.stopPropagation()} // no row expansion when clicking links
+        >
+          {url}
+        </a>,
+      );
+      matchIndex++;
+    }
+  }
+
+  return result;
+}
+
+function getValueType(value: unknown): JsonTableRow["type"] {
+  if (value === null) return "null";
+  if (value === undefined) return "undefined";
+  if (Array.isArray(value)) return "array";
+  return typeof value as JsonTableRow["type"];
+}
+
+function renderArrayValue(arr: unknown[]): JSX.Element {
+  if (arr.length === 0) {
+    return <span className={PREVIEW_TEXT_CLASSES}>empty list</span>;
+  }
+
+  if (arr.length <= SMALL_ARRAY_THRESHOLD) {
+    // Show inline values for small arrays
+    const displayItems = arr
+      .map((item) => {
+        const itemType = getValueType(item);
+        if (itemType === "string") return `"${String(item)}"`;
+        if (itemType === "object" && item !== null) {
+          const obj = item as Record<string, unknown>;
+          const keys = Object.keys(obj);
+          if (keys.length === 0) return "{}";
+          if (keys.length <= OBJECT_PREVIEW_KEYS) {
+            const keyPreview = keys.map((k) => `"${k}": ...`).join(", ");
+            return `{${keyPreview}}`;
+          } else {
+            return `{"${keys[0]}": ...}`;
+          }
+        }
+        if (itemType === "array") return "...";
+        return String(item);
+      })
+      .join(", ");
+    return <span className={PREVIEW_TEXT_CLASSES}>[{displayItems}]</span>;
+  } else {
+    // Show truncated values for large arrays
+    const preview = arr
+      .slice(0, ARRAY_PREVIEW_ITEMS)
+      .map((item) => {
+        const itemType = getValueType(item);
+        if (itemType === "string") return `"${String(item)}"`;
+        if (itemType === "object" || itemType === "array") return "...";
+        return String(item);
+      })
+      .join(", ");
+    return (
+      <span className={PREVIEW_TEXT_CLASSES}>
+        [{preview}, ...{arr.length - ARRAY_PREVIEW_ITEMS} more]
+      </span>
+    );
+  }
+}
+
+function renderObjectValue(obj: Record<string, unknown>): JSX.Element {
+  const keys = Object.keys(obj);
+  if (keys.length === 0) {
+    return <span className={PREVIEW_TEXT_CLASSES}>empty object</span>;
+  }
+  return <span className={PREVIEW_TEXT_CLASSES}>{keys.length} items</span>;
+}
+
+function getValueStringLength(value: unknown): number {
+  if (typeof value === "string") {
+    return value.length;
+  }
+  try {
+    return JSON.stringify(value).length;
+  } catch {
+    return String(value).length;
+  }
+}
+
+function getTruncatedValue(value: string, maxChars: number): string {
+  if (value.length <= maxChars) {
+    return value;
+  }
+
+  const truncated = value.substring(0, maxChars);
+  const lastSpaceIndex = truncated.lastIndexOf(" ");
+
+  // Try to truncate at word boundary if possible
+  if (lastSpaceIndex > maxChars * 0.8) {
+    return truncated.substring(0, lastSpaceIndex) + "...";
+  }
+
+  return truncated + "...";
+}
+
+export const ValueCell = memo(
+  ({
+    row,
+    expandedCells,
+    toggleCellExpansion,
+  }: {
+    row: Row<JsonTableRow>;
+    expandedCells: Set<string>;
+    toggleCellExpansion: (cellId: string) => void;
+  }) => {
+    const { value, type } = row.original;
+    const cellId = `${row.id}-value`;
+    const isCellExpanded = expandedCells.has(cellId);
+
+    const getDisplayValue = () => {
+      switch (type) {
+        case "string": {
+          const stringValue = String(value);
+          const needsTruncation = stringValue.length > MAX_CELL_DISPLAY_CHARS;
+          const displayValue =
+            needsTruncation && !isCellExpanded
+              ? getTruncatedValue(stringValue, MAX_CELL_DISPLAY_CHARS)
+              : stringValue;
+
+          return {
+            content: (
+              <span className="whitespace-pre-line text-green-600 dark:text-green-400">
+                &quot;{renderStringWithLinks(displayValue)}&quot;
+              </span>
+            ),
+            needsTruncation,
+          };
+        }
+        case "number":
+          return {
+            content: (
+              <span className="text-blue-600 dark:text-blue-400">
+                {String(value)}
+              </span>
+            ),
+            needsTruncation: false,
+          };
+        case "boolean":
+          return {
+            content: (
+              <span className="text-orange-600 dark:text-orange-400">
+                {String(value)}
+              </span>
+            ),
+            needsTruncation: false,
+          };
+        case "null":
+          return {
+            content: (
+              <span className="italic text-gray-500 dark:text-gray-400">
+                null
+              </span>
+            ),
+            needsTruncation: false,
+          };
+        case "undefined":
+          return {
+            content: (
+              <span className="text-gray-500 dark:text-gray-400">
+                undefined
+              </span>
+            ),
+            needsTruncation: false,
+          };
+        case "array": {
+          const arrayValue = value as unknown[];
+          // Arrays always show previews, never truncate
+          return {
+            content: renderArrayValue(arrayValue),
+            needsTruncation: false,
+          };
+        }
+        case "object": {
+          const objectValue = value as Record<string, unknown>;
+          // Objects always show previews, never truncate
+          return {
+            content: renderObjectValue(objectValue),
+            needsTruncation: false,
+          };
+        }
+        default: {
+          const stringValue = String(value);
+          const needsTruncation = stringValue.length > MAX_CELL_DISPLAY_CHARS;
+          const displayValue =
+            needsTruncation && !isCellExpanded
+              ? getTruncatedValue(stringValue, MAX_CELL_DISPLAY_CHARS)
+              : stringValue;
+
+          return {
+            content: (
+              <span className="text-gray-600 dark:text-gray-400">
+                {displayValue}
+              </span>
+            ),
+            needsTruncation,
+          };
+        }
+      }
+    };
+
+    const { content, needsTruncation } = getDisplayValue();
+
+    return (
+      <div className={`${MONO_TEXT_CLASSES} max-w-full`}>
+        {content}
+        {needsTruncation && !row.original.hasChildren && (
+          <div
+            className="inline cursor-pointer opacity-50"
+            onClick={(e) => {
+              e.stopPropagation();
+              toggleCellExpansion(cellId);
+            }}
+          >
+            {isCellExpanded
+              ? "\n...collapse"
+              : `\n...expand (${getValueStringLength(value) - MAX_CELL_DISPLAY_CHARS} more characters)`}
+          </div>
+        )}
+      </div>
+    );
+  },
+);
+
+ValueCell.displayName = "ValueCell";
+
+// Export utilities that might be needed elsewhere
+export { getValueStringLength };

web/src/components/table/data-table.tsx

@@ -38,6 +38,7 @@ import { type PeekViewProps } from "@/src/components/table/peek/hooks/usePeekVie
 import { usePeekView } from "@/src/components/table/peek/hooks/usePeekView";
 import { isEqual } from "lodash";
 import { useRouter } from "next/router";
+import { useColumnSizing } from "@/src/components/table/hooks/useColumnSizing";
 
 interface DataTableProps<TData, TValue> {
   columns: LangfuseColumnDef<TData, TValue>[];
@@ -65,6 +66,7 @@ interface DataTableProps<TData, TValue> {
   peekView?: PeekViewProps<TData>;
   pinFirstColumn?: boolean;
   hidePagination?: boolean;
+  tableName: string;
 }
 
 export interface AsyncTableData<T> {
@@ -120,6 +122,7 @@ export function DataTable<TData extends object, TValue>({
   peekView,
   pinFirstColumn = false,
   hidePagination = false,
+  tableName,
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight, customRowHeights);
@@ -136,6 +139,8 @@ export function DataTable<TData extends object, TValue>({
     return flatColumnsByGroup;
   }, [columns]);
 
+  const { columnSizing, setColumnSizing } = useColumnSizing(tableName);
+
   const table = useReactTable({
     data: data.data ?? [],
     columns,
@@ -169,7 +174,9 @@ export function DataTable<TData extends object, TValue>({
         ? insertArrayAfterKey(columnOrder, flattedColumnsByGroup)
         : undefined,
       rowSelection,
+      columnSizing,
     },
+    onColumnSizingChange: setColumnSizing,
     manualFiltering: true,
     defaultColumn: {
       minSize: 20,

web/src/components/table/hooks/useColumnSizing.ts

@@ -0,0 +1,44 @@
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { type ColumnSizingState } from "@tanstack/react-table";
+import { debounce } from "lodash";
+import { useEffect, useMemo, useState } from "react";
+
+/**
+ * Persists table column widths to localStorage with automatic debounced saving.
+ *
+ * @param tableId - Unique identifier for the table to scope column settings
+ * @returns Column sizing state, setter, and utility functions for TanStack Table integration
+ *
+ * @note Limitation: Multi-tab sync occurs only on page refresh/navigation, not in real-time.
+ * @note Column state is initialized once on mount from localStorage.
+ */
+export const useColumnSizing = (tableId: string) => {
+  const [storedSizing, setStoredSizing] = useLocalStorage<ColumnSizingState>(
+    `table-columns-${tableId}`,
+    {},
+  );
+
+  const [columnSizing, setColumnSizing] =
+    useState<ColumnSizingState>(storedSizing);
+
+  // Debounced storage update
+  const debouncedSave = useMemo(
+    () =>
+      debounce((sizing: ColumnSizingState) => {
+        if (tableId && Object.keys(sizing).length > 0) {
+          setStoredSizing(sizing);
+        }
+      }, 500),
+    [tableId, setStoredSizing],
+  );
+
+  // Save to storage when state changes
+  useEffect(() => {
+    debouncedSave(columnSizing);
+  }, [columnSizing, debouncedSave]);
+
+  return {
+    columnSizing,
+    setColumnSizing,
+  };
+};

web/src/components/table/peek/peek-evaluator-config-detail.tsx

@@ -14,6 +14,12 @@ import { LangfuseIcon } from "@/src/components/LangfuseLogo";
 import { UserCircle2Icon } from "lucide-react";
 import { StatusBadge } from "@/src/components/layouts/status-badge";
 import { DeactivateEvalConfig } from "@/src/features/evals/components/deactivate-config";
+import { Switch } from "@/src/components/ui/switch";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useState } from "react";
+import { cn } from "@/src/utils/tailwind";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { api } from "@/src/utils/api";
 
 export const PeekViewEvaluatorConfigDetail = ({
   projectId,
@@ -23,12 +29,16 @@ export const PeekViewEvaluatorConfigDetail = ({
   row?: EvaluatorDataRow;
 }) => {
   const { peekId } = usePeekState();
+  const [isEditMode, setIsEditMode] = useState(false);
+  const utils = api.useUtils();
 
   const { data: evalConfig } = usePeekEvalConfigData({
     jobConfigurationId: peekId,
     projectId,
   });
 
+  const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
+
   if (!evalConfig) {
     return <Skeleton className="h-full w-full" />;
   }
@@ -36,8 +46,8 @@ export const PeekViewEvaluatorConfigDetail = ({
   return (
     <div className="grid h-full flex-1 grid-rows-[auto,auto,1fr] gap-2 overflow-hidden p-3 contain-layout">
       <div className="flex items-center justify-between">
-        <span className="max-h-fit text-lg font-medium">Configuration</span>
-        <div className="flex items-center gap-2">
+        <div className="flex flex-row items-center gap-2">
+          <span className="max-h-fit text-lg font-medium">Configuration</span>
           <div className="flex items-center gap-2">
             <StatusBadge
               type={evalConfig.status.toLowerCase()}
@@ -49,7 +59,22 @@ export const PeekViewEvaluatorConfigDetail = ({
               evalConfig={evalConfig}
             />
           </div>
-          <span className="text-sm text-muted-foreground">View Only</span>
+        </div>
+        <div className="flex items-center gap-2">
+          <span
+            className={cn("text-sm", isEditMode ? "" : "text-muted-foreground")}
+          >
+            Edit Mode
+          </span>
+          <Switch
+            disabled={
+              !hasAccess ||
+              (evalConfig?.timeScope?.length === 1 &&
+                evalConfig.timeScope[0] === "EXISTING")
+            }
+            checked={isEditMode}
+            onCheckedChange={setIsEditMode}
+          />
         </div>
       </div>
       <CardDescription className="flex items-center text-sm">
@@ -76,7 +101,7 @@ export const PeekViewEvaluatorConfigDetail = ({
       </CardDescription>
       <div className="flex max-h-full w-full flex-col items-start justify-between space-y-2 overflow-y-auto pb-4">
         <EvaluatorForm
-          key={evalConfig?.id}
+          key={`${evalConfig?.id}-${evalConfig?.updatedAt}-${isEditMode}`}
           projectId={projectId}
           evalTemplates={
             evalConfig?.evalTemplate ? [evalConfig.evalTemplate] : []
@@ -89,9 +114,18 @@ export const PeekViewEvaluatorConfigDetail = ({
                 }
               : undefined
           }
-          disabled={true}
+          mode="edit"
+          disabled={!isEditMode}
           shouldWrapVariables={true}
           useDialog={false}
+          onFormSuccess={() => {
+            setIsEditMode(false);
+            utils.evals.invalidate();
+            showSuccessToast({
+              title: "Running Evaluator updated",
+              description: "The evaluator configuration has been updated.",
+            });
+          }}
         />
       </div>
     </div>

web/src/components/table/use-cases/models.tsx

@@ -295,6 +295,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
       />
       <SettingsTableCard>
         <DataTable
+          tableName={"models"}
           columns={columns}
           data={
             models.isLoading

web/src/components/table/use-cases/observations.tsx

@@ -1,7 +1,7 @@
 import { api } from "@/src/utils/api";
 import { DataTable } from "@/src/components/table/data-table";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
-import { useEffect, useMemo } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
@@ -15,6 +15,8 @@ import {
   BatchExportTableName,
   type ObservationType,
   TableViewPresetTableName,
+  AnnotationQueueObjectType,
+  BatchActionType,
 } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
@@ -43,7 +45,7 @@ import {
   convertSelectedEnvironmentsToFilter,
 } from "@/src/hooks/use-environment-filter";
 import { Badge } from "@/src/components/ui/badge";
-import { type Row } from "@tanstack/react-table";
+import { type RowSelectionState, type Row } from "@tanstack/react-table";
 import TableIdOrName from "@/src/components/table/table-id";
 import { ItemBadge } from "@/src/components/ItemBadge";
 import { Skeleton } from "@/src/components/ui/skeleton";
@@ -55,6 +57,11 @@ import { useTableViewManager } from "@/src/components/table/table-view-presets/h
 import { useRouter } from "next/router";
 import { useFullTextSearch } from "@/src/components/table/use-cases/useFullTextSearch";
 import { type PeekViewProps } from "@/src/components/table/peek/hooks/usePeekView";
+import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager";
+import { useSelectAll } from "@/src/features/table/hooks/useSelectAll";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { TableActionMenu } from "@/src/features/table/components/TableActionMenu";
+import { type TableAction } from "@/src/features/table/types";
 
 export type ObservationsTableRow = {
   // Shown by default
@@ -117,10 +124,12 @@ export default function ObservationsTable({
   const { viewId } = router.query;
 
   const { setDetailPageList } = useDetailPageLists();
-
+  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
   const { searchQuery, searchType, setSearchQuery, setSearchType } =
     useFullTextSearch();
 
+  const { selectAll, setSelectAll } = useSelectAll(projectId, "observations");
+
   const [paginationState, setPaginationState] = useQueryParams({
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
@@ -272,6 +281,19 @@ export default function ObservationsTable({
     },
   );
 
+  const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
+    onSuccess: (data) => {
+      showSuccessToast({
+        title: "Observations added to queue",
+        description: `Selected observations will be added to queue "${data.queueName}". This may take a minute.`,
+        link: {
+          href: `/project/${projectId}/annotation-queues/${data.queueId}`,
+          text: `View queue "${data.queueName}"`,
+        },
+      });
+    },
+  });
+
   useEffect(() => {
     if (generations.isSuccess) {
       setDetailPageList(
@@ -302,7 +324,54 @@ export default function ObservationsTable({
     );
   };
 
+  const { selectActionColumn } = TableSelectionManager<ObservationsTableRow>({
+    projectId,
+    tableName: "observations",
+    setSelectedRows,
+  });
+
+  const handleAddToAnnotationQueue = async ({
+    projectId,
+    targetId,
+  }: {
+    projectId: string;
+    targetId: string;
+  }) => {
+    const selectedGenerationIds = Object.keys(selectedRows).filter(
+      (generationId) =>
+        generations.data?.generations.map((g) => g.id).includes(generationId),
+    );
+
+    await addToQueueMutation.mutateAsync({
+      projectId,
+      objectIds: selectedGenerationIds,
+      objectType: AnnotationQueueObjectType.OBSERVATION,
+      queueId: targetId,
+      isBatchAction: selectAll,
+      query: {
+        filter: filterState,
+        orderBy: orderByState,
+      },
+    });
+    setSelectedRows({});
+  };
+
+  const tableActions: TableAction[] = [
+    {
+      id: "observation-add-to-annotation-queue",
+      type: BatchActionType.Create,
+      label: "Add to Annotation Queue",
+      description: "Add selected observations to an annotation queue.",
+      targetLabel: "Annotation Queue",
+      execute: handleAddToAnnotationQueue,
+      accessCheck: {
+        scope: "annotationQueues:CUD",
+      },
+    },
+  ];
+
   const columns: LangfuseColumnDef<ObservationsTableRow>[] = [
+    selectActionColumn,
     {
       accessorKey: "startTime",
       id: "startTime",
@@ -975,7 +1044,7 @@ export default function ObservationsTable({
         setRowHeight={setRowHeight}
         selectedOption={selectedOption}
         setDateRangeAndOption={setDateRangeAndOption}
-        actionButtons={
+        actionButtons={[
           <BatchExportTableButton
             {...{
               projectId,
@@ -986,15 +1055,40 @@ export default function ObservationsTable({
             }}
             tableName={BatchExportTableName.Observations}
             key="batchExport"
-          />
-        }
+          />,
+          Object.keys(selectedRows).filter((generationId) =>
+            generations.data?.generations
+              .map((g) => g.id)
+              .includes(generationId),
+          ).length > 0 ? (
+            <TableActionMenu
+              key="observations-multi-select-actions"
+              projectId={projectId}
+              actions={tableActions}
+              tableName={BatchExportTableName.Observations}
+            />
+          ) : null,
+        ]}
         environmentFilter={{
           values: selectedEnvironments,
           onValueChange: setSelectedEnvironments,
           options: environmentOptions.map((env) => ({ value: env })),
         }}
+        multiSelect={{
+          selectAll,
+          setSelectAll,
+          selectedRowIds: Object.keys(selectedRows).filter((generationId) =>
+            generations.data?.generations
+              .map((g) => g.id)
+              .includes(generationId),
+          ),
+          setRowSelection: setSelectedRows,
+          totalCount,
+          ...paginationState,
+        }}
       />
       <DataTable
+        tableName={"observations"}
         columns={columns}
         peekView={peekConfig}
         data={
@@ -1017,6 +1111,8 @@ export default function ObservationsTable({
           onChange: setPaginationState,
           state: paginationState,
         }}
+        rowSelection={selectedRows}
+        setRowSelection={setSelectedRows}
         setOrderBy={setOrderByState}
         orderBy={orderByState}
         columnOrder={columnOrder}

web/src/components/table/use-cases/score-configs.tsx

@@ -254,6 +254,7 @@ export function ScoreConfigsTable({ projectId }: { projectId: string }) {
       />
       <SettingsTableCard>
         <DataTable
+          tableName={"scoreConfigs"}
           columns={columns}
           data={
             configs.isLoading

web/src/components/table/use-cases/scores.tsx

@@ -678,6 +678,7 @@ export default function ScoresTable({
         }}
       />
       <DataTable
+        tableName={"scores"}
         columns={columns}
         data={
           scores.isLoading || isViewLoading

web/src/components/table/use-cases/sessions.tsx

@@ -11,6 +11,8 @@ import {
   sessionsTableColsWithOptions,
   BatchExportTableName,
   TableViewPresetTableName,
+  AnnotationQueueObjectType,
+  BatchActionType,
 } from "@langfuse/shared";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
@@ -19,9 +21,8 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
 import type Decimal from "decimal.js";
-import { useEffect } from "react";
+import { useEffect, useState } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
-import { BatchExportTableButton } from "@/src/components/BatchExportTableButton";
 import { useTableDateRange } from "@/src/hooks/useTableDateRange";
 import { useDebounce } from "@/src/hooks/useDebounce";
 import { joinTableCoreAndMetrics } from "@/src/components/table/utils/joinTableCoreAndMetrics";
@@ -43,6 +44,12 @@ import {
   getScoreGroupColumnProps,
   verifyAndPrefixScoreDataAgainstKeys,
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import { useSelectAll } from "@/src/features/table/hooks/useSelectAll";
+import { type TableAction } from "@/src/features/table/types";
+import { TableActionMenu } from "@/src/features/table/components/TableActionMenu";
+import { type RowSelectionState } from "@tanstack/react-table";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager";
 
 export type SessionTableRow = {
   id: string;
@@ -76,6 +83,7 @@ export default function SessionsTable({
   const { setDetailPageList } = useDetailPageLists();
   const { selectedOption, dateRange, setDateRangeAndOption } =
     useTableDateRange(projectId);
+  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
 
   const [userFilterState, setUserFilterState] = useQueryFilterState(
     [],
@@ -134,6 +142,8 @@ export default function SessionsTable({
     environmentFilter,
   );
 
+  const { selectAll, setSelectAll } = useSelectAll(projectId, "sessions");
+
   const [paginationState, setPaginationState] = useQueryParams({
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
@@ -164,6 +174,19 @@ export default function SessionsTable({
   const sessions = api.sessions.all.useQuery(payloadGetAll);
   const sessionCountQuery = api.sessions.countAll.useQuery(payloadCount);
 
+  const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
+    onSuccess: (data) => {
+      showSuccessToast({
+        title: "Sessions added to queue",
+        description: `Selected sessions will be added to queue "${data.queueName}". This may take a minute.`,
+        link: {
+          href: `/project/${projectId}/annotation-queues/${data.queueId}`,
+          text: `View queue "${data.queueName}"`,
+        },
+      });
+    },
+  });
+
   const { scoreColumns, scoreKeysAndProps, isColumnLoading } =
     useIndividualScoreColumns<SessionTableRow>({
       projectId,
@@ -222,7 +245,53 @@ export default function SessionsTable({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [sessions.isSuccess, sessions.data]);
 
+  const { selectActionColumn } = TableSelectionManager<SessionTableRow>({
+    projectId,
+    tableName: "sessions",
+    setSelectedRows,
+  });
+
+  const handleAddToAnnotationQueue = async ({
+    projectId,
+    targetId,
+  }: {
+    projectId: string;
+    targetId: string;
+  }) => {
+    const selectedSessionIds = Object.keys(selectedRows).filter((sessionId) =>
+      sessions.data?.sessions.map((t) => t.id).includes(sessionId),
+    );
+
+    await addToQueueMutation.mutateAsync({
+      projectId,
+      objectIds: selectedSessionIds,
+      objectType: AnnotationQueueObjectType.SESSION,
+      queueId: targetId,
+      isBatchAction: selectAll,
+      query: {
+        filter: filterState,
+        orderBy: orderByState,
+      },
+    });
+    setSelectedRows({});
+  };
+
+  const tableActions: TableAction[] = [
+    {
+      id: "session-add-to-annotation-queue",
+      type: BatchActionType.Create,
+      label: "Add to Annotation Queue",
+      description: "Add selected sessions to an annotation queue.",
+      targetLabel: "Annotation Queue",
+      execute: handleAddToAnnotationQueue,
+      accessCheck: {
+        scope: "annotationQueues:CUD",
+      },
+    },
+  ];
+
   const columns: LangfuseColumnDef<SessionTableRow>[] = [
+    selectActionColumn,
     {
       accessorKey: "bookmarked",
       id: "bookmarked",
@@ -561,6 +630,18 @@ export default function SessionsTable({
         filterColumnDefinition={transformFilterOptions()}
         filterState={userFilterState}
         setFilterState={useDebounce(setUserFilterState)}
+        actionButtons={[
+          Object.keys(selectedRows).filter((sessionId) =>
+            sessions.data?.sessions.map((s) => s.id).includes(sessionId),
+          ).length > 0 ? (
+            <TableActionMenu
+              key="sessions-multi-select-actions"
+              projectId={projectId}
+              actions={tableActions}
+              tableName={BatchExportTableName.Sessions}
+            />
+          ) : null,
+        ]}
         columns={columns}
         columnVisibility={columnVisibility}
         setColumnVisibility={setColumnVisibility}
@@ -571,18 +652,21 @@ export default function SessionsTable({
           projectId,
           controllers: viewControllers,
         }}
-        actionButtons={[
-          <BatchExportTableButton
-            {...{ projectId, filterState, orderByState }}
-            tableName={BatchExportTableName.Sessions}
-            key="batchExport"
-          />,
-        ]}
         selectedOption={selectedOption}
         setDateRangeAndOption={setDateRangeAndOption}
         columnsWithCustomSelect={["userIds"]}
         rowHeight={rowHeight}
         setRowHeight={setRowHeight}
+        multiSelect={{
+          selectAll,
+          setSelectAll,
+          selectedRowIds: Object.keys(selectedRows).filter((sessionId) =>
+            sessions.data?.sessions.map((s) => s.id).includes(sessionId),
+          ),
+          setRowSelection: setSelectedRows,
+          totalCount,
+          ...paginationState,
+        }}
         environmentFilter={{
           values: selectedEnvironments,
           onValueChange: setSelectedEnvironments,
@@ -590,6 +674,7 @@ export default function SessionsTable({
         }}
       />
       <DataTable
+        tableName={"sessions"}
         columns={columns}
         data={
           sessions.isLoading || isViewLoading
@@ -640,6 +725,8 @@ export default function SessionsTable({
         onColumnVisibilityChange={setColumnVisibility}
         columnOrder={columnOrder}
         onColumnOrderChange={setColumnOrder}
+        rowSelection={selectedRows}
+        setRowSelection={setSelectedRows}
         help={{
           description:
             "A session is a collection of related traces, such as a conversation or thread. To begin, add a sessionId to the trace.",

web/src/components/table/use-cases/traces.tsx

@@ -134,6 +134,7 @@ export type TracesTableProps = {
   hideControls?: boolean;
   externalFilterState?: FilterState;
   externalDateRange?: TableDateRange;
+  limitRows?: number;
 };
 
 export default function TracesTable({
@@ -143,6 +144,7 @@ export default function TracesTable({
   hideControls = false,
   externalFilterState,
   externalDateRange,
+  limitRows,
 }: TracesTableProps) {
   const utils = api.useUtils();
   const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
@@ -246,8 +248,8 @@ export default function TracesTable({
     ...tracesAllCountFilter,
     searchQuery: searchQuery,
     searchType: searchType,
-    page: paginationState.pageIndex,
-    limit: paginationState.pageSize,
+    page: limitRows ? 0 : paginationState.pageIndex,
+    limit: limitRows ?? paginationState.pageSize,
     orderBy: orderByState,
   };
 
@@ -448,6 +450,8 @@ export default function TracesTable({
     },
   ];
 
+  const enableSorting = !hideControls;
+
   const columns: LangfuseColumnDef<TracesTableRow>[] = [
     selectActionColumn,
     ...(hideControls
@@ -474,7 +478,7 @@ export default function TracesTable({
                 />
               ) : undefined;
             },
-            enableSorting: true,
+            enableSorting,
           },
         ]),
     {
@@ -483,7 +487,7 @@ export default function TracesTable({
       id: "timestamp",
       size: 150,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
       cell: ({ row }) => {
         const value: TracesTableRow["timestamp"] = row.getValue("timestamp");
         return value ? <LocalIsoDate date={value} /> : undefined;
@@ -495,7 +499,7 @@ export default function TracesTable({
       id: "name",
       size: 150,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
       cell: ({ row }) => {
         const value: TracesTableRow["name"] = row.getValue("name");
         return value ? (
@@ -583,7 +587,7 @@ export default function TracesTable({
         ) : undefined;
       },
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
 
     {
@@ -612,7 +616,7 @@ export default function TracesTable({
           </BreakdownTooltip>
         );
       },
-      enableSorting: true,
+      enableSorting,
       enableHiding: true,
     },
     {
@@ -637,7 +641,7 @@ export default function TracesTable({
         ) : null;
       },
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "environment",
@@ -736,7 +740,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "userId",
@@ -755,7 +759,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "observationCount",
@@ -798,7 +802,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "version",
@@ -811,7 +815,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "release",
@@ -824,7 +828,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "id",
@@ -840,7 +844,7 @@ export default function TracesTable({
       },
       defaultHidden: true,
       enableHiding: true,
-      enableSorting: true,
+      enableSorting,
     },
     {
       accessorKey: "cost",
@@ -874,7 +878,7 @@ export default function TracesTable({
           },
           defaultHidden: true,
           enableHiding: true,
-          enableSorting: true,
+          enableSorting,
         },
         {
           accessorKey: "outputCost",
@@ -896,7 +900,7 @@ export default function TracesTable({
           },
           enableHiding: true,
           defaultHidden: true,
-          enableSorting: true,
+          enableSorting,
         },
       ],
     },
@@ -924,7 +928,7 @@ export default function TracesTable({
           },
           enableHiding: true,
           defaultHidden: true,
-          enableSorting: true,
+          enableSorting,
         },
         {
           accessorKey: "outputTokens",
@@ -938,7 +942,7 @@ export default function TracesTable({
           },
           enableHiding: true,
           defaultHidden: true,
-          enableSorting: true,
+          enableSorting,
         },
         {
           accessorKey: "totalTokens",
@@ -952,7 +956,7 @@ export default function TracesTable({
           },
           enableHiding: true,
           defaultHidden: true,
-          enableSorting: true,
+          enableSorting,
         },
       ],
     },
@@ -1191,11 +1195,15 @@ export default function TracesTable({
                   data: rows,
                 }
         }
-        pagination={{
-          totalCount,
-          onChange: setPaginationState,
-          state: paginationState,
-        }}
+        pagination={
+          limitRows
+            ? undefined
+            : {
+                totalCount,
+                onChange: setPaginationState,
+                state: paginationState,
+              }
+        }
         setOrderBy={setOrderByState}
         orderBy={orderByState}
         rowSelection={selectedRows}
@@ -1207,6 +1215,7 @@ export default function TracesTable({
         rowHeight={rowHeight}
         pinFirstColumn={!hideControls}
         peekView={peekConfig}
+        tableName={"traces"}
       />
     </>
   );

web/src/components/table/utils/jsonExpansionUtils.ts

@@ -0,0 +1,131 @@
+/**
+ * Utility functions for JSON expansion state management across traces
+ */
+
+// Convert row ID (e.g., "metadata-settings-theme") to key path (e.g., "metadata.settings.theme")
+export function convertRowIdToKeyPath(rowId: string): string {
+  return rowId.replace(/-/g, ".");
+}
+
+// Utility function to get children from lazy-loaded rows
+export function getRowChildren(row: JsonTableRow): JsonTableRow[] {
+  if (row.subRows && row.subRows.length > 0) {
+    return row.subRows;
+  }
+  if (row.rawChildData) {
+    // Prevent infinite recursion by limiting depth; 25 levels of nesting should make a reasonable assumption
+    if (row.level > 25) {
+      return [];
+    }
+    return transformJsonToTableData(
+      row.rawChildData,
+      row.key,
+      row.level + 1,
+      row.id,
+      false, // Don't lazy load for child generation
+    );
+  }
+  return [];
+}
+
+// Types for JSON table rows
+export interface JsonTableRow {
+  id: string;
+  key: string;
+  value: unknown;
+  type:
+    | "string"
+    | "number"
+    | "boolean"
+    | "object"
+    | "array"
+    | "null"
+    | "undefined";
+  hasChildren: boolean;
+  level: number;
+  subRows?: JsonTableRow[];
+  // For lazy loading of sub-row table data
+  rawChildData?: unknown;
+  childrenGenerated?: boolean;
+}
+
+function getValueType(value: unknown): JsonTableRow["type"] {
+  if (value === null) return "null";
+  if (value === undefined) return "undefined";
+  if (Array.isArray(value)) return "array";
+  return typeof value as JsonTableRow["type"];
+}
+
+function hasChildren(value: unknown, valueType: JsonTableRow["type"]): boolean {
+  return (
+    (valueType === "object" &&
+      Object.keys(value as Record<string, unknown>).length > 0) ||
+    (valueType === "array" && Array.isArray(value) && value.length > 0)
+  );
+}
+
+export function transformJsonToTableData(
+  json: unknown,
+  parentKey = "",
+  level = 0,
+  parentId = "",
+  lazy = false,
+): JsonTableRow[] {
+  const rows: JsonTableRow[] = [];
+
+  if (typeof json !== "object" || json === null) {
+    return [
+      {
+        id: parentId || "0",
+        key: parentKey || "root",
+        value: json,
+        type: getValueType(json),
+        hasChildren: false,
+        level,
+      },
+    ];
+  }
+
+  const entries = Array.isArray(json)
+    ? json.map((item, index) => [index.toString(), item])
+    : Object.entries(json);
+
+  entries.forEach(([key, value]) => {
+    const id = parentId ? `${parentId}-${key}` : key;
+    const valueType = getValueType(value);
+    const childrenExist = hasChildren(value, valueType);
+
+    const row: JsonTableRow = {
+      id,
+      key,
+      value,
+      type: valueType,
+      hasChildren: childrenExist,
+      level,
+      childrenGenerated: false,
+    };
+
+    if (childrenExist) {
+      if (lazy && level === 0) {
+        // For lazy loading, store raw data instead of processing children
+        row.rawChildData = value;
+        row.subRows = []; // Empty initially
+      } else {
+        // Normal processing or nested children
+        const children = transformJsonToTableData(
+          value,
+          key,
+          level + 1,
+          id,
+          lazy,
+        );
+        row.subRows = children;
+        row.childrenGenerated = true;
+      }
+    }
+
+    rows.push(row);
+  });
+
+  return rows;
+}

web/src/components/trace/IOPreview.tsx

@@ -15,6 +15,7 @@ import { MarkdownJsonView } from "@/src/components/ui/MarkdownJsonView";
 import { SubHeaderLabel } from "@/src/components/layouts/header";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import useLocalStorage from "@/src/components/useLocalStorage";
 
 export const IOPreview: React.FC<{
   input?: Prisma.JsonValue;
@@ -26,6 +27,14 @@ export const IOPreview: React.FC<{
   hideInput?: boolean;
   currentView?: "pretty" | "json";
   setIsPrettyViewAvailable?: (value: boolean) => void;
+  inputExpansionState?: Record<string, boolean> | boolean;
+  outputExpansionState?: Record<string, boolean> | boolean;
+  onInputExpansionChange?: (
+    expansion: Record<string, boolean> | boolean,
+  ) => void;
+  onOutputExpansionChange?: (
+    expansion: Record<string, boolean> | boolean,
+  ) => void;
 }> = ({
   isLoading = false,
   hideIfNull = false,
@@ -33,11 +42,15 @@ export const IOPreview: React.FC<{
   hideInput = false,
   media,
   currentView,
+  inputExpansionState,
+  outputExpansionState,
+  onInputExpansionChange,
+  onOutputExpansionChange,
   ...props
 }) => {
-  const [localCurrentView, setLocalCurrentView] = useState<"pretty" | "json">(
-    "pretty",
-  );
+  const [localCurrentView, setLocalCurrentView] = useLocalStorage<
+    "pretty" | "json"
+  >("jsonViewPreference", "pretty");
   const selectedView = currentView ?? localCurrentView;
   const capture = usePostHogClientCapture();
   const input = deepParseJson(props.input);
@@ -123,62 +136,99 @@ export const IOPreview: React.FC<{
           </Tabs>
         </div>
       ) : null}
-      {isPrettyViewAvailable && selectedView === "pretty" ? (
+      {/* Always render components to preserve state, just hide via CSS*/}
+      {isPrettyViewAvailable ? (
         <>
-          {inChatMlArray.success ? (
-            <OpenAiMessageView
-              messages={[
-                ...inChatMlArray.data,
-                ...(outChatMlArray.success
-                  ? outChatMlArray.data.map((m) => ({
-                      ...m,
-                      role: m.role ?? "assistant",
-                    }))
-                  : [
-                      {
-                        role: "assistant",
-                        ...(typeof outputClean === "string"
-                          ? { content: outputClean }
-                          : { json: outputClean }),
-                      } as ChatMlMessageSchema,
-                    ]),
-              ]}
-              shouldRenderMarkdown
-              additionalInput={
-                Object.keys(additionalInput ?? {}).length > 0
-                  ? additionalInput
-                  : undefined
-              }
-              media={media ?? []}
-              currentView={selectedView}
-            />
-          ) : (
-            <>
-              {!(hideIfNull && !input) && !hideInput ? (
-                <PrettyJsonView
-                  title="Input"
-                  className="ph-no-capture"
-                  json={input ?? null}
-                  isLoading={isLoading}
-                  media={media?.filter((m) => m.field === "input") ?? []}
-                  currentView={selectedView}
-                />
-              ) : null}
-              {!(hideIfNull && !output) && !hideOutput ? (
-                <PrettyJsonView
-                  title="Output"
-                  className="ph-no-capture"
-                  json={outputClean}
-                  isLoading={isLoading}
-                  media={media?.filter((m) => m.field === "output") ?? []}
-                  currentView={selectedView}
-                />
-              ) : null}
-            </>
-          )}
+          {/* Pretty view content */}
+          <div
+            style={{ display: selectedView === "pretty" ? "block" : "none" }}
+          >
+            {inChatMlArray.success ? (
+              <OpenAiMessageView
+                messages={[
+                  ...inChatMlArray.data,
+                  ...(outChatMlArray.success
+                    ? outChatMlArray.data.map((m) => ({
+                        ...m,
+                        role: m.role ?? "assistant",
+                      }))
+                    : [
+                        {
+                          role: "assistant",
+                          ...(typeof outputClean === "string"
+                            ? { content: outputClean }
+                            : { json: outputClean }),
+                        } as ChatMlMessageSchema,
+                      ]),
+                ]}
+                shouldRenderMarkdown
+                additionalInput={
+                  Object.keys(additionalInput ?? {}).length > 0
+                    ? additionalInput
+                    : undefined
+                }
+                media={media ?? []}
+                currentView={selectedView}
+              />
+            ) : (
+              <>
+                {!(hideIfNull && !input) && !hideInput ? (
+                  <PrettyJsonView
+                    title="Input"
+                    className="ph-no-capture"
+                    json={input ?? null}
+                    isLoading={isLoading}
+                    media={media?.filter((m) => m.field === "input") ?? []}
+                    currentView={selectedView}
+                    externalExpansionState={inputExpansionState}
+                    onExternalExpansionChange={onInputExpansionChange}
+                  />
+                ) : null}
+                {!(hideIfNull && !output) && !hideOutput ? (
+                  <PrettyJsonView
+                    title="Output"
+                    className="ph-no-capture"
+                    json={outputClean}
+                    isLoading={isLoading}
+                    media={media?.filter((m) => m.field === "output") ?? []}
+                    currentView={selectedView}
+                    externalExpansionState={outputExpansionState}
+                    onExternalExpansionChange={onOutputExpansionChange}
+                  />
+                ) : null}
+              </>
+            )}
+          </div>
+
+          {/* JSON view content */}
+          <div style={{ display: selectedView === "json" ? "block" : "none" }}>
+            {!(hideIfNull && !input) && !hideInput ? (
+              <PrettyJsonView
+                title="Input"
+                className="ph-no-capture"
+                json={input ?? null}
+                isLoading={isLoading}
+                media={media?.filter((m) => m.field === "input") ?? []}
+                currentView={selectedView}
+                externalExpansionState={inputExpansionState}
+                onExternalExpansionChange={onInputExpansionChange}
+              />
+            ) : null}
+            {!(hideIfNull && !output) && !hideOutput ? (
+              <PrettyJsonView
+                title="Output"
+                className="ph-no-capture"
+                json={outputClean}
+                isLoading={isLoading}
+                media={media?.filter((m) => m.field === "output") ?? []}
+                currentView={selectedView}
+                externalExpansionState={outputExpansionState}
+                onExternalExpansionChange={onOutputExpansionChange}
+              />
+            ) : null}
+          </div>
         </>
-      ) : null}
-      {selectedView === "json" || !isPrettyViewAvailable ? (
+      ) : (
         <>
           {!(hideIfNull && !input) && !hideInput ? (
             <PrettyJsonView
@@ -188,6 +238,8 @@ export const IOPreview: React.FC<{
               isLoading={isLoading}
               media={media?.filter((m) => m.field === "input") ?? []}
               currentView={selectedView}
+              externalExpansionState={inputExpansionState}
+              onExternalExpansionChange={onInputExpansionChange}
             />
           ) : null}
           {!(hideIfNull && !output) && !hideOutput ? (
@@ -198,10 +250,12 @@ export const IOPreview: React.FC<{
               isLoading={isLoading}
               media={media?.filter((m) => m.field === "output") ?? []}
               currentView={selectedView}
+              externalExpansionState={outputExpansionState}
+              onExternalExpansionChange={onOutputExpansionChange}
             />
           ) : null}
         </>
-      ) : null}
+      )}
     </>
   );
 };
@@ -269,52 +323,77 @@ export const OpenAiMessageView: React.FC<{
             .map((message, index) => (
               <Fragment key={index}>
                 {isPlaceholderMessage(message) ? (
-                  shouldRenderMarkdown ? (
-                    <MarkdownJsonView
-                      title="Placeholder"
-                      content={message.name || "Unnamed placeholder"}
-                      customCodeHeaderClassName={cn("bg-primary-foreground")}
-                    />
-                  ) : (
-                    <PrettyJsonView
-                      title="Placeholder"
-                      json={message.name || "Unnamed placeholder"}
-                      projectIdForPromptButtons={projectIdForPromptButtons}
-                      currentView={currentView}
-                    />
-                  )
+                  <>
+                    <div
+                      style={{
+                        display: shouldRenderMarkdown ? "block" : "none",
+                      }}
+                    >
+                      <MarkdownJsonView
+                        title="Placeholder"
+                        content={message.name || "Unnamed placeholder"}
+                        customCodeHeaderClassName={cn("bg-primary-foreground")}
+                      />
+                    </div>
+                    <div
+                      style={{
+                        display: shouldRenderMarkdown ? "none" : "block",
+                      }}
+                    >
+                      <PrettyJsonView
+                        title="Placeholder"
+                        json={message.name || "Unnamed placeholder"}
+                        projectIdForPromptButtons={projectIdForPromptButtons}
+                        currentView={currentView}
+                      />
+                    </div>
+                  </>
                 ) : (
                   <>
-                    {shouldRenderContent(message) &&
-                      (shouldRenderMarkdown ? (
-                        <MarkdownJsonView
-                          title={message.name ?? message.role}
-                          content={message.content || '""'}
-                          className={cn(
-                            !!message.json &&
-                              !isPlaceholderMessage(message) &&
-                              "rounded-b-none",
-                          )}
-                          customCodeHeaderClassName={cn(
-                            message.role === "assistant" && "bg-secondary",
-                            message.role === "system" &&
-                              "bg-primary-foreground",
-                          )}
-                          audio={message.audio}
-                        />
-                      ) : (
-                        <PrettyJsonView
-                          title={message.name ?? message.role}
-                          json={message.content}
-                          projectIdForPromptButtons={projectIdForPromptButtons}
-                          className={cn(
-                            !!message.json &&
-                              !isPlaceholderMessage(message) &&
-                              "rounded-b-none",
-                          )}
-                          currentView={currentView}
-                        />
-                      ))}
+                    {shouldRenderContent(message) && (
+                      <>
+                        <div
+                          style={{
+                            display: shouldRenderMarkdown ? "block" : "none",
+                          }}
+                        >
+                          <MarkdownJsonView
+                            title={message.name ?? message.role}
+                            content={message.content || '""'}
+                            className={cn(
+                              !!message.json &&
+                                !isPlaceholderMessage(message) &&
+                                "rounded-b-none",
+                            )}
+                            customCodeHeaderClassName={cn(
+                              message.role === "assistant" && "bg-secondary",
+                              message.role === "system" &&
+                                "bg-primary-foreground",
+                            )}
+                            audio={message.audio}
+                          />
+                        </div>
+                        <div
+                          style={{
+                            display: shouldRenderMarkdown ? "none" : "block",
+                          }}
+                        >
+                          <PrettyJsonView
+                            title={message.name ?? message.role}
+                            json={message.content}
+                            projectIdForPromptButtons={
+                              projectIdForPromptButtons
+                            }
+                            className={cn(
+                              !!message.json &&
+                                !isPlaceholderMessage(message) &&
+                                "rounded-b-none",
+                            )}
+                            currentView={currentView}
+                          />
+                        </div>
+                      </>
+                    )}
                     {shouldRenderJson(message) &&
                       !isPlaceholderMessage(message) && (
                         <PrettyJsonView