chore: release v3.95.2

push

.devcontainer.json

@@ -1,6 +1,6 @@
 {
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+  "name": "langfuse-development",
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
 }

.env.dev-azure.example

@@ -68,6 +68,7 @@ LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
 LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 LANGFUSE_USE_AZURE_BLOB=true
+LANGFUSE_AZURE_SKIP_CONTAINER_CHECK=false
 
 # Set during docker build of application
 # Used to disable environment verification at build time

.env.dev-redis-cluster.example

@@ -82,3 +82,11 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+# Use the following settings to enforce running the new AMTs during the tests
+LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT="true"
+LANGFUSE_EXPERIMENT_SAMPLING_RATE=1

.env.dev.example

@@ -83,3 +83,8 @@ NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 # For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10
+
+# Slack credentials for development
+SLACK_CLIENT_ID=your_slack_client_id
+SLACK_CLIENT_SECRET=your_slack_client_secret
+SLACK_STATE_SECRET=your_slack_state_secret

.env.prod.example

@@ -172,6 +172,7 @@ OTEL_SERVICE_NAME="langfuse"
 # REDIS_HOST=
 # REDIS_PORT=
 # REDIS_AUTH=
+# REDIS_USERNAME=default
 # REDIS_CONNECTION_STRING=
 # REDIS_ENABLE_AUTO_PIPELINING=
 

.github/workflows/pipeline.yml

@@ -114,7 +114,9 @@ jobs:
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Build and run both images from compose
         run: |
-          docker compose -f docker-compose.build.yml up -d
+          docker compose --progress plain --verbose -f docker-compose.build.yml build --print > /tmp/bake.json
+          docker buildx bake -f /tmp/bake.json
+          docker compose --progress plain -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
       - name: Ensure no unhealthy status
         run: |
@@ -490,6 +492,14 @@ jobs:
         if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
+      - name: Notify Slack
+        uses: ravsamhq/notify-slack-action@v2
+        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   push-docker-image:
     needs: all-ci-passed

.github/workflows/trivy.yml

@@ -1,43 +0,0 @@
-name: Trivy Container Security
-on:
-  push:
-    branches: ["main"]
-
-jobs:
-  trivy-web:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Run Trivy vulnerability scanner (web)
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "langfuse/langfuse"
-          format: "sarif"
-          output: "trivy-web.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab (web)
-        if: always() && hashFiles('trivy-web.sarif') != ''
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "trivy-web.sarif"
-          category: trivy-web
-
-  trivy-worker:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Run Trivy vulnerability scanner (worker)
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "langfuse/langfuse-worker"
-          format: "sarif"
-          output: "trivy-worker.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab (worker)
-        if: always() && hashFiles('trivy-worker.sarif') != ''
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "trivy-worker.sarif"
-          category: trivy-worker

.gitignore

@@ -50,7 +50,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 
-/generated/typescript-server
+/generated
 
 # openapi spec that is copied during build
 /public/openapi*.yml

CLAUDE.md

@@ -192,3 +192,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## General Coding Guidelines
 - For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so
+
+## Development Tips
+- Before trying to build the package, try running the linter once first

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     ports:
       - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
       DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
       SALT: "mysalt" # CHANGEME
       ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
@@ -73,7 +74,6 @@ services:
       - 3000:3000
     environment:
       <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
       NEXTAUTH_SECRET: mysecret # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}

fern/apis/server/definition/annotation-queues.yml

@@ -105,6 +105,28 @@ service:
           docs: The unique identifier of the annotation queue item
       response: DeleteAnnotationQueueItemResponse
 
+    createQueueAssignment:
+      docs: Create an assignment for a user to an annotation queue
+      method: POST
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: CreateAnnotationQueueAssignmentResponse
+
+    deleteQueueAssignment:
+      docs: Delete an assignment for a user to an annotation queue
+      method: DELETE
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: DeleteAnnotationQueueAssignmentResponse
+
 types:
   AnnotationQueueStatus:
     enum:
@@ -163,3 +185,17 @@ types:
     properties:
       success: boolean
       message: string
+
+  AnnotationQueueAssignmentRequest:
+    properties:
+      userId: string
+
+  DeleteAnnotationQueueAssignmentResponse:
+    properties:
+      success: boolean
+
+  CreateAnnotationQueueAssignmentResponse:
+    properties:
+      userId: string
+      queueId: string
+      projectId: string

fern/apis/server/definition/llm-connections.yml

@@ -0,0 +1,102 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    list:
+      method: GET
+      docs: Get all LLM connections in a project
+      path: /llm-connections
+      request:
+        name: GetLlmConnectionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedLlmConnections
+    upsert:
+      method: PUT
+      docs: Create or update an LLM connection. The connection is upserted on provider.
+      path: /llm-connections
+      request: UpsertLlmConnectionRequest
+      response: LlmConnection
+
+types:
+  LlmConnection:
+    docs: LLM API connection configuration (secrets excluded)
+    properties:
+      id: string
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: string
+        docs: The adapter used to interface with the LLM
+      displaySecretKey:
+        type: string
+        docs: Masked version of the secret key for display purposes
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: list<string>
+        docs: List of custom model names available for this connection
+      withDefaultModels:
+        type: boolean
+        docs: Whether to include default models for this adapter
+      extraHeaderKeys:
+        type: list<string>
+        docs: Keys of extra headers sent with requests (values excluded for security)
+      createdAt: datetime
+      updatedAt: datetime
+
+  PaginatedLlmConnections:
+    properties:
+      data: list<LlmConnection>
+      meta: pagination.MetaResponse
+
+  UpsertLlmConnectionRequest:
+    docs: Request to create or update an LLM connection (upsert)
+    properties:
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: LlmAdapter
+        docs: The adapter used to interface with the LLM
+      secretKey:
+        type: string
+        docs: Secret key for the LLM API.
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: optional<list<string>>
+        docs: List of custom model names
+      withDefaultModels:
+        type: optional<boolean>
+        docs: Whether to include default models. Default is true.
+      extraHeaders:
+        type: optional<map<string, string>>
+        docs: Extra headers to send with requests
+
+  LlmAdapter:
+    enum:
+      - value: anthropic
+        name: Anthropic
+      - value: openai
+        name: OpenAI
+      - value: azure
+        name: Azure
+      - value: bedrock
+        name: Bedrock
+      - value: google-vertex-ai
+        name: GoogleVertexAI
+      - value: google-ai-studio
+        name: GoogleAIStudio

fern/apis/server/generators.yml

@@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://schema.buildwithfern.dev/generators-yml.json
 default-group: local
 groups:
   local:
@@ -7,6 +8,7 @@ groups:
         output:
           location: local-file-system
           path: ../../../web/public/generated/api
+
       - name: fernapi/fern-python-sdk
         version: 2.16.0
         output:
@@ -19,35 +21,32 @@ groups:
           pydantic_config:
             require_optional_fields: false
             use_str_enums: false
-#      - name: fernapi/fern-java-sdk
-#        version: 2.20.1
-#        output:
-#          location: local-file-system
-#          path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
-#        config:
-#          client-class-name: LangfuseClient
+
+      # - name: fernapi/fern-java-sdk
+      #   version: 2.20.1
+      #   output:
+      #     location: local-file-system
+      #     path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
+      #   config:
+      #     client-class-name: LangfuseClient
+
+      - name: fernapi/fern-typescript-node-sdk
+        version: 2.6.1
+        output:
+          location: local-file-system
+          path: ../../../generated/typescript
+        config:
+          namespaceExport: LangfuseAPI
+          outputSourceFiles: true
+          skipResponseValidation: true
+          fetchSupport: native
+          formDataSupport: Node18
+          fileResponseType: binary-response
+          streamType: web
+          omitFernHeaders: true
+
       - name: fernapi/fern-postman
         version: 0.0.45
         output:
           location: local-file-system
           path: ../../../web/public/generated/postman
-  # published:
-  #   generators:
-  #     - name: fernapi/fern-python-sdk
-  #       version: 0.3.7
-  #       output:
-  #         location: pypi
-  #         url: pypi.buildwithfern.com
-  #         package-name: finto-fern-langfuse
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true
-  #     - name: fernapi/fern-typescript-node-sdk
-  #       version: 0.7.1
-  #       output:
-  #         location: npm
-  #         url: npm.buildwithfern.com
-  #         package-name: "@finto-fern/langfuse-node"
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true

generated/.gitignore

@@ -1 +0,0 @@
-python/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.85.2",
+  "version": "3.95.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -40,7 +40,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items ON CLUSTER default (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_7d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_all_amt_mv ON CLUSTER default;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_7d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_all_amt ON CLUSTER default;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt ON CLUSTER default
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv ON CLUSTER default TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv ON CLUSTER default TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv ON CLUSTER default TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv;
+DROP VIEW IF EXISTS traces_7d_amt_mv;
+DROP VIEW IF EXISTS traces_all_amt_mv;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt;
+DROP TABLE IF EXISTS traces_7d_amt;
+DROP TABLE IF EXISTS traces_all_amt;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null;

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/package.json

@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -73,6 +73,8 @@
     "@prisma/client": "^6.10.1",
     "@react-email/components": "^0.1.0",
     "@react-email/render": "^1.1.2",
+    "@slack/oauth": "^2.6.0",
+    "@slack/web-api": "^7.0.0",
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",

packages/shared/prisma/generated/types.ts

@@ -55,6 +55,7 @@ export type AnnotationQueueStatus =
 export const AnnotationQueueObjectType = {
   TRACE: "TRACE",
   OBSERVATION: "OBSERVATION",
+  SESSION: "SESSION",
 } as const;
 export type AnnotationQueueObjectType =
   (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
@@ -138,6 +139,7 @@ export type DashboardWidgetChartType =
   (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export const ActionType = {
   WEBHOOK: "WEBHOOK",
+  SLACK: "SLACK",
 } as const;
 export type ActionType = (typeof ActionType)[keyof typeof ActionType];
 export const ActionExecutionStatus = {
@@ -183,6 +185,14 @@ export type AnnotationQueue = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type AnnotationQueueAssignment = {
+  id: string;
+  project_id: string;
+  user_id: string;
+  queue_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type AnnotationQueueItem = {
   id: string;
   queue_id: string;
@@ -359,6 +369,8 @@ export type Dataset = {
   name: string;
   description: string | null;
   metadata: unknown | null;
+  remote_experiment_url: string | null;
+  remote_experiment_payload: unknown | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -624,6 +636,15 @@ export type OrganizationMembership = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type PendingDeletion = {
+  id: string;
+  project_id: string;
+  object: string;
+  object_id: string;
+  is_deleted: Generated<boolean>;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
   project_id: string;
   encrypted_posthog_api_key: string;
@@ -711,6 +732,16 @@ export type Session = {
   user_id: string;
   expires: Timestamp;
 };
+export type SlackIntegration = {
+  id: string;
+  project_id: string;
+  team_id: string;
+  team_name: string;
+  bot_token: string;
+  bot_user_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type SsoConfig = {
   domain: string;
   created_at: Generated<Timestamp>;
@@ -781,6 +812,7 @@ export type VerificationToken = {
 export type DB = {
   Account: Account;
   actions: Action;
+  annotation_queue_assignments: AnnotationQueueAssignment;
   annotation_queue_items: AnnotationQueueItem;
   annotation_queues: AnnotationQueue;
   api_keys: ApiKey;
@@ -813,6 +845,7 @@ export type DB = {
   observations: LegacyPrismaObservation;
   organization_memberships: OrganizationMembership;
   organizations: Organization;
+  pending_deletions: PendingDeletion;
   posthog_integrations: PosthogIntegration;
   prices: Price;
   project_memberships: ProjectMembership;
@@ -823,6 +856,7 @@ export type DB = {
   score_configs: ScoreConfig;
   scores: LegacyPrismaScore;
   Session: Session;
+  slack_integrations: SlackIntegration;
   sso_configs: SsoConfig;
   table_view_presets: TableViewPreset;
   trace_media: TraceMedia;

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/migrations/20250730100100_add_slack_integration/migration.sql

@@ -0,0 +1,30 @@
+-- Migration: Add Slack Integration Support
+-- This migration adds support for Slack automation actions by:
+-- 1. Adding SLACK to the ActionType enum
+-- 2. Creating slack_integrations table for centralized token storage
+
+-- AlterEnum
+ALTER TYPE "ActionType" ADD VALUE 'SLACK';
+
+-- CreateTable
+CREATE TABLE "slack_integrations" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "team_name" TEXT NOT NULL,
+    "bot_token" TEXT NOT NULL,
+    "bot_user_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "slack_integrations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "slack_integrations_project_id_key" ON "slack_integrations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "slack_integrations_team_id_idx" ON "slack_integrations"("team_id");
+
+-- AddForeignKey
+ALTER TABLE "slack_integrations" ADD CONSTRAINT "slack_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; 

packages/shared/prisma/migrations/20250731100100_add_dataset_run_items_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('8d47f91b-3e5c-4a26-9f85-c12d6e4b9a3d', '20250731_1001_migrate_dataset_run_items_pg_to_ch', 'migrateDatasetRunItemsFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20250731202005_add_trace_deletion_table/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "pending_deletions" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "is_deleted" BOOLEAN NOT NULL DEFAULT false,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "pending_deletions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_project_id_object_is_deleted_idx" ON "pending_deletions"("project_id", "object", "is_deleted");
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_object_id_object_idx" ON "pending_deletions"("object_id", "object");
+
+-- AddForeignKey
+ALTER TABLE "pending_deletions" ADD CONSTRAINT "pending_deletions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250806100613_add_annotation_queue_assignment_table/migration.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "annotation_queue_assignments" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_assignments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queue_assignments_project_id_queue_id_key" ON "annotation_queue_assignments"("project_id", "queue_id", "user_id");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -65,29 +65,30 @@ model Session {
 }
 
 model User {
-  id                      String                   @id @default(cuid())
-  name                    String?
-  email                   String?                  @unique
-  emailVerified           DateTime?                @map("email_verified")
-  password                String?
-  image                   String?
-  admin                   Boolean                  @default(false)
-  accounts                Account[]
-  sessions                Session[]
-  organizationMemberships OrganizationMembership[]
-  projectMemberships      ProjectMembership[]
-  invitations             MembershipInvitation[]
-  createdAt               DateTime                 @default(now()) @map("created_at")
-  updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
-  featureFlags            String[]                 @default([]) @map("feature_flags")
-  annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
-  annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
-  dashboardWidgetsCreated DashboardWidget[]        @relation("CreatedByUser")
-  dashboardWidgetsUpdated DashboardWidget[]        @relation("UpdatedByUser")
-  dashboardCreated        Dashboard[]              @relation("CreatedByUser")
-  dashboardUpdated        Dashboard[]              @relation("UpdatedByUser")
-  tableViewPresetCreated  TableViewPreset[]        @relation("CreatedByUser")
-  tableViewPresetUpdated  TableViewPreset[]        @relation("UpdatedByUser")
+  id                        String                      @id @default(cuid())
+  name                      String?
+  email                     String?                     @unique
+  emailVerified             DateTime?                   @map("email_verified")
+  password                  String?
+  image                     String?
+  admin                     Boolean                     @default(false)
+  accounts                  Account[]
+  sessions                  Session[]
+  organizationMemberships   OrganizationMembership[]
+  projectMemberships        ProjectMembership[]
+  invitations               MembershipInvitation[]
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  featureFlags              String[]                    @default([]) @map("feature_flags")
+  annotatedLockedItem       AnnotationQueueItem[]       @relation("LockedByUser")
+  annotatedCompletedItem    AnnotationQueueItem[]       @relation("AnnotatorUser")
+  dashboardWidgetsCreated   DashboardWidget[]           @relation("CreatedByUser")
+  dashboardWidgetsUpdated   DashboardWidget[]           @relation("UpdatedByUser")
+  dashboardCreated          Dashboard[]                 @relation("CreatedByUser")
+  dashboardUpdated          Dashboard[]                 @relation("UpdatedByUser")
+  tableViewPresetCreated    TableViewPreset[]           @relation("CreatedByUser")
+  tableViewPresetUpdated    TableViewPreset[]           @relation("UpdatedByUser")
+  annotationQueueAssignment AnnotationQueueAssignment[]
 
   @@map("users")
 }
@@ -117,52 +118,55 @@ model Organization {
 }
 
 model Project {
-  id                     String                    @id @default(cuid())
-  orgId                  String                    @map("org_id")
-  createdAt              DateTime                  @default(now()) @map("created_at")
-  updatedAt              DateTime                  @default(now()) @updatedAt @map("updated_at")
-  deletedAt              DateTime?                 @map("deleted_at")
-  name                   String
-  retentionDays          Int?                      @map("retention_days")
-  metadata               Json?
-  projectMembers         ProjectMembership[]
-  organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
-  apiKeys                ApiKey[]
-  dataset                Dataset[]
-  invitations            MembershipInvitation[]
-  sessions               TraceSession[]
-  Prompt                 Prompt[]
-  Model                  Model[]
-  EvalTemplate           EvalTemplate[]
-  JobConfiguration       JobConfiguration[]
-  JobExecution           JobExecution[]
-  LlmApiKeys             LlmApiKeys[]
-  PosthogIntegration     PosthogIntegration[]
-  BlobStorageIntegration BlobStorageIntegration[]
-  scoreConfig            ScoreConfig[]
-  BatchExport            BatchExport[]
-  comment                Comment[]
-  annotationQueue        AnnotationQueue[]
-  annotationQueueItem    AnnotationQueueItem[]
-  TraceMedia             TraceMedia[]
-  Media                  Media[]
-  ObservationMedia       ObservationMedia[]
-  LegacyTrace            LegacyPrismaTrace[]
-  LegacyObservation      LegacyPrismaObservation[]
-  LegacyScore            LegacyPrismaScore[]
-  PromptDependency       PromptDependency[]
-  LlmSchema              LlmSchema[]
-  LlmTool                LlmTool[]
-  PromptProtectedLabels  PromptProtectedLabels[]
-  Dashboard              Dashboard[]
-  DashboardWidget        DashboardWidget[]
-  TableViewPreset        TableViewPreset[]
-  actions                Action[]
-  triggers               Trigger[]
-  automationExecutions   AutomationExecution[]
-  Automation             Automation[]
-  DefaultLlmModel        DefaultLlmModel[]
-  Price                  Price[]
+  id                        String                      @id @default(cuid())
+  orgId                     String                      @map("org_id")
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  deletedAt                 DateTime?                   @map("deleted_at")
+  name                      String
+  retentionDays             Int?                        @map("retention_days")
+  metadata                  Json?
+  projectMembers            ProjectMembership[]
+  organization              Organization                @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+  apiKeys                   ApiKey[]
+  dataset                   Dataset[]
+  invitations               MembershipInvitation[]
+  sessions                  TraceSession[]
+  Prompt                    Prompt[]
+  Model                     Model[]
+  EvalTemplate              EvalTemplate[]
+  JobConfiguration          JobConfiguration[]
+  JobExecution              JobExecution[]
+  LlmApiKeys                LlmApiKeys[]
+  PosthogIntegration        PosthogIntegration[]
+  BlobStorageIntegration    BlobStorageIntegration[]
+  scoreConfig               ScoreConfig[]
+  BatchExport               BatchExport[]
+  comment                   Comment[]
+  annotationQueue           AnnotationQueue[]
+  annotationQueueItem       AnnotationQueueItem[]
+  TraceMedia                TraceMedia[]
+  Media                     Media[]
+  ObservationMedia          ObservationMedia[]
+  LegacyTrace               LegacyPrismaTrace[]
+  LegacyObservation         LegacyPrismaObservation[]
+  LegacyScore               LegacyPrismaScore[]
+  PromptDependency          PromptDependency[]
+  LlmSchema                 LlmSchema[]
+  LlmTool                   LlmTool[]
+  PromptProtectedLabels     PromptProtectedLabels[]
+  Dashboard                 Dashboard[]
+  DashboardWidget           DashboardWidget[]
+  TableViewPreset           TableViewPreset[]
+  actions                   Action[]
+  triggers                  Trigger[]
+  automationExecutions      AutomationExecution[]
+  Automation                Automation[]
+  DefaultLlmModel           DefaultLlmModel[]
+  Price                     Price[]
+  SlackIntegration          SlackIntegration?
+  PendingDeletion           PendingDeletion[]
+  AnnotationQueueAssignment AnnotationQueueAssignment[]
 
   @@index([orgId])
   @@map("projects")
@@ -489,15 +493,16 @@ enum ScoreDataType {
 }
 
 model AnnotationQueue {
-  id                  String                @id @default(cuid())
-  name                String
-  description         String?
-  scoreConfigIds      String[]              @default([]) @map("score_config_ids")
-  projectId           String                @map("project_id")
-  project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt           DateTime              @default(now()) @map("created_at")
-  updatedAt           DateTime              @default(now()) @updatedAt @map("updated_at")
-  annotationQueueItem AnnotationQueueItem[]
+  id                        String                      @id @default(cuid())
+  name                      String
+  description               String?
+  scoreConfigIds            String[]                    @default([]) @map("score_config_ids")
+  projectId                 String                      @map("project_id")
+  project                   Project                     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  annotationQueueItem       AnnotationQueueItem[]
+  annotationQueueAssignment AnnotationQueueAssignment[]
 
   @@unique([projectId, name])
   @@index([id, projectId])
@@ -539,6 +544,22 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
+}
+
+model AnnotationQueueAssignment {
+  id        String          @id @default(cuid())
+  projectId String          @map("project_id")
+  project   Project         @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  userId    String          @map("user_id")
+  user      User            @relation(fields: [userId], references: [id], onDelete: Cascade)
+  queueId   String          @map("queue_id")
+  queue     AnnotationQueue @relation(fields: [queueId], references: [id], onDelete: Cascade)
+  createdAt DateTime        @default(now()) @map("created_at")
+  updatedAt DateTime        @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([projectId, queueId, userId])
+  @@map("annotation_queue_assignments")
 }
 
 model CronJobs {
@@ -551,16 +572,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -1295,6 +1318,7 @@ model Automation {
 
 enum ActionType {
   WEBHOOK
+  SLACK
   // More action types can be added as needed
 }
 
@@ -1335,3 +1359,41 @@ model AutomationExecution {
   @@index([projectId])
   @@map("automation_executions")
 }
+
+// Slack Integration: Stores centralized Slack workspace connection for each project
+// One project can connect to one Slack workspace, supporting multiple channel automations
+model SlackIntegration {
+  id        String  @id @default(cuid())
+  projectId String  @unique @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // Installation details (encrypted using shared encryption utilities)
+  teamId    String @map("team_id") // Slack workspace ID
+  teamName  String @map("team_name") // Human-readable workspace name
+  botToken  String @map("bot_token") // Encrypted bot token for API calls
+  botUserId String @map("bot_user_id") // Bot user ID for workspace
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([teamId])
+  @@map("slack_integrations")
+}
+
+// Pending Deletions: Tracks objects (like traces) that are scheduled for batch deletion
+model PendingDeletion {
+  id        String  @id @default(cuid())
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  object    String  @map("object") // e.g., "trace", "observation", etc.
+  objectId  String  @map("object_id") // The ID of the object to be deleted
+  isDeleted Boolean @default(false) @map("is_deleted")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, object, isDeleted])
+  @@index([objectId, object])
+  @@map("pending_deletions")
+}

packages/shared/scripts/seeder/utils/markdown.txt

@@ -11,29 +11,29 @@
 ## 🎯 Getting Started
 
 ### Prerequisites
-\`\`\`bash
-npm install @langfuse/core
-pip install langfuse
+\`\`\`bash  
+npm install @langfuse/core  
+pip install langfuse  
 \`\`\`
 
 ### Quick Setup
 1. **Initialize your project**
-   \`\`\`typescript
-   import { Langfuse } from 'langfuse'
-   
-   const langfuse = new Langfuse({
-     secretKey: process.env.LANGFUSE_SECRET_KEY,
-     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-     baseUrl: 'https://cloud.langfuse.com'
-   })
+   \`\`\`typescript  
+   import { Langfuse } from 'langfuse'  
+
+   const langfuse = new Langfuse({  
+     secretKey: process.env.LANGFUSE_SECRET_KEY,  
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,  
+     baseUrl: 'https://cloud.langfuse.com'  
+   })  
    \`\`\`
 
 2. **Create your first trace**
-   \`\`\`python
-   from langfuse import Langfuse
-   
-   langfuse = Langfuse()
-   trace = langfuse.trace(name="chat-application")
+   \`\`\`python  
+   from langfuse import Langfuse  
+
+   langfuse = Langfuse()  
+   trace = langfuse.trace(name="chat-application")  
    \`\`\`
 
 ---
@@ -72,75 +72,75 @@ graph TD
 > **Note:** Traces are the foundation of observability in LLM applications.
 
 #### Creating Traces
-\`\`\`typescript
-// Basic trace creation
-const trace = langfuse.trace({
-  name: "user-query-processing",
-  userId: "user-123",
-  sessionId: "session-456",
-  metadata: {
-    environment: "production",
-    version: "2.1.0"
-  }
-})
+\`\`\`typescript  
+// Basic trace creation  
+const trace = langfuse.trace({  
+  name: "user-query-processing",  
+  userId: "user-123",  
+  sessionId: "session-456",  
+  metadata: {  
+    environment: "production",  
+    version: "2.1.0"  
+  }  
+})  
 
-// Nested observations
-const span = trace.span({
-  name: "document-retrieval",
-  input: { query: "What is machine learning?" },
-  metadata: { vectorStore: "pinecone" }
-})
+// Nested observations  
+const span = trace.span({  
+  name: "document-retrieval",  
+  input: { query: "What is machine learning?" },  
+  metadata: { vectorStore: "pinecone" }  
+})  
 
-const generation = span.generation({
-  name: "answer-generation",
-  model: "gpt-4",
-  input: retrievedDocs,
-  output: generatedAnswer,
-  usage: {
-    promptTokens: 1250,
-    completionTokens: 420,
-    totalTokens: 1670
-  }
-})
+const generation = span.generation({  
+  name: "answer-generation",  
+  model: "gpt-4",  
+  input: retrievedDocs,  
+  output: generatedAnswer,  
+  usage: {  
+    promptTokens: 1250,  
+    completionTokens: 420,  
+    totalTokens: 1670  
+  }  
+})  
 \`\`\`
 
 ### Advanced Features
 
 #### 🔄 Async Processing
-\`\`\`python
-import asyncio
-from langfuse import Langfuse
+\`\`\`python  
+import asyncio  
+from langfuse import Langfuse  
 
-async def process_batch():
-    langfuse = Langfuse()
-    
-    tasks = []
-    for item in batch_items:
-        task = asyncio.create_task(
-            process_item_with_tracing(langfuse, item)
-        )
-        tasks.append(task)
-    
-    results = await asyncio.gather(*tasks)
-    return results
+async def process_batch():  
+    langfuse = Langfuse()  
+
+    tasks = []  
+    for item in batch_items:  
+        task = asyncio.create_task(  
+            process_item_with_tracing(langfuse, item)  
+        )  
+        tasks.append(task)  
+
+    results = await asyncio.gather(*tasks)  
+    return results  
 \`\`\`
 
 #### 🎯 Custom Scoring
-\`\`\`typescript
-// Automated scoring
-trace.score({
-  name: "relevance",
-  value: 0.95,
-  comment: "Highly relevant response"
-})
+\`\`\`typescript  
+// Automated scoring  
+trace.score({  
+  name: "relevance",  
+  value: 0.95,  
+  comment: "Highly relevant response"  
+})  
 
-// Human feedback scoring
-trace.score({
-  name: "user-satisfaction", 
-  value: 1,
-  source: "user-feedback",
-  comment: "User rated 5/5 stars"
-})
+// Human feedback scoring  
+trace.score({  
+  name: "user-satisfaction",  
+  value: 1,  
+  source: "user-feedback",  
+  comment: "User rated 5/5 stars"  
+})  
 \`\`\`
 
 ---
@@ -156,20 +156,20 @@ trace.score({
 - **User Satisfaction**: Quality metrics
 
 #### Dashboard Setup
-\`\`\`yaml
-# monitoring-config.yml
-dashboards:
-  - name: "LLM Performance"
-    panels:
-      - type: "time-series"
-        title: "Response Latency"
-        query: "avg(response_time) by (model)"
-      - type: "stat"
-        title: "Daily Token Usage" 
-        query: "sum(tokens_used)"
-      - type: "table"
-        title: "Top Errors"
-        query: "topk(10, count by (error_type))"
+\`\`\`yaml  
+# monitoring-config.yml  
+dashboards:  
+  - name: "LLM Performance"  
+    panels:  
+      - type: "time-series"  
+        title: "Response Latency"  
+        query: "avg(response_time) by (model)"  
+      - type: "stat"  
+        title: "Daily Token Usage"  
+        query: "sum(tokens_used)"  
+      - type: "table"  
+        title: "Top Errors"  
+        query: "topk(10, count by (error_type))"  
 \`\`\`
 
 ### 🔐 Security Considerations
@@ -177,40 +177,40 @@ dashboards:
 > ⚠️ **Important**: Never log sensitive user data in traces
 
 #### Data Sanitization
-\`\`\`python
-def sanitize_input(data):
-    """Remove PII from trace data"""
-    sanitized = data.copy()
-    
-    # Remove email addresses
-    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
-                      '[EMAIL_REDACTED]', sanitized)
-    
-    # Remove phone numbers
-    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
-                      '[PHONE_REDACTED]', sanitized)
-    
-    return sanitized
+\`\`\`python  
+def sanitize_input(data):  
+    """Remove PII from trace data"""  
+    sanitized = data.copy()  
+
+    # Remove email addresses  
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b',  
+                      '[EMAIL_REDACTED]', sanitized)  
+
+    # Remove phone numbers  
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b',  
+                      '[PHONE_REDACTED]', sanitized)  
+
+    return sanitized  
 \`\`\`
 
 ### 🚀 Performance Optimization
 
 #### Batch Processing
-\`\`\`typescript
-// Efficient batch uploads
-const batchSize = 100
-const traces = []
+\`\`\`typescript  
+// Efficient batch uploads  
+const batchSize = 100  
+const traces = []  
 
-for (let i = 0; i < data.length; i += batchSize) {
-  const batch = data.slice(i, i + batchSize)
-  const processedBatch = await Promise.all(
-    batch.map(item => processWithLangfuse(item))
-  )
-  traces.push(...processedBatch)
-}
+for (let i = 0; i < data.length; i += batchSize) {  
+  const batch = data.slice(i, i + batchSize)  
+  const processedBatch = await Promise.all(  
+    batch.map(item => processWithLangfuse(item))  
+  )  
+  traces.push(...processedBatch)  
+}  
 
-// Flush all traces at once
-await langfuse.flushAsync()
+// Flush all traces at once  
+await langfuse.flushAsync()  
 \`\`\`
 
 ---
@@ -219,34 +219,34 @@ await langfuse.flushAsync()
 
 ### Multi-Agent System Tracing
 \`\`\`python
-class MultiAgentTracer:
-    def __init__(self):
-        self.langfuse = Langfuse()
-    
-    async def orchestrate_agents(self, task):
-        # Main orchestration trace
-        main_trace = self.langfuse.trace(
-            name="multi-agent-orchestration",
-            input={"task": task}
-        )
-        
-        # Agent 1: Research
-        research_span = main_trace.span(name="research-agent")
-        research_result = await self.research_agent.process(task)
-        research_span.end(output=research_result)
-        
+class MultiAgentTracer:  
+    def __init__(self):  
+        self.langfuse = Langfuse()  
+
+    async def orchestrate_agents(self, task):  
+        # Main orchestration trace  
+        main_trace = self.langfuse.trace(  
+            name="multi-agent-orchestration",  
+            input={"task": task}  
+        )  
+
+        # Agent 1: Research  
+        research_span = main_trace.span(name="research-agent")  
+        research_result = await self.research_agent.process(task)  
+        research_span.end(output=research_result)  
+
         # Agent 2: Analysis  
-        analysis_span = main_trace.span(name="analysis-agent")
-        analysis_result = await self.analysis_agent.process(research_result)
-        analysis_span.end(output=analysis_result)
-        
-        # Agent 3: Synthesis
-        synthesis_span = main_trace.span(name="synthesis-agent")
-        final_result = await self.synthesis_agent.process(analysis_result)
-        synthesis_span.end(output=final_result)
-        
-        main_trace.end(output=final_result)
-        return final_result
+        analysis_span = main_trace.span(name="analysis-agent")  
+        analysis_result = await self.analysis_agent.process(research_result)  
+        analysis_span.end(output=analysis_result)  
+
+        # Agent 3: Synthesis  
+        synthesis_span = main_trace.span(name="synthesis-agent")  
+        final_result = await self.synthesis_agent.process(analysis_result)  
+        synthesis_span.end(output=final_result)  
+
+        main_trace.end(output=final_result)  
+        return final_result  
 \`\`\`
 
 ---
@@ -256,11 +256,11 @@ class MultiAgentTracer:
 With proper implementation of Langfuse tracing, you can:
 
 - ✅ **Monitor** your LLM applications in real-time
-- ✅ **Debug** issues with detailed trace information  
+- ✅ **Debug** issues with detailed trace information
 - ✅ **Optimize** performance and costs
 - ✅ **Scale** your applications with confidence
 
 ### Next Steps
 1. Review the [official documentation](https://langfuse.com/docs)
 2. Join our [Discord community](https://discord.gg/langfuse)
-3. Check out [example projects](https://github.com/langfuse/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/src/domain/automations.ts

@@ -30,14 +30,14 @@ export type AutomationDomain = {
 };
 
 export type ActionDomain = Omit<Action, "config"> & {
-  config: SafeWebhookActionConfig;
+  config: SafeActionConfig;
 };
 
 export type ActionDomainWithSecrets = Omit<Action, "config"> & {
-  config: WebhookActionConfigWithSecrets;
+  config: ActionConfigWithSecrets;
 };
 
-export const ActionTypeSchema = z.enum(["WEBHOOK"]);
+export const ActionTypeSchema = z.enum(["WEBHOOK", "SLACK"]);
 
 export const AvailableWebhookApiSchema = z.record(
   z.enum(["prompt"]),
@@ -52,12 +52,13 @@ export const RequestHeaderSchema = z.object({
 export const WebhookActionConfigSchema = z.object({
   type: z.literal("WEBHOOK"),
   url: z.url(),
-  headers: z.record(z.string(), z.string()),
-  requestHeaders: z.record(z.string(), RequestHeaderSchema),
-  displayHeaders: z.record(z.string(), RequestHeaderSchema),
+  headers: z.record(z.string(), z.string()).optional(), // deprecated field, use requestHeaders instead
+  requestHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
+  displayHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
   apiVersion: AvailableWebhookApiSchema,
   secretKey: z.string(),
   displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
 });
 
 export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
@@ -77,18 +78,94 @@ export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
   displayHeaders: true,
 });
 
+export const SlackActionConfigSchema = z.object({
+  type: z.literal("SLACK"),
+  channelId: z.string(),
+  channelName: z.string(),
+  messageTemplate: z.string().optional(),
+});
+
+export type SlackActionConfig = z.infer<typeof SlackActionConfigSchema>;
+
 export const ActionConfigSchema = z.discriminatedUnion("type", [
   WebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export const ActionCreateSchema = z.discriminatedUnion("type", [
   WebhookActionCreateSchema,
+  SlackActionConfigSchema,
+]);
+
+export const SafeActionConfigSchema = z.discriminatedUnion("type", [
+  SafeWebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export type ActionTypes = z.infer<typeof ActionTypeSchema>;
 export type ActionConfig = z.infer<typeof ActionConfigSchema>;
 export type ActionCreate = z.infer<typeof ActionCreateSchema>;
+export type SafeActionConfig = z.infer<typeof SafeActionConfigSchema>;
 
+export type WebhookActionCreate = z.infer<typeof WebhookActionCreateSchema>;
 export type WebhookActionConfigWithSecrets = z.infer<
   typeof WebhookActionConfigSchema
 >;
+
+export type ActionConfigWithSecrets = z.infer<typeof ActionConfigSchema>;
+
+// Type Guards for Runtime Validation
+// Using existing Zod schemas to provide both compile-time and runtime type safety
+
+/**
+ * Type guard to check if a config is a valid webhook configuration with secrets
+ */
+export function isWebhookActionConfig(
+  config: unknown,
+): config is WebhookActionConfigWithSecrets {
+  return WebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if a config is a valid Slack configuration
+ */
+export function isSlackActionConfig(
+  config: unknown,
+): config is SlackActionConfig {
+  return SlackActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if an entire action has valid webhook configuration
+ */
+export function isWebhookAction(action: {
+  type: string;
+  config: unknown;
+}): action is { type: "WEBHOOK"; config: WebhookActionConfigWithSecrets } {
+  return action.type === "WEBHOOK" && isWebhookActionConfig(action.config);
+}
+
+/**
+ * Type guard for safe webhook config (without secrets)
+ */
+export function isSafeWebhookActionConfig(
+  config: unknown,
+): config is SafeWebhookActionConfig {
+  return SafeWebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Converts webhook config with secrets to safe config by only including allowed fields
+ */
+export function convertToSafeWebhookConfig(
+  webhookConfig: WebhookActionConfigWithSecrets,
+): SafeWebhookActionConfig {
+  return {
+    type: webhookConfig.type,
+    url: webhookConfig.url,
+    displayHeaders: webhookConfig.displayHeaders,
+    apiVersion: webhookConfig.apiVersion,
+    displaySecretKey: webhookConfig.displaySecretKey,
+    lastFailingExecutionId: webhookConfig.lastFailingExecutionId,
+  };
+}

packages/shared/src/env.ts

@@ -15,6 +15,7 @@ const EnvSchema = z.object({
     .default(6379)
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
+  REDIS_USERNAME: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
   REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
@@ -49,6 +50,10 @@ const EnvSchema = z.object({
     .nonnegative()
     .default(15_000),
   LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
+  LANGFUSE_TRACE_DELETE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(5_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -87,6 +92,9 @@ const EnvSchema = z.object({
   LANGFUSE_S3_MEDIA_UPLOAD_SSE: z.enum(["AES256", "aws:kms"]).optional(),
   LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_AZURE_SKIP_CONTAINER_CHECK: z
+    .enum(["true", "false"])
+    .default("true"),
   LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
@@ -140,6 +148,60 @@ const EnvSchema = z.object({
   LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
+
+  SLACK_CLIENT_ID: z.string().optional(),
+  SLACK_CLIENT_SECRET: z.string().optional(),
+  SLACK_STATE_SECRET: z.string().optional(),
+
+  LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(1_000),
+
+  LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(600_000), // 10 minutes
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/annotation/types.ts

@@ -91,4 +91,11 @@ export const CreateQueueData = z.object({
   }),
 });
 
+export const CreateQueueWithAssignmentsData = CreateQueueData.extend({
+  newAssignmentUserIds: z.array(z.string()),
+});
+
 export type CreateQueue = z.infer<typeof CreateQueueData>;
+export type CreateQueueWithAssignments = z.infer<
+  typeof CreateQueueWithAssignmentsData
+>;

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,8 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
+  "observation-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/index.ts

@@ -18,6 +18,7 @@ export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
 export * from "./domain/webhooks";
+export * from "./domain/dataset-run-items";
 
 // llm api
 export * from "./server/llm/types";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -18,6 +18,21 @@ export const CloudConfigSchema = z.object({
 
   // custom rate limits for an organization
   rateLimitOverrides: CloudConfigRateLimit.optional(),
+
+  // billing alert configuration
+  usageAlerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      type: z.enum(["STRIPE"]).default("STRIPE"),
+      threshold: z.number().int().positive(),
+      alertId: z.string(), // Alert ID for tracking
+      meterId: z.string(), // Meter ID for usage tracking
+      notifications: z.object({
+        email: z.boolean().default(true),
+        recipients: z.array(z.string().email()).default([]),
+      }),
+    })
+    .optional(),
 });
 
 export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/server/auth/types.ts

@@ -1,7 +1,7 @@
 import z from "zod/v4";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
-import { ApiKeyScope } from "../../";
+import { ApiKeyScope, MakeOptional } from "../../";
 
 const ApiKeyBaseSchema = z.object({
   id: z.string(),
@@ -48,12 +48,25 @@ export type AuthHeaderValidVerificationResult = {
   scope: ApiAccessScope;
 };
 
-export type ApiAccessScope = {
+export type AuthHeaderValidVerificationResultIngestion = {
+  validKey: true;
+  scope: ApiAccessScopeIngestion;
+};
+
+type BaseApiAccessScope = {
   projectId: string | null;
   accessLevel: "organization" | "project" | "scores";
+};
+
+type ApiAccessScopeMetadata = {
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
   apiKeyId: string;
   publicKey: string;
 };
+
+export type ApiAccessScopeIngestion = BaseApiAccessScope &
+  MakeOptional<ApiAccessScopeMetadata>;
+
+export type ApiAccessScope = BaseApiAccessScope & ApiAccessScopeMetadata;

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -1,4 +1,4 @@
-import { instrumentAsync } from "../instrumentation";
+import { instrumentAsync, recordDistribution } from "../instrumentation";
 import * as opentelemetry from "@opentelemetry/api";
 import { env } from "../../env";
 import { logger } from "../logger";
@@ -19,12 +19,17 @@ const executionWrapper = async <T, Y>(
   return [res, duration];
 };
 
+/**
+ * Measures the execution time of two functions and returns the result based on the experiment configuration.
+ * This is used to compare the execution of AggregatingMergeTrees with the existing ReplacingMergeTree execution.
+ */
 export const measureAndReturn = async <T, Y>(args: {
   operationName: string;
   projectId: string;
   input: T;
   existingExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
   newExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
+  minStartTime?: Date;
 }): Promise<Y> => {
   return instrumentAsync(
     {
@@ -32,14 +37,34 @@ export const measureAndReturn = async <T, Y>(args: {
       spanKind: opentelemetry.SpanKind.CLIENT,
     },
     async (currentSpan) => {
-      const { input, existingExecution, newExecution } = args;
+      const { input, existingExecution, newExecution, minStartTime } = args;
 
       if (
         env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES !==
         "true"
       ) {
         currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
-        return existingExecution(input);
+
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newExecution(input);
+          }
+        }
+
+        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
+          ? newExecution(input)
+          : existingExecution(input);
       }
 
       // If not whitelisted, apply sampling logic
@@ -68,6 +93,14 @@ export const measureAndReturn = async <T, Y>(args: {
           durationDifference,
         );
 
+        recordDistribution(
+          "langfuse.experiment.amts.duration_difference_distribution",
+          durationDifference,
+          {
+            operation: args.operationName,
+          },
+        );
+
         if (
           env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
             (p) => p === args.projectId,
@@ -83,6 +116,23 @@ export const measureAndReturn = async <T, Y>(args: {
           );
         }
 
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newResult;
+          }
+        }
+
         return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
           ? newResult
           : existingResult;

packages/shared/src/server/clickhouse/schema.ts

@@ -2,6 +2,7 @@ export const ClickhouseTableNames = {
   traces: "traces",
   observations: "observations",
   scores: "scores",
+  dataset_run_items: "dataset_run_items",
 
   // Virtual tables for dashboards
   // TODO: Check if we can do this more elegantly

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/index.ts

@@ -2,6 +2,7 @@ export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
+export * from "./services/email/billingAlert/sendBillingAlertEmail";
 export * from "./services/PromptService";
 export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
@@ -32,6 +33,7 @@ export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -56,6 +58,7 @@ export * from "./logger";
 export * from "./headerPropagation";
 export * from "./queries";
 export * from "./repositories";
+export * from "./utils/rendering";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
 export * from "./services/datasets-ui-table-service";
@@ -63,13 +66,17 @@ export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
 export * from "./clickhouse/measureAndReturn";
+export * from "./services/SlackService";
+
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
 // dataset run items
 export * from "./dataset-run-items/datasetExecution";
 export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
 
 // test utils
 export * from "./test-utils";
 export * from "./utils/headerUtils";
+export * from "./traceDeletionProcessor";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -83,7 +83,7 @@ const getModelFromRedis = async (
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -8,7 +8,7 @@ import {
   LangfuseNotFoundError,
   UnauthorizedError,
 } from "../../errors";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { AuthHeaderValidVerificationResultIngestion } from "../auth/types";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import {
   getCurrentSpan,
@@ -30,6 +30,7 @@ import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -61,7 +62,7 @@ export type TokenCountDelegate = (p: {
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -73,6 +74,10 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
@@ -94,12 +99,12 @@ type ProcessEventBatchOptions = {
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
- * @param authCheck - AuthHeaderValidVerificationResult
+ * @param authCheck - AuthHeaderValidVerificationResultIngestion
  * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
-  authCheck: AuthHeaderValidVerificationResult,
+  authCheck: AuthHeaderValidVerificationResultIngestion,
   options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
@@ -124,8 +129,10 @@ export const processEventBatch = async (
     "langfuse.project.id",
     authCheck.scope.projectId ?? "",
   );
-  currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
-  currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
+  if (authCheck.scope.orgId)
+    currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
+  if (authCheck.scope.plan)
+    currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
@@ -269,6 +276,27 @@ export const processEventBatch = async (
       const shouldSkipS3List =
         isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
 
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
+
       return queue
         ? queue.add(
             QueueJobs.IngestionJob,
@@ -292,7 +320,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -307,7 +335,7 @@ export const processEventBatch = async (
 
 const isAuthorized = (
   event: IngestionEventType,
-  authScope: AuthHeaderValidVerificationResult,
+  authScope: AuthHeaderValidVerificationResultIngestion,
 ): boolean => {
   if (event.type === eventTypes.SDK_LOG) {
     return true;

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -316,22 +316,6 @@ export async function fetchLLMCompletion(
       maxRetries,
       apiKey,
     });
-  } else if (modelParams.adapter === LLMAdapter.Atla) {
-    // Atla models do not support:
-    // - temperature
-    // - max_tokens
-    // - top_p
-    chatModel = new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      callbacks: finalCallbacks,
-      maxRetries,
-      configuration: {
-        baseURL: baseURL,
-        defaultHeaders: extraHeaders,
-      },
-      timeout: 1000 * 60, // 1 minute timeout
-    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;

packages/shared/src/server/llm/types.ts

@@ -113,6 +113,7 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
   Tool = "tool",
+  Model = "model", // Google Gemini assistant format
 }
 
 // Thought: should placeholder not semantically be part of this, because it can be
@@ -124,6 +125,7 @@ export enum ChatMessageType {
   AssistantText = "assistant-text",
   AssistantToolCall = "assistant-tool-call",
   ToolResult = "tool-result",
+  ModelText = "model-text",
   PublicAPICreated = "public-api-created",
   Placeholder = "placeholder",
 }
@@ -156,6 +158,13 @@ export const AssistantTextMessageSchema = z.object({
 });
 export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
 
+export const ModelMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ModelText),
+  role: z.literal(ChatMessageRole.Model),
+  content: z.string(),
+});
+export type ModelMessage = z.infer<typeof ModelMessageSchema>;
+
 export const AssistantToolCallMessageSchema = z.object({
   type: z.literal(ChatMessageType.AssistantToolCall),
   role: z.literal(ChatMessageRole.Assistant),
@@ -193,6 +202,7 @@ export const ChatMessageSchema = z.union([
   AssistantTextMessageSchema,
   AssistantToolCallMessageSchema,
   ToolResultMessageSchema,
+  ModelMessageSchema,
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
@@ -226,7 +236,6 @@ export type PromptVariable = { name: string; value: string; isUsed: boolean };
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
-  Atla = "atla",
   Azure = "azure",
   Bedrock = "bedrock",
   VertexAI = "google-vertex-ai",
@@ -322,6 +331,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const anthropicModels = [
   "claude-sonnet-4-20250514",
+  "claude-opus-4-1-20250805",
   "claude-opus-4-20250514",
   "claude-3-7-sonnet-20250219",
   "claude-3-5-sonnet-20241022",
@@ -368,8 +378,6 @@ export const googleAIStudioModels = [
   "gemini-1.5-flash-8b",
 ] as const;
 
-export const atlaModels = ["atla-selene", "atla-selene-20250214"] as const;
-
 export type AnthropicModel = (typeof anthropicModels)[number];
 export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
@@ -379,7 +387,6 @@ export const supportedModels = {
   [LLMAdapter.GoogleAIStudio]: googleAIStudioModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
-  [LLMAdapter.Atla]: atlaModels,
 } as const;
 
 export type LLMFunctionCall = {

packages/shared/src/server/queries/createSessionsAllQuery.ts

@@ -1,88 +0,0 @@
-import { z } from "zod/v4";
-
-import { Prisma } from "@prisma/client";
-import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
-import { singleFilter } from "../../interfaces/filters";
-import { orderBy } from "../../interfaces/orderBy";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { sessionsViewCols } from "../../tableDefinitions";
-
-const GetSessionTableSQLParamsSchema = z.object({
-  projectId: z.string(),
-  filter: z.array(singleFilter).nullable(),
-  orderBy: orderBy,
-  page: z.number(),
-  limit: z.number(),
-});
-type GetSessionTableSQLParams = z.infer<typeof GetSessionTableSQLParamsSchema>;
-
-export const createSessionsAllQuery = (
-  select: Prisma.Sql,
-  params: GetSessionTableSQLParams,
-  options?: {
-    ignoreOrderBy?: boolean; // used by session.metrics and session.all.totalCount
-    sessionIdList?: string[]; // used by session.metrics
-  },
-): Prisma.Sql => {
-  const { projectId, filter, orderBy, page, limit } =
-    GetSessionTableSQLParamsSchema.parse(params);
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filter ?? [],
-    sessionsViewCols,
-    "sessions",
-  );
-  const orderByCondition = orderByToPrismaSql(orderBy, sessionsViewCols);
-
-  const sessionIdFilter = options?.sessionIdList
-    ? Prisma.sql`AND s.id IN (${Prisma.join(options?.sessionIdList)})`
-    : Prisma.sql``;
-
-  const sql = Prisma.sql`
-      SELECT
-        ${select}
-      FROM
-        trace_sessions AS s
-        LEFT JOIN LATERAL (
-          SELECT
-            t.session_id,
-            MAX(t. "timestamp") AS "max_timestamp",
-            MIN(t. "timestamp") AS "min_timestamp",
-            array_agg(t.id) AS "traceIds",
-            array_agg(DISTINCT t.user_id) AS "userIds",
-            count(t.id)::int AS "countTraces",
-            array_agg(DISTINCT u.tag) AS "tags"
-          FROM
-            traces t
-          LEFT JOIN LATERAL (
-            SELECT DISTINCT UNNEST(t.tags) AS tag) AS u ON TRUE
-          WHERE
-            t.project_id = ${projectId}
-            AND t.session_id = s.id
-          GROUP BY
-            t.session_id) AS t ON TRUE
-        LEFT JOIN LATERAL (
-          SELECT
-            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
-            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
-            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
-            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
-            SUM(o.prompt_tokens) AS "promptTokens",
-            SUM(o.completion_tokens) AS "completionTokens",
-            SUM(o.total_tokens) AS "totalTokens"
-          FROM
-            observations_view o
-          WHERE
-            o.project_id = ${projectId}
-            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
-      WHERE
-        s. "project_id" = ${projectId}
-        ${filterCondition}
-        ${sessionIdFilter}
-      ${options?.ignoreOrderBy ? Prisma.sql`` : orderByCondition}
-      LIMIT ${limit}
-      OFFSET ${page * limit}
-    `;
-
-  return sql;
-};

packages/shared/src/server/queries/index.ts

@@ -1,4 +1,3 @@
-export { createSessionsAllQuery } from "./createSessionsAllQuery";
 export {
   type FullObservations,
   type FullObservationsWithScores,

packages/shared/src/server/queues.ts

@@ -40,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -101,6 +116,24 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
+    z.object({
+      actionId: z.literal("observation-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -144,6 +177,7 @@ export const WebhookInputSchema = z.object({
   payload: WebhookOutboundEnvelopeSchema,
 });
 
+export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   z.object({
     entityType: z.literal("prompt-version"),
@@ -154,8 +188,6 @@ export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   }),
   // Add other entity types here in the future
 ]);
-
-export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export type EntityChangeEventType = z.infer<typeof EntityChangeEventSchema>;
 
 export type CreateEvalQueueEventType = z.infer<
@@ -165,6 +197,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -192,6 +225,13 @@ export type DeadLetterRetryQueueEventType = z.infer<
 
 export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
 
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -214,6 +254,7 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
   WebhookQueue = "webhook-queue",
   EntityChangeQueue = "entity-change-queue",
@@ -241,6 +282,7 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
   WebhookJob = "webhook-job",
   EntityChangeJob = "entity-change-job",
@@ -265,6 +307,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -282,6 +330,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -306,6 +355,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -34,7 +34,7 @@ export class ExperimentCreateQueue {
               attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 1000,
+                delay: 10_000, // 10 seconds
               },
             },
           },

packages/shared/src/server/redis/getQueue.ts

@@ -23,6 +23,7 @@ import { ScoreDeleteQueue } from "./scoreDelete";
 import { DeadLetterRetryQueue } from "./dlqRetryQueue";
 import { WebhookQueue } from "./webhookQueue";
 import { EntityChangeQueue } from "./entityChangeQueue";
+import { DatasetDeleteQueue } from "./datasetDelete";
 
 // IngestionQueue is sharded and requires a sharding key
 // Use IngestionQueue.getInstance({ shardName: queueName }) directly instead
@@ -36,6 +37,8 @@ export function getQueue(
       return CloudUsageMeteringQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.DatasetDelete:
+      return DatasetDeleteQueue.getInstance();
     case QueueName.EvaluationExecution:
       return EvalExecutionQueue.getInstance();
     case QueueName.ExperimentCreate:

packages/shared/src/server/redis/redis.ts

@@ -77,6 +77,7 @@ const createRedisClusterInstance = (
       callback(null, address);
     },
     redisOptions: {
+      username: env.REDIS_USERNAME || undefined,
       password: env.REDIS_AUTH || undefined,
       ...defaultRedisOptions,
       ...additionalOptions,
@@ -129,6 +130,7 @@ export const createNewRedisInstance = (
       ? new Redis({
           host: String(env.REDIS_HOST),
           port: Number(env.REDIS_PORT),
+          username: env.REDIS_USERNAME || undefined,
           password: String(env.REDIS_AUTH),
           ...defaultRedisOptions,
           ...additionalOptions,

packages/shared/src/server/redis/webhookQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
 import { logger } from "../logger";
 
 export class WebhookQueue {
@@ -23,6 +27,7 @@ export class WebhookQueue {
           QueueName.WebhookQueue,
           {
             connection: newRedis,
+            prefix: getQueuePrefix(QueueName.WebhookQueue),
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,

packages/shared/src/server/repositories/automation-repository.ts

@@ -2,17 +2,22 @@ import {
   Action,
   ActionExecutionStatus,
   JobConfigState,
+  Prisma,
   prisma,
   Trigger,
 } from "../../db";
 import {
   TriggerEventSource,
-  WebhookActionConfigWithSecrets,
   TriggerDomain,
   TriggerEventAction,
   ActionDomain,
   AutomationDomain,
-  SafeWebhookActionConfig,
+  ActionDomainWithSecrets,
+  SafeActionConfig,
+  isWebhookActionConfig,
+  WebhookActionConfigWithSecrets,
+  isSafeWebhookActionConfig,
+  convertToSafeWebhookConfig,
 } from "../../domain/automations";
 import { FilterState } from "../../types";
 import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";
@@ -23,7 +28,7 @@ export const getActionByIdWithSecrets = async ({
 }: {
   projectId: string;
   actionId: string;
-}) => {
+}): Promise<ActionDomainWithSecrets | null> => {
   const actionConfig = await prisma.action.findFirst({
     where: {
       id: actionId,
@@ -35,31 +40,41 @@ export const getActionByIdWithSecrets = async ({
     return null;
   }
 
-  const config = actionConfig.config as WebhookActionConfigWithSecrets;
+  if (isWebhookActionConfig(actionConfig.config)) {
+    const config = actionConfig.config; // Type guard ensures this is WebhookActionConfigWithSecrets
 
-  // Decrypt secret headers for webhook execution using new structure
-  const decryptedHeaders = config.requestHeaders
-    ? decryptSecretHeaders(mergeHeaders(config.headers, config.requestHeaders))
-    : Object.entries(config.headers).reduce(
-        (acc, [key, value]) => {
-          acc[key] = { secret: false, value };
-          return acc;
-        },
-        {} as Record<string, { secret: boolean; value: string }>,
-      );
+    // Decrypt secret headers for webhook execution using new structure
+    const decryptedHeaders = config.requestHeaders
+      ? decryptSecretHeaders(
+          mergeHeaders(config.headers, config.requestHeaders),
+        )
+      : config.headers
+        ? Object.entries(config.headers).reduce(
+            (acc, [key, value]) => {
+              acc[key] = { secret: false, value };
+              return acc;
+            },
+            {} as Record<string, { secret: boolean; value: string }>,
+          )
+        : {};
 
-  return {
-    ...actionConfig,
-    config: {
-      type: config.type,
-      url: config.url,
-      requestHeaders: decryptedHeaders,
-      displayHeaders: config.displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-      secretKey: config.secretKey,
-    },
-  };
+    return {
+      ...actionConfig,
+      config: {
+        type: config.type,
+        url: config.url,
+        requestHeaders: decryptedHeaders,
+        displayHeaders: getDisplayHeaders(config),
+        apiVersion: config.apiVersion,
+        displaySecretKey: config.displaySecretKey,
+        secretKey: config.secretKey,
+        lastFailingExecutionId: config.lastFailingExecutionId,
+      },
+    };
+  }
+
+  // For SLACK and others, return as stored (already safe)
+  return actionConfig as ActionDomainWithSecrets;
 };
 
 export const getActionById = async ({
@@ -128,10 +143,7 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {
   };
 };
 
-const convertActionToDomain = (action: Action): ActionDomain => {
-  const config = action.config as WebhookActionConfigWithSecrets;
-
-  // Handle legacy headers - convert them to displayHeaders format if displayHeaders is undefined
+const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {
   let displayHeaders = config.displayHeaders;
   if (!displayHeaders && config.headers) {
     // Convert legacy headers to displayHeaders format
@@ -143,17 +155,25 @@ const convertActionToDomain = (action: Action): ActionDomain => {
       {} as Record<string, { secret: boolean; value: string }>,
     );
   }
+  return displayHeaders;
+};
 
+const convertActionToDomain = (action: Action): ActionDomain => {
+  if (isWebhookActionConfig(action.config)) {
+    const config = action.config;
+    config.displayHeaders = getDisplayHeaders(config);
+
+    return {
+      ...action,
+      config: convertToSafeWebhookConfig(config),
+    };
+  }
+
+  // For SLACK (or future types) return config as-is
   return {
     ...action,
-    config: {
-      type: config.type,
-      url: config.url,
-      displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-    } as SafeWebhookActionConfig,
-  };
+    config: action.config as SafeActionConfig,
+  } as ActionDomain;
 };
 
 export const getAutomationById = async ({
@@ -225,28 +245,49 @@ export const getConsecutiveAutomationFailures = async ({
   automationId: string;
   projectId: string;
 }): Promise<number> => {
-  // First get the automation to extract triggerId and actionId
-  const automation = await prisma.automation.findFirst({
-    where: {
-      id: automationId,
-      projectId,
-    },
+  const automation = await getAutomationById({
+    automationId,
+    projectId,
   });
 
   if (!automation) {
     return 0;
   }
 
-  const { triggerId, actionId } = automation;
-  const executions = await prisma.automationExecution.findMany({
-    where: {
-      triggerId,
-      actionId,
-      projectId,
-      status: {
-        in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
-      },
+  // Build where clause - if lastFailingExecutionId is set, only consider executions newer than it
+  const whereClause: Prisma.AutomationExecutionWhereInput = {
+    triggerId: automation.trigger.id,
+    actionId: automation.action.id,
+    projectId,
+    status: {
+      in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
     },
+  };
+
+  // If there's a lastFailingExecutionId, we need to get executions that are newer than that execution
+  if (
+    isSafeWebhookActionConfig(automation.action.config) &&
+    automation.action.config.lastFailingExecutionId
+  ) {
+    // First get the timestamp of the last failing execution
+    const lastFailingExecution = await prisma.automationExecution.findUnique({
+      where: {
+        id: automation.action.config.lastFailingExecutionId,
+      },
+      select: {
+        createdAt: true,
+      },
+    });
+
+    if (lastFailingExecution) {
+      whereClause.createdAt = {
+        gt: lastFailingExecution.createdAt,
+      };
+    }
+  }
+
+  const executions = await prisma.automationExecution.findMany({
+    where: whereClause,
     orderBy: {
       createdAt: "desc",
     },

packages/shared/src/server/repositories/clickhouse.ts

@@ -36,7 +36,7 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces" | "observations";
+  table: "scores" | "traces" | "observations" | "traces_null";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>; // eslint-disable-line no-unused-vars
   tags?: Record<string, string>;

packages/shared/src/server/repositories/dataset-run-items.ts

@@ -0,0 +1,458 @@
+import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
+import { type OrderByState } from "../../interfaces/orderBy";
+import { datasetRunItemsTableUiColumnDefinitions } from "../../tableDefinitions";
+import { FilterState } from "../../types";
+import {
+  createFilterFromFilterState,
+  FilterList,
+  orderByToClickhouseSql,
+  StringFilter,
+} from "../queries";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "./clickhouse";
+import { convertDatasetRunItemClickhouseToDomain } from "./dataset-run-items-converters";
+import { DatasetRunItemRecordReadType } from "./definitions";
+import { env } from "../../env";
+import { commandClickhouse } from "./clickhouse";
+import Decimal from "decimal.js";
+
+type DatasetRunItemsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  filter: FilterState;
+  orderBy?: OrderByState | OrderByState[];
+  limit?: number;
+  offset?: number;
+};
+
+type DatasetRunsMetricsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+};
+
+export type DatasetRunsMetrics = {
+  id: string;
+  projectId: string;
+  createdAt: Date;
+  datasetId: string;
+  countRunItems: number;
+  avgTotalCost: Decimal;
+  avgLatency: number;
+};
+
+type DatasetRunsMetricsRecordType = {
+  dataset_run_id: string;
+  project_id: string;
+  dataset_run_created_at: string;
+  dataset_id: string;
+  count_run_items: number;
+  avg_latency_seconds: number;
+  avg_total_cost: number;
+};
+
+const convertDatasetRunsMetricsRecord = (
+  record: DatasetRunsMetricsRecordType,
+): DatasetRunsMetrics => {
+  return {
+    id: record.dataset_run_id,
+    projectId: record.project_id,
+    createdAt: parseClickhouseUTCDateTimeFormat(record.dataset_run_created_at),
+    datasetId: record.dataset_id,
+    countRunItems: record.count_run_items,
+    avgTotalCost: record.avg_total_cost
+      ? new Decimal(record.avg_total_cost)
+      : new Decimal(0),
+    avgLatency: record.avg_latency_seconds ?? 0,
+  };
+};
+
+const getProjectDatasetIdDefaultFilter = (
+  projectId: string,
+  datasetId: string,
+) => {
+  return {
+    datasetRunItemsFilter: new FilterList([
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "project_id",
+        operator: "=",
+        value: projectId,
+      }),
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "dataset_id",
+        operator: "=",
+        value: datasetId,
+      }),
+    ]),
+  };
+};
+
+const getDatasetRunsTableInternal = async <T>(
+  opts: DatasetRunsMetricsTableQuery & {
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, orderBy, limit, offset } = opts;
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    orderByArray.push(orderBy);
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    WITH observations_filtered AS (
+      SELECT
+        o.id,
+        o.trace_id,
+        o.project_id,
+        o.start_time,
+        o.end_time,
+        o.total_cost
+      FROM observations o FINAL
+      WHERE o.project_id = {projectId: String}
+        AND o.start_time >= (
+          SELECT min(dri.dataset_run_created_at) - INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+        AND o.start_time <= (
+          SELECT max(dri.dataset_run_created_at) + INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+    ),
+    traces_aggregated AS (
+      SELECT
+        of.trace_id,
+        of.project_id,
+        dateDiff('millisecond', min(of.start_time), max(of.end_time)) as latency_ms,
+        sum(of.total_cost) as total_cost
+      FROM observations_filtered of
+      JOIN dataset_run_items dri ON dri.trace_id = of.trace_id 
+        AND dri.project_id = of.project_id
+        AND dri.observation_id IS NULL  -- Only for trace-level dataset run items
+      WHERE dri.dataset_id = {datasetId: String}
+      GROUP BY of.trace_id, of.project_id
+    ),
+    observations_direct AS (
+      SELECT
+        dri.observation_id,
+        dri.project_id,
+        dri.trace_id,
+        of.total_cost,
+        dateDiff('millisecond', of.start_time, of.end_time) as latency_ms
+      FROM dataset_run_items dri
+      JOIN observations_filtered of ON dri.observation_id = of.id
+        AND dri.project_id = of.project_id
+        AND dri.trace_id = of.trace_id
+      WHERE dri.dataset_id = {datasetId: String}
+        AND dri.observation_id IS NOT NULL  -- Only for observation-level dataset run items
+    )
+    SELECT DISTINCT
+      dri.dataset_run_id as dataset_run_id,
+      dri.project_id as project_id,
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count_run_items,
+      
+      -- Latency metrics (priority: observation > trace)
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.latency_ms IS NOT NULL
+        THEN od.latency_ms / 1000.0
+        ELSE COALESCE(ta.latency_ms / 1000.0, 0)
+      END) as avg_latency_seconds,
+      
+      -- Cost metrics (priority: observation > trace)  
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.total_cost IS NOT NULL
+        THEN od.total_cost
+        ELSE COALESCE(ta.total_cost, 0)
+      END) as avg_total_cost
+    FROM dataset_run_items dri 
+    LEFT JOIN traces_aggregated ta
+      ON dri.trace_id = ta.trace_id
+      AND dri.project_id = ta.project_id
+    LEFT JOIN observations_direct od
+      ON dri.observation_id = od.observation_id
+      AND dri.project_id = od.project_id
+      AND dri.trace_id = od.trace_id
+    WHERE ${appliedFilter.query}
+    GROUP BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_run_created_at
+    ORDER BY dri.dataset_run_created_at DESC
+    ${orderByClause}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      projectId,
+      datasetId,
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunsTableMetricsCh = async (
+  opts: DatasetRunsMetricsTableQuery,
+): Promise<DatasetRunsMetrics[]> => {
+  // First get the metrics (latency, cost, counts)
+  const rows = await getDatasetRunsTableInternal<DatasetRunsMetricsRecordType>({
+    ...opts,
+    tags: { kind: "list" },
+  });
+
+  return rows.map(convertDatasetRunsMetricsRecord);
+};
+
+const getDatasetRunItemsTableInternal = async <T>(
+  opts: DatasetRunItemsTableQuery & {
+    select: "count" | "rows";
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, filter, orderBy, limit, offset } = opts;
+
+  let selectString = "";
+
+  switch (opts.select) {
+    case "count":
+      selectString =
+        "count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count";
+      break;
+    case "rows":
+      selectString = `
+      dri.id as id, 
+      dri.project_id as project_id, 
+      dri.trace_id as trace_id, 
+      dri.observation_id as observation_id, 
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_id as dataset_run_id, 
+      dri.dataset_item_id as dataset_item_id, 
+      dri.error as error,
+      dri.created_at as created_at, 
+      dri.updated_at as updated_at,
+      dri.dataset_run_name as dataset_run_name,
+      dri.dataset_run_description as dataset_run_description,
+      dri.dataset_run_metadata as dataset_run_metadata,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      dri.dataset_item_input as dataset_item_input,
+      dri.dataset_item_expected_output as dataset_item_expected_output,
+      dri.dataset_item_metadata as dataset_item_metadata,
+      dri.is_deleted as is_deleted,
+      dri.event_ts as event_ts`;
+      break;
+    default:
+      throw new Error(`Unknown select type: ${opts.select}`);
+  }
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+
+  datasetRunItemsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      datasetRunItemsTableUiColumnDefinitions,
+    ),
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    if (Array.isArray(orderBy)) {
+      orderByArray.push(...orderBy);
+    } else {
+      orderByArray.push(orderBy);
+    }
+  }
+
+  // Add event_ts DESC for row queries (for deduplication)
+  if (opts.select === "rows") {
+    orderByArray.push({
+      column: "eventTs",
+      order: "DESC",
+    });
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    SELECT
+      ${selectString}
+    FROM dataset_run_items dri 
+    WHERE ${appliedFilter.query}
+    ${orderByClause}
+    ${opts.select === "rows" ? "LIMIT 1 BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id" : ""}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunItemsByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<DatasetRunItemDomain[]> => {
+  const rows =
+    await getDatasetRunItemsTableInternal<DatasetRunItemRecordReadType>({
+      ...opts,
+      select: "rows",
+      tags: { kind: "list" },
+    });
+
+  return rows.map(convertDatasetRunItemClickhouseToDomain);
+};
+
+export const getDatasetRunItemsCountByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<number> => {
+  const rows = await getDatasetRunItemsTableInternal<{ count: string }>({
+    ...opts,
+    select: "count",
+    tags: { kind: "list" },
+  });
+
+  return Number(rows[0]?.count);
+};
+
+export const deleteDatasetRunItemsByProjectId = async ({
+  projectId,
+}: {
+  projectId: string;
+}) => {
+  const query = `
+      DELETE FROM dataset_run_items
+      WHERE project_id = {projectId: String};
+    `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetId = async ({
+  projectId,
+  datasetId,
+}: {
+  projectId: string;
+  datasetId: string;
+}) => {
+  const query = `
+  DELETE FROM dataset_run_items
+  WHERE project_id = {projectId: String}
+  AND dataset_id = {datasetId: String}
+`;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetRunIds = async ({
+  projectId,
+  datasetRunIds,
+  datasetId,
+}: {
+  projectId: string;
+  datasetRunIds: string[];
+  datasetId: string;
+}) => {
+  const query = `
+    DELETE FROM dataset_run_items
+    WHERE project_id = {projectId: String}
+    AND dataset_id = {datasetId: String}
+    AND dataset_run_id IN ({datasetRunIds: Array(String)})
+  `;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};

packages/shared/src/server/repositories/definitions.ts

@@ -122,7 +122,7 @@ export const traceRecordInsertSchema = traceRecordBaseSchema.extend({
 });
 export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;
 
-export const traceMtRecordInsertSchema = z.object({
+export const traceNullRecordInsertSchema = z.object({
   // Identifiers
   project_id: z.string(),
   id: z.string(),
@@ -157,7 +157,9 @@ export const traceMtRecordInsertSchema = z.object({
   updated_at: z.number(),
   event_ts: z.number(),
 });
-export type TraceMtRecordInsertType = z.infer<typeof traceMtRecordInsertSchema>;
+export type TraceNullRecordInsertType = z.infer<
+  typeof traceNullRecordInsertSchema
+>;
 
 export const scoreRecordBaseSchema = z.object({
   id: z.string(),
@@ -355,31 +357,37 @@ export const convertPostgresDatasetRunItemToInsert = (
     project_id: datasetRunItem.project_id,
     dataset_run_id: datasetRunItem.dataset_run_id,
     dataset_item_id: datasetRunItem.dataset_item_id,
+    dataset_id: datasetRunItem.dataset_id,
     trace_id: datasetRunItem.trace_id,
     observation_id: datasetRunItem.observation_id,
     error: datasetRunItem.error,
     created_at: datasetRunItem.created_at?.getTime(),
     updated_at: datasetRunItem.updated_at?.getTime(),
     // denormalized run data
-    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
-    dataset_id: datasetRunItem.dataset_id,
     dataset_run_name: datasetRunItem.dataset_run_name,
     dataset_run_description: datasetRunItem.dataset_run_description,
     dataset_run_metadata:
-      typeof datasetRunItem.dataset_run_metadata === "string"
-        ? { dataset_run_metadata: datasetRunItem.dataset_run_metadata }
+      typeof datasetRunItem.dataset_run_metadata === "string" ||
+      typeof datasetRunItem.dataset_run_metadata === "number" ||
+      typeof datasetRunItem.dataset_run_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_run_metadata }
         : Array.isArray(datasetRunItem.dataset_run_metadata)
-          ? { dataset_run_metadata: datasetRunItem.dataset_run_metadata }
-          : datasetRunItem.dataset_run_metadata,
+          ? { metadata: datasetRunItem.dataset_run_metadata }
+          : (datasetRunItem.dataset_run_metadata ?? {}),
+    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
     // denormalized item data
-    dataset_item_input: datasetRunItem.dataset_item_input,
-    dataset_item_expected_output: datasetRunItem.dataset_item_expected_output,
+    dataset_item_input: JSON.stringify(datasetRunItem.dataset_item_input),
+    dataset_item_expected_output: JSON.stringify(
+      datasetRunItem.dataset_item_expected_output,
+    ),
     dataset_item_metadata:
-      typeof datasetRunItem.dataset_item_metadata === "string"
-        ? { dataset_item_metadata: datasetRunItem.dataset_item_metadata }
+      typeof datasetRunItem.dataset_item_metadata === "string" ||
+      typeof datasetRunItem.dataset_item_metadata === "number" ||
+      typeof datasetRunItem.dataset_item_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_item_metadata }
         : Array.isArray(datasetRunItem.dataset_item_metadata)
-          ? { dataset_item_metadata: datasetRunItem.dataset_item_metadata }
-          : datasetRunItem.dataset_item_metadata,
+          ? { metadata: datasetRunItem.dataset_item_metadata }
+          : (datasetRunItem.dataset_item_metadata ?? {}),
     event_ts: datasetRunItem.created_at?.getTime(),
     is_deleted: 0,
   };
@@ -489,9 +497,9 @@ export const convertPostgresScoreToInsert = (
   };
 };
 
-export const convertTraceToTraceMt = (
+export const convertTraceToTraceNull = (
   traceRecord: TraceRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: traceRecord.project_id,
@@ -529,13 +537,13 @@ export const convertTraceToTraceMt = (
   };
 };
 
-export const convertObservationToTraceMt = (
+export const convertObservationToTraceNull = (
   observationRecord: ObservationRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: observationRecord.project_id,
-    // Use trace_id as the id in traces_mt. Always set given the conditions around calling the function
+    // Use trace_id as the id in traces_null. Always set given the conditions around calling the function
     id: observationRecord.trace_id || "",
     start_time: observationRecord.start_time,
     end_time: observationRecord.end_time || null,
@@ -573,13 +581,13 @@ export const convertObservationToTraceMt = (
   };
 };
 
-export const convertScoreToTraceMt = (
+export const convertScoreToTraceNull = (
   scoreRecord: ScoreRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: scoreRecord.project_id,
-    // Use trace_id as the id in traces_mt. Always set given the conditions around calling the function
+    // Use trace_id as the id in traces_null. Always set given the conditions around calling the function
     id: scoreRecord.trace_id || "",
     start_time: scoreRecord.timestamp,
     end_time: null, // scores don't have end_time

packages/shared/src/server/repositories/index.ts

@@ -14,3 +14,4 @@ export * from "./blobStorageLog";
 export * from "./environments";
 export * from "./automation-repository";
 export * from "./dataset-run-items-converters";
+export * from "./dataset-run-items";

packages/shared/src/server/repositories/observations.ts

@@ -23,7 +23,8 @@ import {
   observationsTableUiColumnDefinitions,
 } from "../../tableDefinitions";
 import { OrderByState } from "../../interfaces/orderBy";
-import { getTracesByIds } from "./traces";
+import { getTimeframesTracesAMT, getTracesByIds } from "./traces";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertObservation } from "./observations_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -36,6 +37,7 @@ import { TracingSearchType } from "../../interfaces/search";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { ObservationType } from "../../domain";
 import { recordDistribution } from "../instrumentation";
+import { DEFAULT_RENDERING_PROPS, RenderingProps } from "../utils/rendering";
 
 /**
  * Checks if observation exists in clickhouse.
@@ -288,7 +290,7 @@ export const getObservationForTraceIdByName = async (
     },
   });
 
-  return records.map(convertObservation);
+  return records.map((record) => convertObservation(record));
 };
 
 export const getObservationById = async ({
@@ -298,6 +300,7 @@ export const getObservationById = async ({
   startTime,
   type,
   traceId,
+  renderingProps = DEFAULT_RENDERING_PROPS,
 }: {
   id: string;
   projectId: string;
@@ -305,6 +308,7 @@ export const getObservationById = async ({
   startTime?: Date;
   type?: ObservationType;
   traceId?: string;
+  renderingProps?: RenderingProps;
 }) => {
   const records = await getObservationByIdInternal({
     id,
@@ -313,8 +317,11 @@ export const getObservationById = async ({
     startTime,
     type,
     traceId,
+    renderingProps,
   });
-  const mapped = records.map(convertObservation);
+  const mapped = records.map((record) =>
+    convertObservation(record, renderingProps),
+  );
 
   mapped.forEach((observation) => {
     recordDistribution(
@@ -384,7 +391,7 @@ export const getObservationsById = async (
     query,
     params: { ids, projectId },
   });
-  return records.map(convertObservation);
+  return records.map((record) => convertObservation(record));
 };
 
 const getObservationByIdInternal = async ({
@@ -394,6 +401,7 @@ const getObservationByIdInternal = async ({
   startTime,
   type,
   traceId,
+  renderingProps = DEFAULT_RENDERING_PROPS,
 }: {
   id: string;
   projectId: string;
@@ -401,6 +409,7 @@ const getObservationByIdInternal = async ({
   startTime?: Date;
   type?: ObservationType;
   traceId?: string;
+  renderingProps?: RenderingProps;
 }) => {
   const query = `
   SELECT
@@ -417,7 +426,7 @@ const getObservationByIdInternal = async ({
     level,
     status_message,
     version,
-    ${fetchWithInputOutput ? "input, output," : ""}
+    ${fetchWithInputOutput ? (renderingProps.truncated ? `left(input, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as input, left(output, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as output,` : "input, output,") : ""}
     provided_model_name,
     internal_model_id,
     model_parameters,
@@ -612,12 +621,14 @@ const getObservationsTableInternal = async <T>(
   } = opts;
 
   const selectString = selectIOAndMetadata
-    ? `
-    ${select},
-    ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
-  `
+    ? `${select}, o.input, o.output, o.metadata`
     : select;
 
+  const timeFilter = filter.find(
+    (f) =>
+      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
+  );
+
   const scoresFilter = new FilterList([
     new StringFilter({
       clickhouseTable: "scores",
@@ -627,33 +638,22 @@ const getObservationsTableInternal = async <T>(
     }),
   ]);
 
-  const timeFilter = opts.filter.find(
-    (f) =>
-      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
-  );
-
-  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
-  const traceTableFilter = opts.filter.filter(
-    (f) =>
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(f.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(f.column),
-  );
-
   const hasScoresFilter = filter.some((f) =>
     f.column.toLowerCase().includes("scores"),
   );
 
-  const orderByTraces = opts.orderBy
-    ? observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(opts.orderBy.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(opts.orderBy.column)
+  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
+  const traceTableFilter = filter.filter((f) =>
+    observationsTableTraceUiColumnDefinitions.some(
+      (c) => c.uiTableId === f.column || c.uiTableName === f.column,
+    ),
+  );
+
+  const orderByTraces = orderBy
+    ? observationsTableTraceUiColumnDefinitions.some(
+        (c) =>
+          c.uiTableId === orderBy.column || c.uiTableName === orderBy.column,
+      )
     : undefined;
 
   timeFilter
@@ -759,7 +759,7 @@ const getObservationsTableInternal = async <T>(
       SELECT
        ${selectString}
       FROM observations o 
-        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN __TRACE_TABLE__ t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
         ${hasScoresFilter ? `LEFT JOIN scores_agg AS s ON s.trace_id = o.trace_id and s.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
         
@@ -769,30 +769,52 @@ const getObservationsTableInternal = async <T>(
       ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-  const res = await queryClickhouse<T>({
-    query,
-    params: {
-      ...appliedScoresFilter.params,
-      ...appliedObservationsFilter.params,
-      ...(timeFilter
-        ? {
-            tracesTimestampFilter: convertDateToClickhouseDateTime(
-              timeFilter.value as Date,
-            ),
-          }
-        : {}),
-      ...search.params,
+  return measureAndReturn({
+    operationName: "getObservationsTableInternal",
+    projectId,
+    minStartTime: (timeFilter?.value as Date) || undefined,
+    input: {
+      params: {
+        ...appliedScoresFilter.params,
+        ...appliedObservationsFilter.params,
+        ...(timeFilter
+          ? {
+              tracesTimestampFilter: convertDateToClickhouseDateTime(
+                timeFilter.value as Date,
+              ),
+            }
+          : {}),
+        ...search.params,
+      },
+      tags: {
+        ...(opts.tags ?? {}),
+        feature: "tracing",
+        type: "observation",
+        projectId,
+        kind: opts.select,
+        operation_name: "getObservationsTableInternal",
+      },
     },
-    tags: {
-      ...(opts.tags ?? {}),
-      feature: "tracing",
-      type: "observation",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(
+        (timeFilter?.value as Date) || undefined,
+      );
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };
 
 export const getObservationsGroupedByModel = async (
@@ -1473,6 +1495,9 @@ export const getObservationsForBlobStorageExport = function (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
+    },
   });
 
   return records;
@@ -1531,7 +1556,7 @@ export const getGenerationsForPostHog = async function* (
       projectId,
     },
     clickhouseConfigs: {
-      request_timeout: 300_000, // 5 minutes
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
       clickhouse_settings: {
         join_algorithm: "grace_hash",
         grace_hash_join_initial_buckets: "32",
@@ -1545,6 +1570,7 @@ export const getGenerationsForPostHog = async function* (
       timestamp: record.start_time,
       langfuse_generation_name: record.name,
       langfuse_trace_name: record.trace_name,
+      langfuse_trace_id: record.trace_id,
       langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
       langfuse_id: record.id,
       langfuse_cost_usd: record.total_cost,

packages/shared/src/server/repositories/observations_converters.ts

@@ -1,15 +1,20 @@
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
-import { parseJsonPrioritised } from "../../utils/json";
 import {
   Observation,
   ObservationLevelType,
   ObservationType,
 } from "../../domain";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import {
+  RenderingProps,
+  DEFAULT_RENDERING_PROPS,
+  applyInputOutputRendering,
+} from "../utils/rendering";
 
 export const convertObservation = (
   record: ObservationRecordReadType,
+  renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,
 ): Observation => {
   const reducedCostDetails = reduceUsageOrCostDetails(record.cost_details);
   const reducedUsageDetails = reduceUsageOrCostDetails(record.usage_details);
@@ -30,10 +35,8 @@ export const convertObservation = (
     level: record.level as ObservationLevelType,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
-    output: record.output
-      ? (parseJsonPrioritised(record.output) ?? null)
-      : null,
+    input: applyInputOutputRendering(record.input, renderingProps),
+    output: applyInputOutputRendering(record.output, renderingProps),
     modelParameters: record.model_parameters
       ? (JSON.parse(record.model_parameters) ?? null)
       : null,

packages/shared/src/server/repositories/scores.ts

@@ -32,6 +32,7 @@ import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { recordDistribution } from "../instrumentation";
 import { prisma } from "../../db";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -284,6 +285,53 @@ export const getScoresForDatasetRuns = async <
   return rows.map(convertToScore);
 };
 
+export const getTraceScoresForDatasetRuns = async (
+  projectId: string,
+  datasetRunIds: string[],
+): Promise<Array<{ dataset_run_id: string } & any>> => {
+  if (datasetRunIds.length === 0) return [];
+
+  const query = `
+    SELECT 
+      s.* EXCEPT (metadata),
+      length(mapKeys(s.metadata)) > 0 AS has_metadata,
+      dri.dataset_run_id as run_id
+    FROM dataset_run_items dri 
+    JOIN scores s FINAL ON dri.trace_id = s.trace_id 
+      AND dri.project_id = s.project_id
+    WHERE dri.project_id = {projectId: String}
+      AND dri.dataset_run_id IN {datasetRunIds: Array(String)}
+      AND s.project_id = {projectId: String}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id, dri.dataset_run_id
+  `;
+
+  const rows = await queryClickhouse<
+    Omit<ScoreRecordReadType, "metadata"> & {
+      has_metadata: 0 | 1;
+      run_id: string;
+    }
+  >({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+    },
+    tags: {
+      feature: "dataset-run-items",
+      type: "trace-scores",
+      kind: "list",
+      projectId,
+    },
+  });
+
+  return rows.map((row) => ({
+    ...convertToScore({ ...row, metadata: {} }),
+    datasetRunId: row.run_id,
+    hasMetadata: !!row.has_metadata,
+  }));
+};
+
 // Used in multiple places, including the public API, hence the non-default exclusion of metadata via excludeMetadata flag
 export const getScoresForTraces = async <
   ExcludeMetadata extends boolean,
@@ -895,31 +943,49 @@ const getScoresUiGeneric = async <T>(props: {
       SELECT 
           ${select}
       FROM scores s final
-      ${performTracesJoin ? "LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
+      ${performTracesJoin ? "LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId: projectId,
-      ...(scoresFilterRes ? scoresFilterRes.params : {}),
-      limit: limit,
-      offset: offset,
+  return measureAndReturn({
+    operationName: "getScoresUiGeneric",
+    projectId,
+    input: {
+      params: {
+        projectId: projectId,
+        ...(scoresFilterRes ? scoresFilterRes.params : {}),
+        limit: limit,
+        offset: offset,
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "score",
+        projectId,
+        select: props.select,
+        operation_name: "getScoresUiGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "score",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces_all_amt"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return rows;
 };
 
 export const getScoreNames = async (
@@ -1320,6 +1386,9 @@ export const getScoresForBlobStorageExport = function (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
+    },
   });
 
   return records;
@@ -1339,6 +1408,7 @@ export const getScoresForPostHog = async function* (
       s.data_type as data_type,
       s.comment as comment,
       s.environment as environment,
+      t.id as trace_id,
       t.name as trace_name,
       t.session_id as trace_session_id,
       t.user_id as trace_user_id,
@@ -1370,7 +1440,7 @@ export const getScoresForPostHog = async function* (
       projectId,
     },
     clickhouseConfigs: {
-      request_timeout: 300_000, // 5 minutes
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
       clickhouse_settings: {
         join_algorithm: "grace_hash",
         grace_hash_join_initial_buckets: "32",
@@ -1389,6 +1459,7 @@ export const getScoresForPostHog = async function* (
       langfuse_score_string_value: record.string_value,
       langfuse_score_data_type: record.data_type,
       langfuse_trace_name: record.trace_name,
+      langfuse_trace_id: record.trace_id,
       langfuse_id: record.id,
       langfuse_session_id: record.trace_session_id,
       langfuse_project_id: projectId,

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,9 +1,13 @@
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import { parseJsonPrioritised } from "../../utils/json";
 import { TraceDomain } from "../../domain";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import {
+  RenderingProps,
+  DEFAULT_RENDERING_PROPS,
+  applyInputOutputRendering,
+} from "../utils/rendering";
 
 export const convertTraceDomainToClickhouse = (
   trace: TraceDomain,
@@ -33,6 +37,7 @@ export const convertTraceDomainToClickhouse = (
 
 export const convertClickhouseToDomain = (
   record: TraceRecordReadType,
+  renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,
 ): TraceDomain => {
   return {
     id: record.id,
@@ -47,10 +52,8 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
-    output: record.output
-      ? (parseJsonPrioritised(record.output) ?? null)
-      : null,
+    input: applyInputOutputRendering(record.input, renderingProps),
+    output: applyInputOutputRendering(record.output, renderingProps),
     metadata: parseMetadataCHRecordToDomain(record.metadata),
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),

packages/shared/src/server/services/SlackService.ts

@@ -0,0 +1,386 @@
+/**
+ * Slack Integration Service
+ *
+ * Simplified service that properly uses the official Slack SDK libraries:
+ * - @slack/oauth InstallProvider for OAuth flow management
+ * - @slack/web-api WebClient for Slack API operations
+ * - Metadata-based project-to-team mapping
+ */
+
+import { WebClient } from "@slack/web-api";
+import { InstallProvider } from "@slack/oauth";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { prisma } from "../../db";
+import { encrypt, decrypt } from "../../encryption";
+
+// Types for Slack integration
+export interface SlackChannel {
+  id: string;
+  name: string;
+  isPrivate: boolean;
+  isMember: boolean;
+}
+
+export interface SlackMessageParams {
+  client: WebClient;
+  channelId: string;
+  blocks: any[];
+  text?: string;
+}
+
+export interface SlackMessageResponse {
+  messageTs: string;
+  channel: string;
+}
+
+// Interface for Slack installation metadata
+export interface SlackInstallationMetadata {
+  projectId: string;
+}
+
+/**
+ * Type guard to validate Slack installation metadata
+ */
+function isSlackInstallationMetadata(
+  metadata: unknown,
+): metadata is SlackInstallationMetadata {
+  return (
+    typeof metadata === "object" &&
+    metadata !== null &&
+    "projectId" in metadata &&
+    typeof metadata.projectId === "string" &&
+    metadata.projectId.length > 0
+  );
+}
+
+/**
+ * Helper function to safely parse and validate Slack installation metadata
+ */
+export function parseSlackInstallationMetadata(
+  metadata: unknown,
+): SlackInstallationMetadata {
+  if (typeof metadata !== "string") {
+    throw new Error("Installation metadata must be a string");
+  }
+
+  let parsedMetadata: unknown;
+  try {
+    parsedMetadata = JSON.parse(metadata);
+  } catch {
+    throw new Error("Failed to parse installation metadata as JSON");
+  }
+
+  if (!isSlackInstallationMetadata(parsedMetadata)) {
+    throw new Error(
+      "Invalid installation metadata: missing or invalid projectId",
+    );
+  }
+
+  return parsedMetadata;
+}
+
+/**
+ * Slack Service Class
+ *
+ * Uses InstallProvider for OAuth flow and metadata-based project mapping.
+ * Much simpler than the previous implementation while maintaining all functionality.
+ */
+export class SlackService {
+  private static instance: SlackService | null = null;
+  private installer: InstallProvider;
+
+  private constructor() {
+    this.installer = new InstallProvider({
+      clientId: env.SLACK_CLIENT_ID!,
+      clientSecret: env.SLACK_CLIENT_SECRET!,
+      stateSecret: env.SLACK_STATE_SECRET!,
+      installUrlOptions: {
+        scopes: ["channels:read", "chat:write", "chat:write.public"],
+      },
+      installationStore: {
+        storeInstallation: async (installation) => {
+          try {
+            const metadata = parseSlackInstallationMetadata(
+              installation.metadata,
+            );
+            const projectId = metadata.projectId;
+
+            logger.info("Storing Slack installation for project", {
+              projectId,
+              teamId: installation.team?.id,
+              teamName: installation.team?.name,
+            });
+
+            // Store by projectId (one integration per project)
+            await prisma.slackIntegration.upsert({
+              where: { projectId },
+              create: {
+                projectId,
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+              update: {
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+            });
+
+            logger.info("Slack installation stored successfully", {
+              projectId,
+              teamId: installation.team?.id,
+            });
+          } catch (error) {
+            logger.error("Failed to store Slack installation", { error });
+            throw error;
+          }
+        },
+
+        fetchInstallation: async (installQuery) => {
+          try {
+            // Handle both teamId and projectId lookups
+            // When SDK calls with teamId, we treat it as projectId
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided");
+            }
+
+            const integration = await prisma.slackIntegration.findFirst({
+              where: {
+                OR: [
+                  { teamId: lookupId }, // Actual team ID lookup
+                  { projectId: lookupId }, // Project ID lookup (our custom usage)
+                ],
+              },
+            });
+
+            if (!integration) {
+              throw new Error("Slack integration not found");
+            }
+
+            // Return full Installation interface as expected by SDK
+            return {
+              team: {
+                id: integration.teamId,
+                name: integration.teamName,
+              },
+              bot: {
+                id: integration.botUserId,
+                token: decrypt(integration.botToken),
+                userId: integration.botUserId,
+                scopes: [],
+              },
+              enterprise: undefined,
+              user: {
+                token: undefined,
+                refreshToken: undefined,
+                expiresAt: undefined,
+                scopes: undefined,
+                id: integration.botUserId,
+              },
+            };
+          } catch (error) {
+            logger.error("Failed to fetch Slack installation", { error });
+            throw error;
+          }
+        },
+
+        deleteInstallation: async (installQuery) => {
+          try {
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided for deletion");
+            }
+
+            await prisma.slackIntegration.deleteMany({
+              where: {
+                OR: [{ teamId: lookupId }, { projectId: lookupId }],
+              },
+            });
+
+            logger.info("Slack installation deleted successfully", {
+              lookupId,
+            });
+          } catch (error) {
+            logger.error("Failed to delete Slack installation", { error });
+            throw error;
+          }
+        },
+      },
+    });
+  }
+
+  /**
+   * Get singleton instance of SlackService
+   */
+  static getInstance(): SlackService {
+    if (!SlackService.instance) {
+      SlackService.instance = new SlackService();
+    }
+    return SlackService.instance;
+  }
+
+  /**
+   * Get the configured InstallProvider instance for OAuth handling
+   */
+  getInstaller(): InstallProvider {
+    return this.installer;
+  }
+
+  /**
+   * Reset the singleton instance (useful for testing)
+   */
+  static resetInstance(): void {
+    SlackService.instance = null;
+  }
+
+  /**
+   * Delete Slack integration for a project
+   */
+  async deleteIntegration(projectId: string): Promise<void> {
+    try {
+      if (!this.installer.installationStore?.deleteInstallation) {
+        throw new Error("Installation store not configured");
+      }
+
+      await this.installer.installationStore.deleteInstallation({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      logger.info("Slack integration deleted for project", { projectId });
+    } catch (error) {
+      logger.error("Failed to delete Slack integration", { error, projectId });
+      throw new Error(
+        `Failed to delete integration: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get WebClient for a specific project
+   */
+  async getWebClientForProject(projectId: string): Promise<WebClient> {
+    try {
+      // Use projectId as the teamId parameter (handled by our fetchInstallation)
+      const auth = await this.installer.authorize({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      if (!auth.botToken) {
+        throw new Error("No bot token found for project");
+      }
+
+      const client = new WebClient(auth.botToken);
+      logger.debug("Created WebClient for project", { projectId });
+
+      return client;
+    } catch (error) {
+      logger.error("Failed to create WebClient for project", {
+        error,
+        projectId,
+      });
+      throw new Error(
+        `Failed to create WebClient: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get channels accessible to the bot
+   */
+  async getChannels(client: WebClient): Promise<SlackChannel[]> {
+    try {
+      const result = await client.conversations.list({
+        exclude_archived: true,
+        types: "public_channel",
+        limit: 200,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Slack API error: ${result.error}`);
+      }
+
+      const channels: SlackChannel[] = (result.channels || []).map(
+        (channel) => ({
+          id: channel.id!,
+          name: channel.name!,
+          isPrivate: channel.is_private || false,
+          isMember: channel.is_member || false,
+        }),
+      );
+
+      logger.debug("Retrieved channels from Slack", {
+        channelCount: channels.length,
+      });
+
+      return channels;
+    } catch (error) {
+      logger.error("Failed to fetch channels", { error });
+      throw new Error(
+        `Failed to fetch channels: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Send a message to a Slack channel
+   */
+  async sendMessage(params: SlackMessageParams): Promise<SlackMessageResponse> {
+    try {
+      const result = await params.client.chat.postMessage({
+        channel: params.channelId,
+        blocks: params.blocks,
+        text: params.text || "Langfuse Notification",
+        unfurl_links: false,
+        unfurl_media: false,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Failed to send message: ${result.error}`);
+      }
+
+      const response = {
+        messageTs: result.ts!,
+        channel: result.channel!,
+      };
+
+      logger.info("Message sent successfully to Slack", {
+        channel: params.channelId,
+        messageTs: response.messageTs,
+      });
+
+      return response;
+    } catch (error) {
+      logger.error("Failed to send message", {
+        error,
+        channelId: params.channelId,
+      });
+      throw new Error(
+        `Failed to send message: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Validate a WebClient instance
+   */
+  async validateClient(client: WebClient): Promise<boolean> {
+    try {
+      const result = await client.auth.test();
+      return result.ok || false;
+    } catch (error) {
+      logger.warn("Client validation failed", { error });
+      return false;
+    }
+  }
+}

packages/shared/src/server/services/StorageService.ts

@@ -104,6 +104,7 @@ export class StorageServiceFactory {
   }
 }
 
+let azureContainersExists: Record<string, boolean> = {};
 class AzureBlobStorageService implements StorageService {
   private client: ContainerClient;
   private container: string;
@@ -139,8 +140,18 @@ class AzureBlobStorageService implements StorageService {
   }
 
   private async createContainerIfNotExists(): Promise<void> {
+    // Skip container existence check if environment variable is set
+    if (env.LANGFUSE_AZURE_SKIP_CONTAINER_CHECK === "true") {
+      return;
+    }
+
     try {
+      if (azureContainersExists[this.container]) {
+        return; // Container already exists, no need to create it again
+      }
       await this.client.createIfNotExists();
+      azureContainersExists[this.container] = true; // Mark container as created
+      logger.info(`Azure Blob Storage container ${this.container} created`);
     } catch (err) {
       logger.error(
         `Failed to create Azure Blob Storage container ${this.container}`,

packages/shared/src/server/services/email/billingAlert/BillingAlertEmailTemplate.tsx

@@ -0,0 +1,134 @@
+import React from "react";
+import {
+  Body,
+  Button,
+  Container,
+  Head,
+  Heading,
+  Hr,
+  Html,
+  Img,
+  Preview,
+  Section,
+  Text,
+  Tailwind,
+  Row,
+  Column,
+} from "@react-email/components";
+
+interface BillingAlertEmailProps {
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const BillingAlertEmailTemplate = ({
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  return (
+    <Html>
+      <Head />
+      <Preview>
+        Your Langfuse Cloud usage is {`${currentUsage}`} events for the current
+        billing period
+      </Preview>
+      <Tailwind>
+        <Body className="bg-background my-auto mx-auto font-sans">
+          <Container className="mx-auto my-10 w-[465px] rounded border border-solid border-[#eaeaea] p-5">
+            <Section className="mt-8">
+              <Img
+                src="https://static.langfuse.com/langfuse_logo_transactional_email.png"
+                width="40"
+                height="40"
+                alt="Langfuse"
+                className="mx-auto my-0"
+              />
+            </Section>
+
+            <Section>
+              <Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
+                Usage Threshold Exceeded
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                Your organization &quot;{organizationName}&quot; has exceeded
+                the configured billing threshold
+              </Text>
+            </Section>
+
+            <Section className="mt-8">
+              <div className="bg-gray-50 border border-gray-200 rounded-lg p-4">
+                <Row>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Current Usage (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {currentUsage}
+                    </Text>
+                  </Column>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Alert Threshold (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {threshold}
+                    </Text>
+                  </Column>
+                </Row>
+              </div>
+            </Section>
+
+            <Section className="mt-8 text-center">
+              <Button
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
+                href={billingUrl}
+              >
+                View Billing Page and Manage Alerts
+              </Button>
+            </Section>
+
+            <Section className="mt-8">
+              <Heading className="text-black text-[18px] font-semibold">
+                What happens next?
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                • Your current billing cycle continues normally
+                <br />
+                • Charges will appear on your next invoice
+                <br />
+                • You can adjust usage or modify alert thresholds
+                <br />• Contact support if you have questions about your bill
+              </Text>
+            </Section>
+
+            <Hr className="border border-solid border-[#eaeaea] my-[26px] mx-0 w-full" />
+
+            <Section>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                This email was sent to {receiverEmail} regarding billing alerts
+                for &quot;{organizationName}&quot;.
+              </Text>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                Questions? Contact us at{" "}
+                <a
+                  href="mailto:support@langfuse.com"
+                  className="text-blue-600 no-underline"
+                >
+                  support@langfuse.com
+                </a>
+              </Text>
+            </Section>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  );
+};
+
+export default BillingAlertEmailTemplate;

packages/shared/src/server/services/email/billingAlert/sendBillingAlertEmail.ts

@@ -0,0 +1,59 @@
+import { createTransport } from "nodemailer";
+import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
+import { render } from "@react-email/render";
+import { BillingAlertEmailTemplate } from "./BillingAlertEmailTemplate";
+import { logger } from "../../../logger";
+
+export interface BillingAlertEmailProps {
+  env: Partial<
+    Record<"EMAIL_FROM_ADDRESS" | "SMTP_CONNECTION_URL", string | undefined>
+  >;
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const sendBillingAlertEmail = async ({
+  env,
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
+    logger.error(
+      "Missing environment variables for sending billing alert email.",
+    );
+    return;
+  }
+
+  try {
+    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
+
+    const emailSubject = `Langfuse Cloud Billing Alert: ${organizationName} usage exceeded ${threshold} events`;
+    const emailHtml = await render(
+      BillingAlertEmailTemplate({
+        organizationName,
+        currentUsage,
+        threshold,
+        billingUrl,
+        receiverEmail,
+      }),
+    );
+
+    await mailer.sendMail({
+      to: receiverEmail,
+      from: {
+        address: env.EMAIL_FROM_ADDRESS,
+        name: "Langfuse",
+      },
+      subject: emailSubject,
+      html: emailHtml,
+    });
+  } catch (error) {
+    logger.error(`Failed to send billing alert email`, error);
+  }
+};

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -3,6 +3,7 @@ import { OrderByState } from "../../interfaces/orderBy";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { FilterState } from "../../types";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
 import {
   getProjectIdDefaultFilter,
@@ -11,6 +12,7 @@ import {
 import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
   queryClickhouse,
+  getTimeframesTracesAMT,
 } from "../repositories";
 
 export type SessionDataReturnType = {
@@ -22,6 +24,8 @@ export type SessionDataReturnType = {
   trace_count: number;
   trace_tags: string[];
   trace_environment?: string;
+  scores_avg?: Array<Array<[string, number]>>;
+  score_categories?: Array<Array<string>>;
 };
 
 export type SessionWithMetricsReturnType = SessionDataReturnType & {
@@ -157,7 +161,9 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         session_total_cost,
         session_input_usage,
         session_output_usage,
-        session_total_usage`;
+        session_total_usage,
+        scores_avg,
+        score_categories`;
       break;
     default: {
       const exhaustiveCheckDefault: never = select;
@@ -165,7 +171,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     }
   }
 
-  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+  const { tracesFilter, scoresFilter } = getProjectIdDefaultFilter(projectId, {
     tracesPrefix: "s",
   });
 
@@ -174,6 +180,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
   const tracesFilterRes = tracesFilter
     .filter((f) => f.field !== "environment")
     .apply();
+  const scoresFilterRes = scoresFilter.apply();
 
   const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
     (f) =>
@@ -193,20 +200,25 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     );
   }
 
-  const additionalSingleTraceFilter = tracesFilter.find(
-    (f) =>
-      f.field === "bookmarked" ||
-      f.field === "session_id" ||
-      f.field === "environment",
-  );
-
-  if (additionalSingleTraceFilter) {
-    filters.push(additionalSingleTraceFilter);
-  }
+  tracesFilter
+    .filter(
+      (f) =>
+        f.field === "bookmarked" ||
+        f.field === "session_id" ||
+        f.field === "environment",
+    )
+    .forEach((f) => filters.push(f));
 
   const singleTraceFilter =
     filters.length > 0 ? new FilterList(filters).apply() : undefined;
 
+  const requiresScoresJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "scores") !== undefined ||
+    sessionCols.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "scores";
+
   const hasMetricsFilter =
     tracesFilter.find((f) =>
       [
@@ -217,6 +229,8 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         "session_total_usage",
         "session_output_usage",
         "session_input_usage",
+        "scores_avg",
+        "score_categories",
       ].includes(f.field),
     ) ||
     (orderBy &&
@@ -233,115 +247,188 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
 
   const selectMetrics = select === "metrics" || hasMetricsFilter;
 
+  const scoresCte = `scores_agg AS (
+    SELECT
+      project_id,
+      session_id AS score_session_id,
+      -- For numeric scores, use tuples of (name, avg_value)
+      groupArrayIf(
+        tuple(name, avg_value),
+        data_type IN ('NUMERIC', 'BOOLEAN')
+      ) AS scores_avg,
+      -- For categorical scores, use name:value format for improved query performance
+      groupArrayIf(
+        concat(name, ':', string_value),
+        data_type = 'CATEGORICAL' AND notEmpty(string_value)
+      ) AS score_categories
+    FROM (
+      SELECT
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value,
+        avg(value) avg_value
+      FROM scores s FINAL
+      WHERE 
+        project_id = {projectId: String}
+        ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
+      GROUP BY
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value
+      ) tmp
+    GROUP BY
+      project_id, session_id
+  )`;
+
   // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
   const query = `
-    WITH deduplicated_traces AS (
-      SELECT * EXCEPT input, output, metadata
-      FROM traces t
-      WHERE t.session_id IS NOT NULL 
-        AND t.project_id = {projectId: String}
-        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    deduplicated_observations AS (
-        SELECT * 
-        FROM observations o
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        AND o.trace_id IN (
-          SELECT id
-          FROM deduplicated_traces
-        )
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    observations_agg AS (
-      SELECT o.trace_id,
-            count(*) as obs_count,
-            min(o.start_time) as min_start_time,
-            max(o.end_time) as max_end_time,
-            sumMap(usage_details) as sum_usage_details,
-            sumMap(cost_details) as sum_cost_details,
-            anyLast(project_id) as project_id
-      FROM deduplicated_observations o
-      WHERE o.project_id = {projectId: String}
-      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-      GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            anyLast(t.environment) as trace_environment
-            -- Aggregate observations data at session level
-            ${
-              selectMetrics
-                ? `
-            ,
-            sum(o.obs_count) as total_observations,
-            -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
-            -- LLM session started on that date so this behaviour should yield better results.
-            date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage`
-                : ""
-            }
-        FROM deduplicated_traces t
-        ${
-          selectMetrics
-            ? `LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id`
-            : ""
-        }
-        WHERE t.session_id IS NOT NULL
+        WITH ${select === "metrics" || requiresScoresJoin ? `${scoresCte},` : ""}
+        deduplicated_traces AS (
+          SELECT * EXCEPT input, output, metadata
+          FROM __TRACE_TABLE__ t
+          WHERE t.session_id IS NOT NULL 
             AND t.project_id = {projectId: String}
             ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${sqlSelect}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
+            LIMIT 1 BY id, project_id
+        ),
+        deduplicated_observations AS (
+            SELECT * 
+            FROM observations o
+            WHERE o.project_id = {projectId: String}
+            ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+            AND o.trace_id IN (
+              SELECT id
+              FROM deduplicated_traces
+            )
+            ORDER BY event_ts DESC
+            LIMIT 1 BY id, project_id
+        ),
+        observations_agg AS (
+          SELECT o.trace_id,
+                count(*) as obs_count,
+                min(o.start_time) as min_start_time,
+                max(o.end_time) as max_end_time,
+                sumMap(usage_details) as sum_usage_details,
+                sumMap(cost_details) as sum_cost_details,
+                anyLast(project_id) as project_id
+          FROM deduplicated_observations o
+          WHERE o.project_id = {projectId: String}
+          ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+          GROUP BY o.trace_id
+        ),
+        session_data AS (
+            SELECT
+                t.session_id,
+                anyLast(t.project_id) as project_id,
+                max(t.timestamp) as max_timestamp,
+                min(t.timestamp) as min_timestamp,
+                groupArray(t.id) AS trace_ids,
+                groupUniqArray(t.user_id) AS user_ids,
+                count(*) as trace_count,
+                groupUniqArrayArray(t.tags) as trace_tags,
+                anyLast(t.environment) as trace_environment
+                -- Aggregate observations data at session level
+                ${
+                  selectMetrics
+                    ? `,
+                      sum(o.obs_count) as total_observations,
+                      -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
+                      -- LLM session started on that date so this behaviour should yield better results.
+                      date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
+                      sumMap(o.sum_usage_details) as session_usage_details,
+                      sumMap(o.sum_cost_details) as session_cost_details,
+                      ${
+                        select === "metrics" || requiresScoresJoin
+                          ? `groupUniqArrayArray(s.scores_avg) as scores_avg,
+                      groupUniqArrayArray(s.score_categories) as score_categories,`
+                          : ""
+                      }
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+                      sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+                      sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                    : ""
+                }
+            FROM deduplicated_traces t
+            ${
+              selectMetrics
+                ? `LEFT JOIN observations_agg o
+                   ON t.id = o.trace_id AND t.project_id = o.project_id`
+                : ""
+            }
+           ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_agg s on s.project_id = t.project_id and t.session_id = s.score_session_id` : ""}
+            WHERE t.session_id IS NOT NULL
+                AND t.project_id = {projectId: String}
+                ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+            GROUP BY t.session_id
+        )
+        SELECT ${sqlSelect}
+        FROM session_data s
+        WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+        ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+        `;
 
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
+  return measureAndReturn({
+    operationName: "getSessionsTableGeneric",
+    projectId,
+    minStartTime: filter?.find(
+      (f) =>
+        f.column === "min_timestamp" &&
+        (f.operator === ">=" || f.operator === ">"),
+    )?.value as Date | undefined,
+    input: {
+      params: {
+        projectId,
+        limit: limit,
+        offset: limit && page ? limit * page : 0,
+        ...tracesFilterRes.params,
+        ...singleTraceFilter?.params,
+        ...scoresFilterRes.params,
+        ...(traceTimestampFilter
+          ? {
+              observationsStartTime: convertDateToClickhouseDateTime(
+                traceTimestampFilter.value,
+              ),
+            }
+          : {}),
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "sessions-table",
+        projectId,
+        operation_name: "getSessionsTableGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "sessions-table",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      // Extract the timestamp from filter for AMT table selection
+      const fromTimestamp = filter?.find(
+        (f) =>
+          f.column === "min_timestamp" &&
+          (f.operator === ">=" || f.operator === ">"),
+      )?.value as Date | undefined;
+      const traceAmt = getTimeframesTracesAMT(fromTimestamp);
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -350,6 +350,7 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
   return measureAndReturn({
     operationName: "getTracesTableGeneric",
     projectId: props.projectId,
+    minStartTime: select !== "metrics" ? timeStampFilter?.value : undefined,
     input: props,
     existingExecution: async (props) => {
       let sqlSelect: string;
@@ -479,6 +480,8 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
           feature: "tracing",
           type: "traces-table",
           projectId,
+          experiment_amt: "original",
+          operation_name: "getTracesTableGeneric",
         },
         clickhouseConfigs,
       });
@@ -541,12 +544,8 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
         true,
       );
 
-      const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
       const chOrderBy = orderByToClickhouseSql(
-        [
-          defaultOrder ? [{ column: "timestamp", order: orderBy.order }] : null,
-          orderBy ?? null,
-        ].flat(),
+        [orderBy ?? null].flat(),
         tracesTableUiColumnDefinitions,
       );
 
@@ -556,7 +555,7 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
           : getTimeframesTracesAMT(timeStampFilter?.value);
 
       const query = `
-          ${observationsAndScoresCTE}        
+        ${observationsAndScoresCTE}
 
         SELECT ${sqlSelect}
         FROM ${tracesAmt} t FINAL
@@ -588,6 +587,8 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
           feature: "tracing",
           type: "traces-table",
           projectId,
+          experiment_amt: "new",
+          operation_name: "getTracesTableGeneric",
         },
         clickhouseConfigs,
       });

packages/shared/src/server/test-utils/clickhouse-helpers.ts

@@ -4,18 +4,16 @@ import {
   ObservationRecordInsertType,
   ScoreRecordInsertType,
   DatasetRunItemRecordInsertType,
-  convertTraceToTraceMt,
+  convertTraceToTraceNull,
 } from "../repositories/definitions";
 import { env } from "../../env";
 
 export const createTracesCh = async (trace: TraceRecordInsertType[]) => {
-  if (
-    env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES === "true"
-  ) {
+  if (env.LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES === "true") {
     await clickhouseClient().insert({
-      table: "traces_mt",
+      table: "traces_null",
       format: "JSONEachRow",
-      values: trace.map(convertTraceToTraceMt),
+      values: trace.map(convertTraceToTraceNull),
     });
   }
   return await clickhouseClient().insert({

packages/shared/src/server/traceDeletionProcessor.ts

@@ -0,0 +1,90 @@
+import { randomUUID } from "crypto";
+import { prisma } from "../db";
+import { TraceDeleteQueue } from "./redis/traceDelete";
+import { QueueJobs } from "./queues";
+import { logger } from "./logger";
+import { env } from "../env";
+
+export interface TraceDeletionProcessorOptions {
+  delayMs?: number; // Default from LANGFUSE_TRACE_DELETE_DELAY_MS env var
+}
+
+/**
+ * Efficient trace deletion processor that batches deletions for better performance.
+ *
+ * This function:
+ * 1. Creates a record in the pending_deletions table for each trace
+ * 2. Sends a deletion event to the queue with a configurable delay
+ * 3. The worker will batch delete all pending traces from ClickHouse
+ * 4. Sets the is_deleted flag to true after successful deletion
+ *
+ * @param projectId - The project ID
+ * @param traceIds - Array of trace IDs to delete
+ * @param options - Configuration options including delay
+ */
+export async function traceDeletionProcessor(
+  projectId: string,
+  traceIds: string[],
+  options: TraceDeletionProcessorOptions = {},
+): Promise<void> {
+  const { delayMs = env.LANGFUSE_TRACE_DELETE_DELAY_MS } = options;
+
+  if (traceIds.length === 0) {
+    logger.warn("traceDeletionProcessor called with empty traceIds array", {
+      projectId,
+    });
+    return;
+  }
+
+  logger.info(
+    `Processing trace deletion for ${traceIds.length} traces in project ${projectId}`,
+    {
+      projectId,
+      traceIds,
+      delayMs,
+    },
+  );
+
+  try {
+    // Create pending deletion records for all traces
+    await prisma.pendingDeletion.createMany({
+      data: traceIds.map((traceId) => ({
+        projectId,
+        object: "trace",
+        objectId: traceId,
+        isDeleted: false,
+      })),
+      skipDuplicates: true, // Avoid conflicts if trace is already pending deletion
+    });
+
+    // Get the trace delete queue
+    const traceDeleteQueue = TraceDeleteQueue.getInstance();
+    if (!traceDeleteQueue) {
+      throw new Error("TraceDeleteQueue not available");
+    }
+
+    // Send deletion event with delay
+    await traceDeleteQueue.add(
+      QueueJobs.TraceDelete,
+      {
+        timestamp: new Date(),
+        id: randomUUID(),
+        name: QueueJobs.TraceDelete,
+        payload: {
+          projectId,
+          traceIds,
+        },
+      },
+      {
+        delay: delayMs,
+      },
+    );
+  } catch (error) {
+    logger.error(`Failed to process trace deletion for project ${projectId}`, {
+      projectId,
+      traceIds,
+      error,
+    });
+    throw error;
+  }
+}

packages/shared/src/server/utils/rendering.ts

@@ -0,0 +1,58 @@
+import { JsonNested } from "../../utils/zod";
+import { parseJsonPrioritised } from "../../utils/json";
+import { env } from "../../env";
+
+/**
+ * Rendering properties used to control how data is processed and returned
+ * in tRPC routes and repository functions.
+ */
+export interface RenderingProps {
+  /**
+   * Whether to truncate input/output fields to a specific character limit
+   */
+  truncated: boolean;
+
+  /**
+   * Whether to skip JSON parsing of input/output fields and return them as raw strings.
+   * This is useful when the client will handle JSON parsing to avoid double parsing.
+   */
+  shouldJsonParse: boolean;
+}
+
+/**
+ * Default rendering properties
+ */
+export const DEFAULT_RENDERING_PROPS: RenderingProps = {
+  truncated: false,
+  shouldJsonParse: true,
+};
+
+/**
+ * Transform input/output fields based on rendering properties.
+ */
+export const applyInputOutputRendering = (
+  io: string | null | undefined,
+  renderingProps: RenderingProps,
+): JsonNested | string | null => {
+  if (!io) return null;
+  let result: JsonNested | string = io;
+
+  if (
+    renderingProps.truncated &&
+    io.length > env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT
+  ) {
+    result =
+      io.slice(0, env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT) + "\n...[truncated]";
+  }
+
+  if (
+    renderingProps.truncated &&
+    io.length === env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT
+  ) {
+    result = io + "\n...[truncated]";
+  }
+
+  return renderingProps.shouldJsonParse
+    ? (parseJsonPrioritised(result) ?? null)
+    : result;
+};

packages/shared/src/tableDefinitions/index.ts

@@ -5,3 +5,4 @@ export * from "./mapTracesTable";
 export * from "./mapDashboards";
 export * from "./mapScoresTable";
 export * from "./promptsTable";
+export * from "./mapDatasetRunItemsTable";

packages/shared/src/tableDefinitions/mapDatasetRunItemsTable.ts

@@ -0,0 +1,28 @@
+import { UiColumnMappings } from "./types";
+
+export const datasetRunItemsTableUiColumnDefinitions: UiColumnMappings = [
+  {
+    uiTableName: "Dataset Run ID",
+    uiTableId: "datasetRunId",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."dataset_run_id"',
+  },
+  {
+    uiTableName: "Created At",
+    uiTableId: "createdAt",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."created_at"',
+  },
+  {
+    uiTableName: "Event Timestamp",
+    uiTableId: "eventTs",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."event_ts"',
+  },
+  {
+    uiTableName: "Dataset Item ID",
+    uiTableId: "datasetItemId",
+    clickhouseTableName: "dataset_run_items",
+    clickhouseSelect: 'dri."dataset_item_id"',
+  },
+];

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -138,4 +138,16 @@ export const sessionCols: UiColumnMappings = [
     clickhouseTableName: "traces",
     clickhouseSelect: "session_id",
   },
+  {
+    uiTableName: "Scores (numeric)",
+    uiTableId: "scores",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "scores_avg",
+  },
+  {
+    uiTableName: "Scores (categorical)",
+    uiTableId: "scores",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "score_categories",
+  },
 ];

packages/shared/src/tableDefinitions/sessionsView.ts

@@ -1,6 +1,7 @@
 import {
   type ColumnDefinition,
   type SingleValueOption,
+  type MultiValueOption,
 } from "../tableDefinitions/types";
 import { formatColumnOptions } from "./typeHelpers";
 
@@ -87,11 +88,27 @@ export const sessionsViewCols: ColumnDefinition[] = [
     internal: 't."tags"',
     options: [], // to be filled in at runtime
   },
+  {
+    name: "Scores (numeric)",
+    id: "scores_avg",
+    type: "numberObject",
+    internal: "scores",
+  },
+  {
+    name: "Scores (categorical)",
+    id: "score_categories",
+    type: "categoryOptions",
+    internal: "score_categories",
+    options: [], // to be added at runtime
+    nullable: true,
+  },
 ];
 
 export type SessionOptions = {
   userIds: Array<SingleValueOption>;
   tags: Array<SingleValueOption>;
+  scores_avg?: Array<string>;
+  score_categories?: Array<MultiValueOption>;
 };
 
 export function sessionsTableColsWithOptions(
@@ -104,6 +121,12 @@ export function sessionsTableColsWithOptions(
     if (col.id === "tags") {
       return formatColumnOptions(col, options?.tags ?? []);
     }
+    if (col.id === "scores_avg") {
+      return formatColumnOptions(col, options?.scores_avg ?? []);
+    }
+    if (col.id === "score_categories") {
+      return formatColumnOptions(col, options?.score_categories ?? []);
+    }
     return col;
   });
 }

packages/shared/src/tableDefinitions/types.ts

@@ -77,6 +77,7 @@ export const tableNames = [
   "job_configurations",
   "job_executions",
   "dataset_items",
+  "annotation_queue_assignments",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/types.ts

@@ -7,7 +7,7 @@ export type FilterCondition = z.infer<typeof singleFilter>;
 export type FilterState = FilterCondition[];
 
 // to be used in the client during editing
-type MakeOptional<T> = {
+export type MakeOptional<T> = {
   [K in keyof T]?: T[K];
 };
 // if key is value, add string as value

packages/shared/src/utils/zod.ts

@@ -41,11 +41,11 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().nonnegative().default(1),
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().nonnegative().lte(100).default(50),
   ),
 };
 
@@ -80,7 +80,7 @@ export const paginationMetaResponseZod = z.object({
   totalPages: z.number().int().nonnegative(),
 });
 
-const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
+export const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
 export const noUrlCheck = (value: string) => !urlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);

web/Dockerfile

@@ -5,7 +5,7 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
-RUN npm install turbo@^2.5.4 --global
+RUN npm install turbo@^2.5.5 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/next.config.mjs

@@ -15,7 +15,7 @@ const cspHeader = `
   default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io;
   script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com https://chat.cdn-plain.com;
   style-src 'self' 'unsafe-inline' https://uptime.betterstack.com https://fonts.googleapis.com;
-  img-src 'self' https: blob: data: http://localhost:*;
+  img-src 'self' https: blob: data: http://localhost:* https://prod-uk-services-workspac-workspacefilespublicbuck-vs4gjqpqjkh6.s3.amazonaws.com https://prod-uk-services-attachm-attachmentsbucket28b3ccf-uwfssb4vt2us.s3.eu-west-2.amazonaws.com https://i0.wp.com;
   font-src 'self';
   frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com;
   worker-src 'self' blob:;
@@ -23,7 +23,7 @@ const cspHeader = `
   base-uri 'self';
   form-action 'self';
   frame-ancestors 'none';
-  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.ingest.us.sentry.io https://*.sentry.io https://uptime.betterstack.com https://chat.uk.plain.com https://*.s3.amazonaws.com;
+  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.ingest.us.sentry.io https://*.sentry.io https://uptime.betterstack.com https://chat.uk.plain.com https://*.s3.amazonaws.com https://prod-uk-services-attachm-attachmentsuploadbucket2-1l2e4906o2asm.s3.eu-west-2.amazonaws.com;
   media-src 'self' https: http://localhost:*;
   ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.85.2",
+  "version": "3.95.2",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -9,6 +9,7 @@
   "scripts": {
     "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
+    "dev:https": "dotenv -e ../.env -- next dev --experimental-https --experimental-https-key ./localhost+1-key.pem --experimental-https-cert ./localhost+1.pem",
     "lint": "dotenv -e ../.env -- next lint --max-warnings 0",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
     "clean": "rm -rf node_modules",
@@ -83,6 +84,8 @@
     "@radix-ui/react-tooltip": "^1.2.7",
     "@remixicon/react": "^4.2.0",
     "@sentry/nextjs": "^8.52.0",
+    "@slack/oauth": "^2.6.0",
+    "@slack/web-api": "^7.0.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -125,6 +128,7 @@
     "posthog-js": "^1.176.0",
     "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
+    "prism-react-renderer": "^2.4.1",
     "prisma": "^6.10.1",
     "protobufjs": "^7.4.0",
     "rate-limiter-flexible": "^5.0.3",
@@ -137,7 +141,6 @@
     "react-markdown": "^9.0.1",
     "react-resizable-panels": "^2.1.1",
     "react-responsive": "^10.0.0",
-    "react-syntax-highlighter": "^15.6.1",
     "react18-json-view": "^0.2.8-canary.6",
     "recharts": "^2.15.2",
     "remark-gfm": "^4.0.0",
@@ -178,7 +181,6 @@
     "@types/react": "~18.2.79",
     "@types/react-dom": "~18.2.25",
     "@types/react-grid-layout": "^1.3.5",
-    "@types/react-syntax-highlighter": "^15.5.13",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",

web/public/generated/api/openapi.yml

@@ -409,6 +409,111 @@ paths:
               schema: {}
       security:
         - BasicAuth: []
+  /api/public/annotation-queues/{queueId}/assignments:
+    post:
+      description: Create an assignment for a user to an annotation queue
+      operationId: annotationQueues_createQueueAssignment
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CreateAnnotationQueueAssignmentResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/AnnotationQueueAssignmentRequest'
+    delete:
+      description: Delete an assignment for a user to an annotation queue
+      operationId: annotationQueues_deleteQueueAssignment
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteAnnotationQueueAssignmentResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/AnnotationQueueAssignmentRequest'
   /api/public/comments:
     post:
       description: >-
@@ -1421,6 +1526,109 @@ paths:
                         name: My Score
                         value: 0.9
                         environment: default
+  /api/public/llm-connections:
+    get:
+      description: Get all LLM connections in a project
+      operationId: llmConnections_list
+      tags:
+        - LlmConnections
+      parameters:
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedLlmConnections'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+    put:
+      description: >-
+        Create or update an LLM connection. The connection is upserted on
+        provider.
+      operationId: llmConnections_upsert
+      tags:
+        - LlmConnections
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/LlmConnection'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UpsertLlmConnectionRequest'
   /api/public/media/{mediaId}:
     get:
       description: Get a media record
@@ -3046,7 +3254,7 @@ paths:
       parameters:
         - name: filter
           in: query
-          description: Filter expression (e.g. userName eq "value")
+          description: Filter expression (e.g. userName eq 'value')
           required: false
           schema:
             type: string
@@ -4284,6 +4492,36 @@ components:
       required:
         - success
         - message
+    AnnotationQueueAssignmentRequest:
+      title: AnnotationQueueAssignmentRequest
+      type: object
+      properties:
+        userId:
+          type: string
+      required:
+        - userId
+    DeleteAnnotationQueueAssignmentResponse:
+      title: DeleteAnnotationQueueAssignmentResponse
+      type: object
+      properties:
+        success:
+          type: boolean
+      required:
+        - success
+    CreateAnnotationQueueAssignmentResponse:
+      title: CreateAnnotationQueueAssignmentResponse
+      type: object
+      properties:
+        userId:
+          type: string
+        queueId:
+          type: string
+        projectId:
+          type: string
+      required:
+        - userId
+        - queueId
+        - projectId
     CreateCommentRequest:
       title: CreateCommentRequest
       type: object
@@ -6225,6 +6463,122 @@ components:
             type: integer
         - $ref: '#/components/schemas/OpenAICompletionUsageSchema'
         - $ref: '#/components/schemas/OpenAIResponseUsageSchema'
+    LlmConnection:
+      title: LlmConnection
+      type: object
+      description: LLM API connection configuration (secrets excluded)
+      properties:
+        id:
+          type: string
+        provider:
+          type: string
+          description: >-
+            Provider name (e.g., 'openai', 'my-gateway'). Must be unique in
+            project, used for upserting.
+        adapter:
+          type: string
+          description: The adapter used to interface with the LLM
+        displaySecretKey:
+          type: string
+          description: Masked version of the secret key for display purposes
+        baseURL:
+          type: string
+          nullable: true
+          description: Custom base URL for the LLM API
+        customModels:
+          type: array
+          items:
+            type: string
+          description: List of custom model names available for this connection
+        withDefaultModels:
+          type: boolean
+          description: Whether to include default models for this adapter
+        extraHeaderKeys:
+          type: array
+          items:
+            type: string
+          description: >-
+            Keys of extra headers sent with requests (values excluded for
+            security)
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+      required:
+        - id
+        - provider
+        - adapter
+        - displaySecretKey
+        - customModels
+        - withDefaultModels
+        - extraHeaderKeys
+        - createdAt
+        - updatedAt
+    PaginatedLlmConnections:
+      title: PaginatedLlmConnections
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/LlmConnection'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    UpsertLlmConnectionRequest:
+      title: UpsertLlmConnectionRequest
+      type: object
+      description: Request to create or update an LLM connection (upsert)
+      properties:
+        provider:
+          type: string
+          description: >-
+            Provider name (e.g., 'openai', 'my-gateway'). Must be unique in
+            project, used for upserting.
+        adapter:
+          $ref: '#/components/schemas/LlmAdapter'
+          description: The adapter used to interface with the LLM
+        secretKey:
+          type: string
+          description: Secret key for the LLM API.
+        baseURL:
+          type: string
+          nullable: true
+          description: Custom base URL for the LLM API
+        customModels:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of custom model names
+        withDefaultModels:
+          type: boolean
+          nullable: true
+          description: Whether to include default models. Default is true.
+        extraHeaders:
+          type: object
+          additionalProperties:
+            type: string
+          nullable: true
+          description: Extra headers to send with requests
+      required:
+        - provider
+        - adapter
+        - secretKey
+    LlmAdapter:
+      title: LlmAdapter
+      type: string
+      enum:
+        - anthropic
+        - openai
+        - azure
+        - bedrock
+        - google-vertex-ai
+        - google-ai-studio
     GetMediaResponse:
       title: GetMediaResponse
       type: object

web/public/generated/postman/collection.json

@@ -324,6 +324,88 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create Queue Assignment",
+          "request": {
+            "description": "Create an assignment for a user to an annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/assignments",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "assignments"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                }
+              ]
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"userId\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete Queue Assignment",
+          "request": {
+            "description": "Delete an assignment for a user to an annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/assignments",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "assignments"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"userId\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
         }
       ]
     },
@@ -1129,6 +1211,82 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Llm Connections",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all LLM connections in a project",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/llm-connections?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "llm-connections"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Upsert",
+          "request": {
+            "description": "Create or update an LLM connection. The connection is upserted on provider.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/llm-connections",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "llm-connections"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"provider\": \"example\",\n    \"adapter\": \"anthropic\",\n    \"secretKey\": \"example\",\n    \"baseURL\": \"example\",\n    \"customModels\": [\n        \"example\"\n    ],\n    \"withDefaultModels\": true,\n    \"extraHeaders\": {\n        \"example\": \"example\"\n    }\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,

web/src/__tests__/async/annotation-queue-assignments-api.servertest.ts

@@ -0,0 +1,342 @@
+/** @jest-environment node */
+
+import {
+  makeZodVerifiedAPICall,
+  makeAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import { prisma } from "@langfuse/shared/src/db";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import { v4 as uuidv4 } from "uuid";
+import {
+  CreateAnnotationQueueAssignmentResponse,
+  DeleteAnnotationQueueAssignmentResponse,
+} from "@/src/features/public-api/types/annotation-queues";
+
+describe("/api/public/annotation-queues/:queueId/assignments API", () => {
+  let auth: string;
+  let projectId: string;
+  let orgId: string;
+  let queueId: string;
+  let testUserId: string;
+  let secondTestUserId: string;
+
+  beforeAll(async () => {
+    const {
+      auth: newAuth,
+      projectId: newProjectId,
+      orgId: newOrgId,
+    } = await createOrgProjectAndApiKey();
+    auth = newAuth;
+    projectId = newProjectId;
+    orgId = newOrgId;
+
+    // Create a test annotation queue
+    const queue = await prisma.annotationQueue.create({
+      data: {
+        id: uuidv4(),
+        name: "Test Queue for Assignments",
+        description: "Test queue for assignment testing",
+        projectId,
+        scoreConfigIds: [],
+      },
+    });
+    queueId = queue.id;
+
+    // Create test users with project access
+    const testUser = await prisma.user.create({
+      data: {
+        id: uuidv4(),
+        email: `testuser1-${uuidv4()}@example.com`,
+        name: "Test User 1",
+      },
+    });
+    testUserId = testUser.id;
+
+    const secondTestUser = await prisma.user.create({
+      data: {
+        id: uuidv4(),
+        email: `testuser2-${uuidv4()}@example.com`,
+        name: "Test User 2",
+      },
+    });
+    secondTestUserId = secondTestUser.id;
+
+    // Give users project access - need to create org memberships first
+    await prisma.organizationMembership.createMany({
+      data: [
+        {
+          orgId,
+          userId: testUserId,
+          role: "MEMBER",
+        },
+        {
+          orgId,
+          userId: secondTestUserId,
+          role: "MEMBER",
+        },
+      ],
+    });
+
+    // Get the org membership IDs and create project memberships
+    const orgMembership1 = await prisma.organizationMembership.findFirst({
+      where: { orgId, userId: testUserId },
+    });
+    const orgMembership2 = await prisma.organizationMembership.findFirst({
+      where: { orgId, userId: secondTestUserId },
+    });
+
+    await prisma.projectMembership.createMany({
+      data: [
+        {
+          projectId,
+          userId: testUserId,
+          role: "MEMBER",
+          orgMembershipId: orgMembership1!.id,
+        },
+        {
+          projectId,
+          userId: secondTestUserId,
+          role: "MEMBER",
+          orgMembershipId: orgMembership2!.id,
+        },
+      ],
+    });
+  });
+
+  afterAll(async () => {
+    await pruneDatabase();
+  });
+
+  describe("POST /api/public/annotation-queues/:queueId/assignments", () => {
+    it("should create annotation queue assignment successfully", async () => {
+      const response = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueAssignmentResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: testUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.userId).toBe(testUserId);
+      expect(response.body.projectId).toBe(projectId);
+      expect(response.body.queueId).toBe(queueId);
+
+      // Verify the membership was created in the database
+      const assignment = await prisma.annotationQueueAssignment.findUnique({
+        where: {
+          projectId_queueId_userId: {
+            projectId,
+            queueId,
+            userId: testUserId,
+          },
+        },
+      });
+
+      expect(assignment).toBeTruthy();
+      expect(assignment?.userId).toBe(testUserId);
+      expect(assignment?.projectId).toBe(projectId);
+      expect(assignment?.queueId).toBe(queueId);
+    });
+
+    it("should handle duplicate assignment creation gracefully", async () => {
+      // Create assignment first time
+      await makeZodVerifiedAPICall(
+        CreateAnnotationQueueAssignmentResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: secondTestUserId,
+        },
+        auth,
+      );
+
+      // Create same assignment again - should succeed (upsert behavior)
+      const response = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueAssignmentResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: secondTestUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.userId).toBe(secondTestUserId);
+
+      // Verify only one assignment exists
+      const assignments = await prisma.annotationQueueAssignment.findMany({
+        where: {
+          projectId,
+          queueId,
+          userId: secondTestUserId,
+        },
+      });
+
+      expect(assignments).toHaveLength(1);
+    });
+
+    it("should return 404 for non-existent annotation queue", async () => {
+      const nonExistentQueueId = uuidv4();
+
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/${nonExistentQueueId}/assignments`,
+        {
+          userId: testUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should return 404 for non-existent user", async () => {
+      const nonExistentUserId = uuidv4();
+
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: nonExistentUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should return 404 for user without project access", async () => {
+      // Create a user without project access
+      const userWithoutAccess = await prisma.user.create({
+        data: {
+          id: uuidv4(),
+          email: `noaccess-${uuidv4()}@example.com`,
+          name: "No Access User",
+        },
+      });
+
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: userWithoutAccess.id,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should validate request body", async () => {
+      // Missing userId
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {},
+        auth,
+      );
+
+      expect(response.status).toBe(400);
+    });
+
+    it("should require valid queueId format", async () => {
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/invalid-queue-id/assignments`,
+        {
+          userId: testUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("DELETE /api/public/annotation-queues/:queueId/assignments", () => {
+    beforeEach(async () => {
+      // Ensure assignment exists for delete tests
+      await prisma.annotationQueueAssignment.upsert({
+        where: {
+          projectId_queueId_userId: {
+            projectId,
+            queueId,
+            userId: testUserId,
+          },
+        },
+        create: {
+          projectId,
+          queueId,
+          userId: testUserId,
+        },
+        update: {},
+      });
+    });
+
+    it("should delete annotation queue membership successfully", async () => {
+      const response = await makeZodVerifiedAPICall(
+        DeleteAnnotationQueueAssignmentResponse,
+        "DELETE",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: testUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.success).toBe(true);
+
+      // Verify the membership was deleted from the database
+      const assignment = await prisma.annotationQueueAssignment.findUnique({
+        where: {
+          projectId_queueId_userId: {
+            projectId,
+            queueId,
+            userId: testUserId,
+          },
+        },
+      });
+
+      expect(assignment).toBeNull();
+    });
+
+    it("should handle deletion of non-existent assignment gracefully", async () => {
+      // Delete a assignment that doesn't exist
+      const nonExistentUserId = uuidv4();
+
+      const response = await makeZodVerifiedAPICall(
+        DeleteAnnotationQueueAssignmentResponse,
+        "DELETE",
+        `/api/public/annotation-queues/${queueId}/assignments`,
+        {
+          userId: nonExistentUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.success).toBe(true);
+    });
+
+    it("should return 404 for non-existent annotation queue", async () => {
+      const nonExistentQueueId = uuidv4();
+
+      const response = await makeAPICall(
+        "DELETE",
+        `/api/public/annotation-queues/${nonExistentQueueId}/assignments`,
+        {
+          userId: testUserId,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+});

web/src/__tests__/async/automations-trpc.servertest.ts

@@ -12,6 +12,7 @@ import {
   JobConfigState,
   type SafeWebhookActionConfig,
   type WebhookActionConfigWithSecrets,
+  isWebhookAction,
 } from "@langfuse/shared";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { generateWebhookSecret } from "@langfuse/shared/encryption";
@@ -204,11 +205,10 @@ describe("automations trpc", () => {
               "x-api-key": encrypt("secret-api-key-123"),
               authorization: encrypt("Bearer secret-token-456"),
             },
-            secretHeaderKeys: ["x-api-key", "authorization"],
             displayHeaders: {
-              "content-type": "application/json",
-              "x-api-key": "secr***123",
-              authorization: "Bear***456",
+              "content-type": { secret: false, value: "application/json" },
+              "x-api-key": { secret: true, value: "secr***123" },
+              authorization: { secret: true, value: "Bear***456" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -232,13 +232,14 @@ describe("automations trpc", () => {
       });
 
       expect(response).toHaveLength(1);
-      const automationConfig = response[0].action.config;
+      const automationConfig = response[0].action
+        .config as SafeWebhookActionConfig;
 
       // Should have display values, not encrypted values
       expect(automationConfig.displayHeaders).toEqual({
-        "content-type": "application/json",
-        "x-api-key": "secr***123",
-        authorization: "Bear***456",
+        "content-type": { secret: false, value: "application/json" },
+        "x-api-key": { secret: true, value: "secr***123" },
+        authorization: { secret: true, value: "Bear***456" },
       });
 
       // Should NOT have the raw headers with encrypted values
@@ -321,7 +322,9 @@ describe("automations trpc", () => {
             type: "WEBHOOK",
             url: "https://example.com/webhook",
             headers: { "Content-Type": "application/json" },
-            displayHeaders: { "Content-Type": "application/json" },
+            displayHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
             displaySecretKey,
@@ -353,20 +356,22 @@ describe("automations trpc", () => {
         }),
       });
 
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
+
       // check that the action does not have a secret key in the config
-      expect(response.action.config).not.toHaveProperty("secretKey");
-      expect(response.action.config).toHaveProperty("displaySecretKey");
-      expect(response.action.config.url).toBe("https://example.com/webhook");
-      expect(response.action.config).not.toHaveProperty("headers");
-      expect(response.action.config.displayHeaders).toEqual({
-        "Content-Type": "application/json",
+      expect(actionConfig).not.toHaveProperty("secretKey");
+      expect(actionConfig).toHaveProperty("displaySecretKey");
+      expect(actionConfig.url).toBe("https://example.com/webhook");
+      expect(actionConfig).not.toHaveProperty("headers");
+      expect(actionConfig.displayHeaders).toEqual({
+        "Content-Type": { secret: false, value: "application/json" },
       });
-      expect(response.action.config.apiVersion).toEqual({ prompt: "v1" });
-      expect(response.action.config.type).toBe("WEBHOOK");
-      expect(response.action.config.displaySecretKey).toBe(displaySecretKey);
-      expect(response.action.config).not.toHaveProperty("headers");
-      expect(response.action.config).not.toHaveProperty("decryptedHeaders");
-      expect(response.action.config).not.toHaveProperty("requestHeaders");
+      expect(actionConfig.apiVersion).toEqual({ prompt: "v1" });
+      expect(actionConfig.type).toBe("WEBHOOK");
+      expect(actionConfig.displaySecretKey).toBe(displaySecretKey);
+      expect(actionConfig).not.toHaveProperty("headers");
+      expect(actionConfig).not.toHaveProperty("decryptedHeaders");
+      expect(actionConfig).not.toHaveProperty("requestHeaders");
     });
 
     it("should not expose secret headers in single automation response", async () => {
@@ -398,11 +403,10 @@ describe("automations trpc", () => {
               "x-custom": "public-value",
               "x-secret": encrypt("secret-value-789"),
             },
-            secretHeaderKeys: ["x-secret"],
             displayHeaders: {
-              "content-type": "application/json",
-              "x-custom": "public-value",
-              "x-secret": "secr***789",
+              "content-type": { secret: false, value: "application/json" },
+              "x-custom": { secret: false, value: "public-value" },
+              "x-secret": { secret: true, value: "secr***789" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -425,13 +429,13 @@ describe("automations trpc", () => {
         automationId: automation.id,
       });
 
-      const config = response.action.config;
+      const config = response.action.config as SafeWebhookActionConfig;
 
       // Should have display values
       expect(config.displayHeaders).toEqual({
-        "content-type": "application/json",
-        "x-custom": "public-value",
-        "x-secret": "secr***789",
+        "content-type": { secret: false, value: "application/json" },
+        "x-custom": { secret: false, value: "public-value" },
+        "x-secret": { secret: true, value: "secr***789" },
       });
 
       // Should NOT have raw encrypted headers
@@ -488,7 +492,7 @@ describe("automations trpc", () => {
         automationId: automation.id,
       });
 
-      const config = response.action.config;
+      const config = response.action.config as SafeWebhookActionConfig;
 
       // Should have display values with all legacy header values returned
       // Legacy headers are converted to the new format with secret: false
@@ -629,6 +633,7 @@ describe("automations trpc", () => {
         type: "WEBHOOK",
         url: "https://example.com/mixed-headers",
       });
+      expect(isWebhookAction(createdAction as any)).toBe(true);
 
       // Headers should be encrypted for secret ones, plain for others
       const config = createdAction?.config as WebhookActionConfigWithSecrets;
@@ -795,6 +800,85 @@ describe("automations trpc", () => {
         }),
       ).rejects.toThrow("Name is required");
     });
+
+    it("should create a new Slack automation", async () => {
+      const { project, caller } = await prepare();
+
+      // Create Slack integration first
+      await prisma.slackIntegration.create({
+        data: {
+          projectId: project.id,
+          teamId: "T123456",
+          teamName: "Test Team",
+          botToken: encrypt("xoxb-test-token"),
+          botUserId: "U123456",
+        },
+      });
+
+      const response = await caller.automations.createAutomation({
+        projectId: project.id,
+        name: "New Slack Automation",
+        eventSource: "prompt",
+        eventAction: ["created"],
+        filter: [],
+        status: JobConfigState.ACTIVE,
+        actionType: "SLACK",
+        actionConfig: {
+          type: "SLACK",
+          channelId: "C123456",
+          channelName: "general",
+          messageTemplate: JSON.stringify([
+            {
+              type: "section",
+              text: { type: "mrkdwn", text: "Custom template" },
+            },
+          ]),
+        },
+      });
+
+      expect(response.trigger).toMatchObject({
+        projectId: project.id,
+        eventSource: "prompt",
+        eventActions: ["created"],
+        status: JobConfigState.ACTIVE,
+      });
+
+      expect(response.action).toMatchObject({
+        projectId: project.id,
+        type: "SLACK",
+        config: expect.objectContaining({
+          type: "SLACK",
+          channelId: "C123456",
+          channelName: "general",
+        }),
+      });
+
+      // Ensure no bot token is exposed in response
+      expect(JSON.stringify(response)).not.toContain("xoxb-");
+    });
+
+    it("should fail to create Slack automation without integration", async () => {
+      const { project, caller } = await prepare();
+
+      await expect(
+        caller.automations.createAutomation({
+          projectId: project.id,
+          name: "Invalid Slack Automation",
+          eventSource: "prompt",
+          eventAction: ["created"],
+          filter: [],
+          status: JobConfigState.ACTIVE,
+          actionType: "SLACK",
+          actionConfig: {
+            type: "SLACK",
+            channelId: "C123456",
+            channelName: "general",
+          },
+        }),
+      ).rejects.toThrow(
+        "Slack integration not found. Please connect your Slack workspace first.",
+      );
+    });
   });
 
   describe("automations.updateAutomation", () => {
@@ -922,9 +1006,9 @@ describe("automations trpc", () => {
               "x-case-key": { secret: false, value: "some-value" },
             },
             displayHeaders: {
-              "content-type": "application/json",
-              "x-old-header": "old-value",
-              "x-case-key": "some-value",
+              "content-type": { secret: false, value: "application/json" },
+              "x-old-header": { secret: false, value: "old-value" },
+              "x-case-key": { secret: false, value: "some-value" },
             },
             apiVersion: { prompt: "v1" },
             secretKey: encrypt(secretKey),
@@ -963,19 +1047,20 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the API response contains safe display values
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" }, // header preserved
         "x-public": { secret: false, value: "new-public-value" }, // new public header
         "x-secret-key": { secret: true, value: "new-...-123" }, // new secret header
         "x-Case-KEY": { secret: false, value: "new-value" }, // matched existing key, but new value
       });
 
-      expect(response.action.config.displayHeaders).not.toHaveProperty(
+      expect(actionConfig.displayHeaders).not.toHaveProperty(
         "x-old-header", // header deleted
       );
-      expect(response.action.config.displayHeaders).not.toHaveProperty(
+      expect(actionConfig.displayHeaders).not.toHaveProperty(
         "x-case-key", // new case replaced the old header name
       );
 
@@ -1083,9 +1168,10 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the API response contains safe display values reflecting the switch
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "x-currently-public": { secret: true, value: "now-...alue" },
         "x-currently-secret": { secret: false, value: "now-public-value" },
       });
@@ -1097,7 +1183,7 @@ describe("automations trpc", () => {
         where: { id: action.id },
       });
 
-      const config = updatedAction?.config as any;
+      const config = updatedAction?.config as WebhookActionConfigWithSecrets;
 
       // x-currently-public should now be encrypted (was plain, now secret)
       expect(config.requestHeaders["x-currently-public"].value).not.toBe(
@@ -1283,14 +1369,13 @@ describe("automations trpc", () => {
           apiVersion: { prompt: "v1" },
         },
       });
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
 
       // Verify the URL was updated
-      expect(response.action.config.url).toBe(
-        "https://example.com/new-webhook-url",
-      );
+      expect(actionConfig.url).toBe("https://example.com/new-webhook-url");
 
       // Verify secret headers were preserved
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" },
         "x-api-key": { secret: true, value: "secr...-123" },
         authorization: { secret: true, value: "Bear...-456" },
@@ -1388,8 +1473,10 @@ describe("automations trpc", () => {
         },
       });
 
+      const actionConfig = response.action.config as SafeWebhookActionConfig;
+
       // Verify the API response
-      expect(response.action.config.displayHeaders).toMatchObject({
+      expect(actionConfig.displayHeaders).toMatchObject({
         "content-type": { secret: false, value: "application/json" },
         "x-api-key": { secret: true, value: "new-...-key" },
       });
@@ -1780,6 +1867,199 @@ describe("automations trpc", () => {
 
       expect(response.count).toBe(3);
     });
+
+    it("should return 0 consecutive failures when lastFailingExecutionId is set", async () => {
+      const { project, caller } = await prepare();
+
+      // Create automation
+      const trigger = await prisma.trigger.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          eventSource: "prompt",
+          eventActions: ["created"],
+          filter: [],
+          status: JobConfigState.INACTIVE, // Disabled due to failures
+        },
+      });
+
+      const { secretKey, displaySecretKey } = generateWebhookSecret();
+      const action = await prisma.action.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          type: "WEBHOOK",
+          config: {
+            type: "WEBHOOK",
+            url: "https://example.com/webhook",
+            requestHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
+            apiVersion: { prompt: "v1" },
+            secretKey: encrypt(secretKey),
+            displaySecretKey,
+            lastFailingExecutionId: "some-failing-execution-id", // This simulates a webhook that was disabled
+          },
+        },
+      });
+
+      const automation = await prisma.automation.create({
+        data: {
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          name: "Test Automation",
+        },
+      });
+
+      // Create failed executions that occurred BEFORE the lastFailingExecutionId
+      await prisma.automationExecution.create({
+        data: {
+          id: "some-failing-execution-id",
+          automationId: automation.id,
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          status: ActionExecutionStatus.ERROR,
+          sourceId: v4(),
+          input: { iteration: 0 },
+          error: "Old failure",
+          createdAt: new Date(Date.now() - 60000), // 1 minute ago
+        },
+      });
+
+      // Create more failed executions that occurred BEFORE the lastFailingExecutionId
+      for (let i = 1; i < 5; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `Old failure ${i}`,
+            createdAt: new Date(Date.now() - (60000 + i * 1000)), // Before the lastFailingExecutionId
+          },
+        });
+      }
+
+      const response = await caller.automations.getCountOfConsecutiveFailures({
+        projectId: project.id,
+        automationId: automation.id,
+      });
+
+      // Should return 0 because all failures occurred before the lastFailingExecutionId
+      expect(response.count).toBe(0);
+    });
+
+    it("should count failures after lastFailingExecutionId correctly", async () => {
+      const { project, caller } = await prepare();
+
+      // Create automation
+      const trigger = await prisma.trigger.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          eventSource: "prompt",
+          eventActions: ["created"],
+          filter: [],
+          status: JobConfigState.ACTIVE,
+        },
+      });
+
+      const { secretKey, displaySecretKey } = generateWebhookSecret();
+      const lastFailingExecutionId = v4();
+      const action = await prisma.action.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          type: "WEBHOOK",
+          config: {
+            type: "WEBHOOK",
+            url: "https://example.com/webhook",
+            requestHeaders: {
+              "Content-Type": { secret: false, value: "application/json" },
+            },
+            apiVersion: { prompt: "v1" },
+            secretKey: encrypt(secretKey),
+            displaySecretKey,
+            lastFailingExecutionId,
+          },
+        },
+      });
+
+      const automation = await prisma.automation.create({
+        data: {
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          name: "Test Automation",
+        },
+      });
+
+      // Create the lastFailingExecution
+      await prisma.automationExecution.create({
+        data: {
+          id: lastFailingExecutionId,
+          automationId: automation.id,
+          projectId: project.id,
+          triggerId: trigger.id,
+          actionId: action.id,
+          status: ActionExecutionStatus.ERROR,
+          sourceId: v4(),
+          input: { iteration: 0 },
+          error: "Last failing execution",
+          createdAt: new Date(Date.now() - 60000), // 1 minute ago
+        },
+      });
+
+      // Create old failures that should be ignored
+      for (let i = 0; i < 3; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `Old failure ${i}`,
+            createdAt: new Date(Date.now() - (120000 + i * 1000)), // Before the lastFailingExecutionId
+          },
+        });
+      }
+
+      // Create new failures AFTER the lastFailingExecutionId
+      for (let i = 0; i < 2; i++) {
+        await prisma.automationExecution.create({
+          data: {
+            id: v4(),
+            automationId: automation.id,
+            projectId: project.id,
+            triggerId: trigger.id,
+            actionId: action.id,
+            status: ActionExecutionStatus.ERROR,
+            sourceId: v4(),
+            input: { iteration: i },
+            error: `New failure ${i}`,
+            createdAt: new Date(Date.now() - (30000 - i * 1000)), // After the lastFailingExecutionId
+          },
+        });
+      }
+
+      const response = await caller.automations.getCountOfConsecutiveFailures({
+        projectId: project.id,
+        automationId: automation.id,
+      });
+
+      // Should return 2 because only the 2 new failures after lastFailingExecutionId should be counted
+      expect(response.count).toBe(2);
+    });
   });
 
   describe("automations.regenerateWebhookSecret", () => {

web/src/__tests__/async/generateUserProjectRolesQuery.servertest.ts

@@ -0,0 +1,443 @@
+/** @jest-environment node */
+
+import { prisma } from "@langfuse/shared/src/db";
+import { Prisma, type Role } from "@langfuse/shared";
+import { v4 } from "uuid";
+import { getUserProjectRoles } from "@/src/features/rbac/utils/userProjectRole";
+
+export const createOrgAndProject = async () => {
+  const org = await prisma.organization.create({
+    data: {
+      id: v4(),
+      name: v4(),
+    },
+  });
+
+  const project = await prisma.project.create({
+    data: {
+      id: v4(),
+      name: v4(),
+      orgId: org.id,
+    },
+  });
+
+  return { org, project };
+};
+
+describe("find user project roles", () => {
+  it("should find users with org role", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    const user = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    await prisma.organizationMembership.create({
+      data: {
+        userId: user.id,
+        orgId: org.id,
+        role: "MEMBER",
+      },
+    });
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual([
+      expect.objectContaining({
+        id: user.id,
+        name: user.name,
+        email: user.email,
+      }),
+    ]);
+  });
+
+  it("should exclude users with NONE role", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    const user = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    await prisma.organizationMembership.create({
+      data: {
+        userId: user.id,
+        orgId: org.id,
+        role: "NONE",
+      },
+    });
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual([]);
+  });
+
+  it("should find users with org and project role", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    const user = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    await prisma.organizationMembership.create({
+      data: {
+        userId: user.id,
+        orgId: org.id,
+        role: "MEMBER",
+      },
+    });
+    const user2 = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    const orgMembershipOfUser2 = await prisma.organizationMembership.create({
+      data: {
+        userId: user2.id,
+        orgId: org.id,
+        role: "MEMBER",
+      },
+    });
+
+    // overwrite org role to ADMIN
+    await prisma.projectMembership.create({
+      data: {
+        userId: user2.id,
+        projectId: project.id,
+        role: "ADMIN",
+        orgMembershipId: orgMembershipOfUser2.id,
+      },
+    });
+
+    const user3 = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    const orgMembershipOfUser3 = await prisma.organizationMembership.create({
+      data: {
+        userId: user3.id,
+        orgId: org.id,
+        role: "ADMIN",
+      },
+    });
+
+    // Downgrade org role to VIEWER
+    await prisma.projectMembership.create({
+      data: {
+        userId: user3.id,
+        projectId: project.id,
+        role: "VIEWER",
+        orgMembershipId: orgMembershipOfUser3.id,
+      },
+    });
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: user.id,
+          name: user.name,
+          email: user.email,
+          role: "MEMBER",
+        }),
+        expect.objectContaining({
+          id: user2.id,
+          name: user2.name,
+          email: user2.email,
+          role: "ADMIN",
+        }),
+        expect.objectContaining({
+          id: user3.id,
+          name: user3.name,
+          email: user3.email,
+          role: "VIEWER",
+        }),
+      ]),
+    );
+  });
+
+  it("should not select users from different projects", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    // valid user
+    const user = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    await prisma.organizationMembership.create({
+      data: {
+        userId: user.id,
+        orgId: org.id,
+        role: "MEMBER",
+      },
+    });
+
+    const { org: org2, project: project2 } = await createOrgAndProject();
+
+    const user2 = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    await prisma.organizationMembership.create({
+      data: {
+        userId: user2.id,
+        orgId: org2.id,
+        role: "MEMBER",
+      },
+    });
+
+    const user3 = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    const orgMembershipOfUser3 = await prisma.organizationMembership.create({
+      data: {
+        userId: user3.id,
+        orgId: org2.id,
+        role: "MEMBER",
+      },
+    });
+
+    await prisma.projectMembership.create({
+      data: {
+        userId: user3.id,
+        projectId: project2.id,
+        role: "MEMBER",
+        orgMembershipId: orgMembershipOfUser3.id,
+      },
+    });
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual([
+      expect.objectContaining({
+        id: user.id,
+        name: user.name,
+        email: user.email,
+      }),
+    ]);
+  });
+
+  it("should return empty array for empty organization", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual([]);
+  });
+
+  it("should exclude users with project role NONE even if they have org membership", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    const user = await prisma.user.create({
+      data: {
+        id: v4(),
+        email: v4(),
+        name: v4(),
+      },
+    });
+
+    const orgMembership = await prisma.organizationMembership.create({
+      data: {
+        userId: user.id,
+        orgId: org.id,
+        role: "ADMIN",
+      },
+    });
+
+    // Project role NONE should exclude user despite ADMIN org role
+    await prisma.projectMembership.create({
+      data: {
+        userId: user.id,
+        projectId: project.id,
+        role: "NONE",
+        orgMembershipId: orgMembership.id,
+      },
+    });
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    expect(users).toEqual([]);
+  });
+
+  it("should test all role combinations inheritance vs override", async () => {
+    const { org, project } = await createOrgAndProject();
+
+    // Test inheritance: users with only org roles
+    const inheritanceTests = [
+      { orgRole: "OWNER", expectedRole: "OWNER" },
+      { orgRole: "ADMIN", expectedRole: "ADMIN" },
+      { orgRole: "MEMBER", expectedRole: "MEMBER" },
+      { orgRole: "VIEWER", expectedRole: "VIEWER" },
+    ];
+
+    const inheritanceUsers = [];
+    for (const test of inheritanceTests) {
+      const id = v4();
+      const user = await prisma.user.create({
+        data: {
+          id,
+          email: `${id}-${test.orgRole}@test.com`,
+          name: `${id}-${test.orgRole} User`,
+        },
+      });
+
+      await prisma.organizationMembership.create({
+        data: {
+          userId: user.id,
+          orgId: org.id,
+          role: test.orgRole as Role,
+        },
+      });
+
+      inheritanceUsers.push({
+        id,
+        role: test.expectedRole,
+      });
+    }
+
+    // Test overrides: users with project roles that override org roles
+    const overrideTests = [
+      { orgRole: "OWNER", projectRole: "VIEWER", expectedRole: "VIEWER" },
+      { orgRole: "MEMBER", projectRole: "ADMIN", expectedRole: "ADMIN" },
+      { orgRole: "VIEWER", projectRole: "MEMBER", expectedRole: "MEMBER" },
+      { orgRole: "ADMIN", projectRole: "VIEWER", expectedRole: "VIEWER" },
+    ];
+
+    const overrideUsers = [];
+    for (const test of overrideTests) {
+      const id = v4();
+      const user = await prisma.user.create({
+        data: {
+          id,
+          email: `${id}-${test.orgRole}-${test.projectRole}@test.com`,
+          name: `${id}-${test.orgRole}-${test.projectRole} User`,
+        },
+      });
+
+      const orgMembership = await prisma.organizationMembership.create({
+        data: {
+          userId: user.id,
+          orgId: org.id,
+          role: test.orgRole as Role,
+        },
+      });
+
+      await prisma.projectMembership.create({
+        data: {
+          userId: user.id,
+          projectId: project.id,
+          role: test.projectRole as Role,
+          orgMembershipId: orgMembership.id,
+        },
+      });
+
+      overrideUsers.push({
+        id: user.id,
+        role: test.expectedRole,
+      });
+    }
+
+    const users = await getUserProjectRoles({
+      projectId: project.id,
+      orgId: org.id,
+      filterCondition: [],
+      searchFilter: Prisma.empty,
+      orderBy: Prisma.empty,
+    });
+
+    // Verify all inheritance cases
+    for (const expectedUser of inheritanceUsers) {
+      expect(users).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: expectedUser.id,
+            role: expectedUser.role,
+          }),
+        ]),
+      );
+    }
+
+    // Verify all override cases
+    for (const expectedUser of overrideUsers) {
+      expect(users).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: expectedUser.id,
+            role: expectedUser.role,
+          }),
+        ]),
+      );
+    }
+
+    // Verify total count
+    expect(users).toHaveLength(inheritanceUsers.length + overrideUsers.length);
+  });
+});

web/src/__tests__/async/ingestion-api.servertest.ts

@@ -316,36 +316,40 @@ describe("/api/public/ingestion API Endpoint", () => {
     "<",
     "#",
     "|",
-  ])("should test special S3 characters in IDs (%s)", async (char: string) => {
-    const traceId = randomUUID();
+  ])(
+    "should test special S3 characters in IDs (%s)",
+    async (char: string) => {
+      const traceId = randomUUID();
 
-    const response = await makeAPICall("POST", "/api/public/ingestion", {
-      batch: [
-        {
-          id: randomUUID(),
-          type: "trace-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: `${traceId}-${char}-test`,
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [
+          {
+            id: randomUUID(),
+            type: "trace-create",
             timestamp: new Date().toISOString(),
+            body: {
+              id: `${traceId}-${char}-test`,
+              timestamp: new Date().toISOString(),
+            },
           },
-        },
-      ],
-    });
-
-    expect(response.status).toBe(207);
-
-    await waitForExpect(async () => {
-      const trace = await getTraceById({
-        traceId: `${traceId}-${char}-test`,
-        projectId,
+        ],
       });
-      expect(trace).toBeDefined();
-      expect(trace!.id).toBe(`${traceId}-${char}-test`);
-      expect(trace!.projectId).toBe(projectId);
-      expect(trace!.environment).toEqual("default");
-    });
-  });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const trace = await getTraceById({
+          traceId: `${traceId}-${char}-test`,
+          projectId,
+        });
+        expect(trace).toBeDefined();
+        expect(trace!.id).toBe(`${traceId}-${char}-test`);
+        expect(trace!.projectId).toBe(projectId);
+        expect(trace!.environment).toEqual("default");
+      });
+    },
+    10000,
+  );
 
   it("should fail for \\r in id", async () => {
     const traceId = v4();
@@ -543,6 +547,7 @@ describe("/api/public/ingestion API Endpoint", () => {
         );
       });
     },
+    10000,
   );
 
   it.each([
@@ -630,6 +635,65 @@ describe("/api/public/ingestion API Endpoint", () => {
     },
   );
 
+  it("should merge metadata correctly across multiple trace updates", async () => {
+    const traceId = randomUUID();
+
+    // First update with initial metadata: {"step": 1, "status": "started"}
+    const traceUpdate1 = {
+      id: randomUUID(),
+      type: "trace-create",
+      timestamp: new Date().toISOString(),
+      body: {
+        id: traceId,
+        name: "operation",
+        timestamp: new Date().toISOString(),
+        metadata: { step: 1, status: "started" },
+      },
+    };
+
+    const response1 = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [traceUpdate1],
+    });
+    expect(response1.status).toBe(207);
+
+    // Second update with additional metadata: {"step": 2, "error": ""}
+    // This should merge with the first update
+    const traceUpdate2 = {
+      id: randomUUID(),
+      type: "trace-create",
+      timestamp: new Date(Date.now() + 1000).toISOString(), // Later timestamp
+      body: {
+        id: traceId,
+        name: "operation",
+        timestamp: new Date(Date.now() + 1000).toISOString(),
+        metadata: { step: 2, error: "" },
+      },
+    };
+
+    const response2 = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [traceUpdate2],
+    });
+    expect(response2.status).toBe(207);
+
+    await waitForExpect(async () => {
+      const trace = await getTraceById({ traceId, projectId });
+      expect(trace).toBeDefined();
+      expect(trace!.id).toBe(traceId);
+      expect(trace!.projectId).toBe(projectId);
+
+      // Expected final metadata: {"step": 2, "status": "started", "error": ""}
+      // This verifies that:
+      // - "step" is updated to the latest value (2)
+      // - "status" is preserved from the first update ("started")
+      // - "error" is added from the second update ("")
+      expect(trace!.metadata).toEqual({
+        step: 2,
+        status: "started",
+        error: "",
+      });
+    });
+  }, 20000);
+
   it("#4900: should clear score comment on update with `null`", async () => {
     const scoreId = randomUUID();
     const score1 = {
@@ -674,5 +738,5 @@ describe("/api/public/ingestion API Endpoint", () => {
       expect(score!.value).toEqual(100.5);
       expect(score!.comment).toBe(null);
     });
-  });
+  }, 10000);
 });

web/src/__tests__/async/observations-api.servertest.ts

@@ -118,7 +118,7 @@ describe("/api/public/observations API Endpoint", () => {
         eventType: "click",
         target: "submit-button",
       });
-    }, 15_000);
+    }, 20_000);
 
     it("should filter observations by level parameter", async () => {
       const traceId = randomUUID();

web/src/__tests__/async/scim-api.servertest.ts

@@ -18,16 +18,20 @@ const ScimUserSchema = z.object({
   schemas: z.array(z.string()),
   id: z.string(),
   userName: z.string(),
-  name: z.object({
-    formatted: z.string().nullable(),
-  }),
-  emails: z.array(
-    z.object({
-      primary: z.boolean(),
-      value: z.string(),
-      type: z.string(),
-    }),
-  ),
+  name: z
+    .object({
+      formatted: z.string().nullable(),
+    })
+    .optional(),
+  emails: z
+    .array(
+      z.object({
+        primary: z.boolean(),
+        value: z.string(),
+        type: z.string(),
+      }),
+    )
+    .optional(),
   meta: z.object({
     resourceType: z.string(),
     created: z.string().optional(),
@@ -533,6 +537,45 @@ describe("SCIM API", () => {
         expect(result.body.detail).toContain("userName is required");
       });
 
+      it("should create a new user with specified role", async () => {
+        const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "POST",
+          "/api/public/scim/Users",
+          {
+            userName: uniqueEmail,
+            name: {
+              formatted: "Test User With Role",
+            },
+            emails: [
+              {
+                primary: true,
+                value: uniqueEmail,
+                type: "work",
+              },
+            ],
+            active: true,
+            roles: ["ADMIN"],
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          201,
+        );
+
+        expect(response.status).toBe(201);
+        expect(response.body.userName).toBe(uniqueEmail);
+        expect(response.body.name.formatted).toBe("Test User With Role");
+
+        testUserId = response.body.id;
+
+        // Verify the user was created with the specified role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("ADMIN");
+      });
+
       it("should return 409 when user with the same userName already exists", async () => {
         const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
 
@@ -617,6 +660,147 @@ describe("SCIM API", () => {
       });
     });
 
+    describe("PUT /api/public/scim/Users/{id}", () => {
+      beforeEach(async () => {
+        // Create a test user
+        const uniqueEmail = `test.user.${randomUUID().substring(0, 8)}@example.com`;
+        const user = await prisma.user.create({
+          data: {
+            email: uniqueEmail,
+            name: "Test User",
+          },
+        });
+        await prisma.organizationMembership.create({
+          data: {
+            userId: user.id,
+            orgId: orgId,
+            role: "NONE",
+          },
+        });
+        testUserId = user.id;
+      });
+
+      it("should deactivate a user when active is false", async () => {
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was removed from the organization
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(0);
+      });
+
+      it("should reactivate a user when active is true", async () => {
+        // First deactivate the user
+        await prisma.organizationMembership.deleteMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: true,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was re-added to the organization with default role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("NONE");
+      });
+
+      it("should reactivate a user with specified role", async () => {
+        // First deactivate the user
+        await prisma.organizationMembership.deleteMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+
+        const response = await makeZodVerifiedAPICall(
+          ScimUserSchema,
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: true,
+            roles: ["MEMBER"],
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+          200,
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.body.id).toBe(testUserId);
+
+        // Verify the user was re-added to the organization with specified role
+        const orgMemberships = await prisma.organizationMembership.findMany({
+          where: { userId: testUserId, orgId: orgId },
+        });
+        expect(orgMemberships.length).toBe(1);
+        expect(orgMemberships[0].role).toBe("MEMBER");
+      });
+
+      it("should return 400 when SCIM schema is missing", async () => {
+        const result = await makeAPICall(
+          "PUT",
+          `/api/public/scim/Users/${testUserId}`,
+          {
+            id: testUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+        );
+        expect(result.status).toBe(400);
+        expect(result.body.detail).toContain("schemas");
+      });
+
+      it("should return 404 when user does not exist", async () => {
+        const nonExistentUserId = randomUUID();
+        const result = await makeAPICall(
+          "PUT",
+          `/api/public/scim/Users/${nonExistentUserId}`,
+          {
+            schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+            id: nonExistentUserId,
+            userName: "test.user@example.com",
+            active: false,
+          },
+          createBasicAuthHeader(orgApiKey, orgSecretKey),
+        );
+        expect(result.status).toBe(404);
+        expect(result.body.detail).toContain("User not found");
+      });
+    });
+
     describe("DELETE /api/public/scim/Users/{id}", () => {
       beforeEach(async () => {
         // Create a test user

web/src/__tests__/async/scores-api-v1.servertest.ts

@@ -641,10 +641,9 @@ describe("/api/public/scores API Endpoint", () => {
           totalPages: 1,
         });
         for (const val of getAllScore.body.data) {
-          expect(val).toMatchObject({
-            traceId: traceId,
-            trace: { tags: ["prod", "test"], userId: "user-name" },
-          });
+          expect(val.traceId).toBe(traceId);
+          expect(val.trace?.tags?.sort()).toEqual(["prod", "test"].sort());
+          expect(val.trace?.userId).toBe("user-name");
         }
       });