chore: release v3.97.2

* fix: fail silently when /api/latest-releases returns invalid schema

* push

.claude/settings.json

@@ -0,0 +1,14 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(find:*)",
+      "Bash(rg:*)",
+      "Bash(grep:*)",
+      "Bash(ls:*)",
+      "Bash(cat:*)",
+      "Bash(head:*)",
+      "Bash(tail:*)"
+    ],
+    "deny": []
+  }
+}

.devcontainer.json

@@ -1,6 +0,0 @@
-{
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
-}

.devcontainer/Dockerfile

@@ -0,0 +1,13 @@
+# Dev container Dockerfile
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:20-bookworm
+
+# Install golang-migrate for database migrations
+RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && \
+    chmod +x migrate && \
+    mv migrate /usr/local/bin/migrate
+
+# Install pnpm globally
+RUN npm install -g pnpm@9.5.0
+
+# Install Claude Code CLI
+RUN npm install -g @anthropic-ai/claude-code

.devcontainer/devcontainer.json

@@ -0,0 +1,8 @@
+{
+  "name": "langfuse-development",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "postCreateCommand": "cp .env.dev.example .env && pnpm i"
+}

.env.dev-azure.example

@@ -68,6 +68,7 @@ LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
 LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 LANGFUSE_USE_AZURE_BLOB=true
+LANGFUSE_AZURE_SKIP_CONTAINER_CHECK=false
 
 # Set during docker build of application
 # Used to disable environment verification at build time

.env.dev-redis-cluster.example

@@ -76,9 +76,18 @@ REDIS_AUTH="bitnami"
 REDIS_CLUSTER_ENABLED="true"
 REDIS_CLUSTER_NODES="127.0.0.1:6370,127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373,127.0.0.1:6374,127.0.0.1:6375"
 LANGFUSE_INGESTION_QUEUE_SHARD_COUNT=8
+LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT=4
 
 # openssl rand -hex 32 used only here
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+# Use the following settings to enforce running the new AMTs during the tests
+LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT="true"
+LANGFUSE_EXPERIMENT_SAMPLING_RATE=1

.env.dev.example

@@ -83,3 +83,8 @@ NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 # For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10
+
+# Slack credentials for development
+SLACK_CLIENT_ID=your_slack_client_id
+SLACK_CLIENT_SECRET=your_slack_client_secret
+SLACK_STATE_SECRET=your_slack_state_secret

.env.prod.example

@@ -172,6 +172,7 @@ OTEL_SERVICE_NAME="langfuse"
 # REDIS_HOST=
 # REDIS_PORT=
 # REDIS_AUTH=
+# REDIS_USERNAME=default
 # REDIS_CONNECTION_STRING=
 # REDIS_ENABLE_AUTO_PIPELINING=
 

.github/dependabot.yml

@@ -36,6 +36,7 @@ updates:
         patterns:
           - "express"
           - "@types/express"
+          - "@types/express-serve-static-core"
       observability:
         patterns:
           - "dd-trace"

.github/workflows/pipeline.yml

@@ -114,7 +114,9 @@ jobs:
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Build and run both images from compose
         run: |
-          docker compose -f docker-compose.build.yml up -d
+          docker compose --progress plain --verbose -f docker-compose.build.yml build --print > /tmp/bake.json
+          docker buildx bake -f /tmp/bake.json
+          docker compose --progress plain -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
       - name: Ensure no unhealthy status
         run: |
@@ -490,6 +492,14 @@ jobs:
         if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
+      - name: Notify Slack
+        uses: ravsamhq/notify-slack-action@v2
+        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   push-docker-image:
     needs: all-ci-passed

.github/workflows/snyk.yml

@@ -1,60 +1,28 @@
 name: Snyk Container
 on:
   push:
-    branches:
-      - "main"
-  # Snyk cannot upload results in merge group. Hence, we only run when pushing https://github.com/github/codeql-action/issues/1572
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
+    branches: ["main"]
 
 jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
-        continue-on-error: true
+      - name: Scan web image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse/langfuse
-          args: --file=web/Dockerfile
+          image: langfuse/langfuse # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)
 
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: 'sed -i ''s/"security-severity": "undefined"/"security-severity": "0"/g'' snyk.sarif'
-
-      - name: Replace security-severity null for license-related findings
-        run: 'sed -i ''s/"security-severity": "null"/"security-severity": "0"/g'' snyk.sarif'
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: web
-
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
-        continue-on-error: true
+      - name: Scan worker image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse/langfuse-worker
-          args: --file=worker/Dockerfile
-
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: 'sed -i ''s/"security-severity": "undefined"/"security-severity": "0"/g'' snyk.sarif'
-
-      - name: Replace security-severity null for license-related findings
-        run: 'sed -i ''s/"security-severity": "null"/"security-severity": "0"/g'' snyk.sarif'
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: worker
+          image: langfuse/langfuse-worker # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)

.gitignore

@@ -50,15 +50,13 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 
-/generated/typescript-server
+/generated
 
 # openapi spec that is copied during build
 /public/openapi*.yml
 
 
 # vscode
-.devcontainer
-
 node_modules
 **/node_modules
 **/dist

.nvmrc

@@ -1 +1 @@
-v20
+v20.19.2

CLAUDE.md

@@ -192,3 +192,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## General Coding Guidelines
 - For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so
+
+## Development Tips
+- Before trying to build the package, try running the linter once first

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     ports:
       - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
       DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
       SALT: "mysalt" # CHANGEME
       ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
@@ -62,6 +63,8 @@ services:
       REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
       REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
       REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
+      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-}
+      SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-}
 
   langfuse-web:
     image: docker.io/langfuse/langfuse:3
@@ -71,7 +74,6 @@ services:
       - 3000:3000
     environment:
       <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
       NEXTAUTH_SECRET: mysecret # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}

ee/package.json

@@ -26,7 +26,6 @@
   "dependencies": {
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
     "next": "^14.2.30",
     "next-auth": "^4.24.11",

fern/apis/server/definition/annotation-queues.yml

@@ -105,6 +105,28 @@ service:
           docs: The unique identifier of the annotation queue item
       response: DeleteAnnotationQueueItemResponse
 
+    createQueueAssignment:
+      docs: Create an assignment for a user to an annotation queue
+      method: POST
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: CreateAnnotationQueueAssignmentResponse
+
+    deleteQueueAssignment:
+      docs: Delete an assignment for a user to an annotation queue
+      method: DELETE
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: DeleteAnnotationQueueAssignmentResponse
+
 types:
   AnnotationQueueStatus:
     enum:
@@ -163,3 +185,17 @@ types:
     properties:
       success: boolean
       message: string
+
+  AnnotationQueueAssignmentRequest:
+    properties:
+      userId: string
+
+  DeleteAnnotationQueueAssignmentResponse:
+    properties:
+      success: boolean
+
+  CreateAnnotationQueueAssignmentResponse:
+    properties:
+      userId: string
+      queueId: string
+      projectId: string

fern/apis/server/definition/llm-connections.yml

@@ -0,0 +1,102 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    list:
+      method: GET
+      docs: Get all LLM connections in a project
+      path: /llm-connections
+      request:
+        name: GetLlmConnectionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedLlmConnections
+    upsert:
+      method: PUT
+      docs: Create or update an LLM connection. The connection is upserted on provider.
+      path: /llm-connections
+      request: UpsertLlmConnectionRequest
+      response: LlmConnection
+
+types:
+  LlmConnection:
+    docs: LLM API connection configuration (secrets excluded)
+    properties:
+      id: string
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: string
+        docs: The adapter used to interface with the LLM
+      displaySecretKey:
+        type: string
+        docs: Masked version of the secret key for display purposes
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: list<string>
+        docs: List of custom model names available for this connection
+      withDefaultModels:
+        type: boolean
+        docs: Whether to include default models for this adapter
+      extraHeaderKeys:
+        type: list<string>
+        docs: Keys of extra headers sent with requests (values excluded for security)
+      createdAt: datetime
+      updatedAt: datetime
+
+  PaginatedLlmConnections:
+    properties:
+      data: list<LlmConnection>
+      meta: pagination.MetaResponse
+
+  UpsertLlmConnectionRequest:
+    docs: Request to create or update an LLM connection (upsert)
+    properties:
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: LlmAdapter
+        docs: The adapter used to interface with the LLM
+      secretKey:
+        type: string
+        docs: Secret key for the LLM API.
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: optional<list<string>>
+        docs: List of custom model names
+      withDefaultModels:
+        type: optional<boolean>
+        docs: Whether to include default models. Default is true.
+      extraHeaders:
+        type: optional<map<string, string>>
+        docs: Extra headers to send with requests
+
+  LlmAdapter:
+    enum:
+      - value: anthropic
+        name: Anthropic
+      - value: openai
+        name: OpenAI
+      - value: azure
+        name: Azure
+      - value: bedrock
+        name: Bedrock
+      - value: google-vertex-ai
+        name: GoogleVertexAI
+      - value: google-ai-studio
+        name: GoogleAIStudio

fern/apis/server/generators.yml

@@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://schema.buildwithfern.dev/generators-yml.json
 default-group: local
 groups:
   local:
@@ -7,6 +8,7 @@ groups:
         output:
           location: local-file-system
           path: ../../../web/public/generated/api
+
       - name: fernapi/fern-python-sdk
         version: 2.16.0
         output:
@@ -19,35 +21,32 @@ groups:
           pydantic_config:
             require_optional_fields: false
             use_str_enums: false
-#      - name: fernapi/fern-java-sdk
-#        version: 2.20.1
-#        output:
-#          location: local-file-system
-#          path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
-#        config:
-#          client-class-name: LangfuseClient
+
+      # - name: fernapi/fern-java-sdk
+      #   version: 2.20.1
+      #   output:
+      #     location: local-file-system
+      #     path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
+      #   config:
+      #     client-class-name: LangfuseClient
+
+      - name: fernapi/fern-typescript-node-sdk
+        version: 2.6.1
+        output:
+          location: local-file-system
+          path: ../../../generated/typescript
+        config:
+          namespaceExport: LangfuseAPI
+          outputSourceFiles: true
+          skipResponseValidation: true
+          fetchSupport: native
+          formDataSupport: Node18
+          fileResponseType: binary-response
+          streamType: web
+          omitFernHeaders: true
+
       - name: fernapi/fern-postman
         version: 0.0.45
         output:
           location: local-file-system
           path: ../../../web/public/generated/postman
-  # published:
-  #   generators:
-  #     - name: fernapi/fern-python-sdk
-  #       version: 0.3.7
-  #       output:
-  #         location: pypi
-  #         url: pypi.buildwithfern.com
-  #         package-name: finto-fern-langfuse
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true
-  #     - name: fernapi/fern-typescript-node-sdk
-  #       version: 0.7.1
-  #       output:
-  #         location: npm
-  #         url: npm.buildwithfern.com
-  #         package-name: "@finto-fern/langfuse-node"
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true

generated/.gitignore

@@ -1 +0,0 @@
-python/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.84.0",
+  "version": "3.97.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -40,7 +40,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {
@@ -91,7 +91,8 @@
       "nanoid": "^3.3.8",
       "katex": "^0.16.21",
       "tar-fs": "^2.1.2",
-      "rollup@^4.0.0": "^4.22.4"
+      "rollup@^4.0.0": "^4.22.4",
+      "@types/node-fetch": "^2.6.13"
     },
     "patchedDependencies": {
       "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items ON CLUSTER default (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_7d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_all_amt_mv ON CLUSTER default;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_7d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_all_amt ON CLUSTER default;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt ON CLUSTER default
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv ON CLUSTER default TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv ON CLUSTER default TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv ON CLUSTER default TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv;
+DROP VIEW IF EXISTS traces_7d_amt_mv;
+DROP VIEW IF EXISTS traces_all_amt_mv;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt;
+DROP TABLE IF EXISTS traces_7d_amt;
+DROP TABLE IF EXISTS traces_all_amt;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null;

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/package.json

@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -73,13 +73,14 @@
     "@prisma/client": "^6.10.1",
     "@react-email/components": "^0.1.0",
     "@react-email/render": "^1.1.2",
+    "@slack/oauth": "^3.0.3",
+    "@slack/web-api": "^7.9.3",
     "@types/bcryptjs": "^2.4.6",
-    "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.34.10",
     "dd-trace": "^5.36.0",
     "decimal.js": "^10.4.3",
-    "exponential-backoff": "^3.1.1",
+    "exponential-backoff": "^3.1.2",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "jsonpath-plus": "10.3.0",

packages/shared/prisma/generated/types.ts

@@ -55,6 +55,7 @@ export type AnnotationQueueStatus =
 export const AnnotationQueueObjectType = {
   TRACE: "TRACE",
   OBSERVATION: "OBSERVATION",
+  SESSION: "SESSION",
 } as const;
 export type AnnotationQueueObjectType =
   (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
@@ -138,6 +139,7 @@ export type DashboardWidgetChartType =
   (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export const ActionType = {
   WEBHOOK: "WEBHOOK",
+  SLACK: "SLACK",
 } as const;
 export type ActionType = (typeof ActionType)[keyof typeof ActionType];
 export const ActionExecutionStatus = {
@@ -148,6 +150,11 @@ export const ActionExecutionStatus = {
 } as const;
 export type ActionExecutionStatus =
   (typeof ActionExecutionStatus)[keyof typeof ActionExecutionStatus];
+export const SurveyName = {
+  ORG_ONBOARDING: "org_onboarding",
+  USER_ONBOARDING: "user_onboarding",
+} as const;
+export type SurveyName = (typeof SurveyName)[keyof typeof SurveyName];
 export type Account = {
   id: string;
   user_id: string;
@@ -183,6 +190,14 @@ export type AnnotationQueue = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type AnnotationQueueAssignment = {
+  id: string;
+  project_id: string;
+  user_id: string;
+  queue_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type AnnotationQueueItem = {
   id: string;
   queue_id: string;
@@ -359,6 +374,8 @@ export type Dataset = {
   name: string;
   description: string | null;
   metadata: unknown | null;
+  remote_experiment_url: string | null;
+  remote_experiment_payload: unknown | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -624,6 +641,15 @@ export type OrganizationMembership = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type PendingDeletion = {
+  id: string;
+  project_id: string;
+  object: string;
+  object_id: string;
+  is_deleted: Generated<boolean>;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
   project_id: string;
   encrypted_posthog_api_key: string;
@@ -711,6 +737,16 @@ export type Session = {
   user_id: string;
   expires: Timestamp;
 };
+export type SlackIntegration = {
+  id: string;
+  project_id: string;
+  team_id: string;
+  team_name: string;
+  bot_token: string;
+  bot_user_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type SsoConfig = {
   domain: string;
   created_at: Generated<Timestamp>;
@@ -718,6 +754,15 @@ export type SsoConfig = {
   auth_provider: string;
   auth_config: unknown | null;
 };
+export type Survey = {
+  id: string;
+  created_at: Generated<Timestamp>;
+  survey_name: SurveyName;
+  response: unknown;
+  user_id: string | null;
+  user_email: string | null;
+  org_id: string | null;
+};
 export type TableViewPreset = {
   id: string;
   created_at: Generated<Timestamp>;
@@ -781,6 +826,7 @@ export type VerificationToken = {
 export type DB = {
   Account: Account;
   actions: Action;
+  annotation_queue_assignments: AnnotationQueueAssignment;
   annotation_queue_items: AnnotationQueueItem;
   annotation_queues: AnnotationQueue;
   api_keys: ApiKey;
@@ -813,6 +859,7 @@ export type DB = {
   observations: LegacyPrismaObservation;
   organization_memberships: OrganizationMembership;
   organizations: Organization;
+  pending_deletions: PendingDeletion;
   posthog_integrations: PosthogIntegration;
   prices: Price;
   project_memberships: ProjectMembership;
@@ -823,7 +870,9 @@ export type DB = {
   score_configs: ScoreConfig;
   scores: LegacyPrismaScore;
   Session: Session;
+  slack_integrations: SlackIntegration;
   sso_configs: SsoConfig;
+  surveys: Survey;
   table_view_presets: TableViewPreset;
   trace_media: TraceMedia;
   trace_sessions: TraceSession;

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/migrations/20250730100100_add_slack_integration/migration.sql

@@ -0,0 +1,30 @@
+-- Migration: Add Slack Integration Support
+-- This migration adds support for Slack automation actions by:
+-- 1. Adding SLACK to the ActionType enum
+-- 2. Creating slack_integrations table for centralized token storage
+
+-- AlterEnum
+ALTER TYPE "ActionType" ADD VALUE 'SLACK';
+
+-- CreateTable
+CREATE TABLE "slack_integrations" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "team_name" TEXT NOT NULL,
+    "bot_token" TEXT NOT NULL,
+    "bot_user_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "slack_integrations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "slack_integrations_project_id_key" ON "slack_integrations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "slack_integrations_team_id_idx" ON "slack_integrations"("team_id");
+
+-- AddForeignKey
+ALTER TABLE "slack_integrations" ADD CONSTRAINT "slack_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; 

packages/shared/prisma/migrations/20250731100100_add_dataset_run_items_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('8d47f91b-3e5c-4a26-9f85-c12d6e4b9a3d', '20250731_1001_migrate_dataset_run_items_pg_to_ch', 'migrateDatasetRunItemsFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20250731202005_add_trace_deletion_table/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "pending_deletions" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "is_deleted" BOOLEAN NOT NULL DEFAULT false,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "pending_deletions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_project_id_object_is_deleted_idx" ON "pending_deletions"("project_id", "object", "is_deleted");
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_object_id_object_idx" ON "pending_deletions"("object_id", "object");
+
+-- AddForeignKey
+ALTER TABLE "pending_deletions" ADD CONSTRAINT "pending_deletions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250806100613_add_annotation_queue_assignment_table/migration.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "annotation_queue_assignments" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_assignments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queue_assignments_project_id_queue_id_key" ON "annotation_queue_assignments"("project_id", "queue_id", "user_id");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250808081624_add_surveys_table/migration.sql

@@ -0,0 +1,24 @@
+-- CreateEnum
+CREATE TYPE "SurveyName" AS ENUM ('org_onboarding', 'user_onboarding');
+
+-- CreateTable
+CREATE TABLE "surveys" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "survey_name" "SurveyName" NOT NULL,
+    "response" JSONB NOT NULL,
+    "user_id" TEXT,
+    "user_email" TEXT,
+    "org_id" TEXT,
+
+    CONSTRAINT "surveys_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "surveys" ADD CONSTRAINT "surveys_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "surveys" ADD CONSTRAINT "surveys_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- RenameIndex
+ALTER INDEX "annotation_queue_assignments_project_id_queue_id_key" RENAME TO "annotation_queue_assignments_project_id_queue_id_user_id_key";

packages/shared/prisma/schema.prisma

@@ -65,29 +65,31 @@ model Session {
 }
 
 model User {
-  id                      String                   @id @default(cuid())
-  name                    String?
-  email                   String?                  @unique
-  emailVerified           DateTime?                @map("email_verified")
-  password                String?
-  image                   String?
-  admin                   Boolean                  @default(false)
-  accounts                Account[]
-  sessions                Session[]
-  organizationMemberships OrganizationMembership[]
-  projectMemberships      ProjectMembership[]
-  invitations             MembershipInvitation[]
-  createdAt               DateTime                 @default(now()) @map("created_at")
-  updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
-  featureFlags            String[]                 @default([]) @map("feature_flags")
-  annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
-  annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
-  dashboardWidgetsCreated DashboardWidget[]        @relation("CreatedByUser")
-  dashboardWidgetsUpdated DashboardWidget[]        @relation("UpdatedByUser")
-  dashboardCreated        Dashboard[]              @relation("CreatedByUser")
-  dashboardUpdated        Dashboard[]              @relation("UpdatedByUser")
-  tableViewPresetCreated  TableViewPreset[]        @relation("CreatedByUser")
-  tableViewPresetUpdated  TableViewPreset[]        @relation("UpdatedByUser")
+  id                        String                      @id @default(cuid())
+  name                      String?
+  email                     String?                     @unique
+  emailVerified             DateTime?                   @map("email_verified")
+  password                  String?
+  image                     String?
+  admin                     Boolean                     @default(false)
+  accounts                  Account[]
+  sessions                  Session[]
+  organizationMemberships   OrganizationMembership[]
+  projectMemberships        ProjectMembership[]
+  invitations               MembershipInvitation[]
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  featureFlags              String[]                    @default([]) @map("feature_flags")
+  annotatedLockedItem       AnnotationQueueItem[]       @relation("LockedByUser")
+  annotatedCompletedItem    AnnotationQueueItem[]       @relation("AnnotatorUser")
+  dashboardWidgetsCreated   DashboardWidget[]           @relation("CreatedByUser")
+  dashboardWidgetsUpdated   DashboardWidget[]           @relation("UpdatedByUser")
+  dashboardCreated          Dashboard[]                 @relation("CreatedByUser")
+  dashboardUpdated          Dashboard[]                 @relation("UpdatedByUser")
+  tableViewPresetCreated    TableViewPreset[]           @relation("CreatedByUser")
+  tableViewPresetUpdated    TableViewPreset[]           @relation("UpdatedByUser")
+  annotationQueueAssignment AnnotationQueueAssignment[]
+  surveys                   Survey[]
 
   @@map("users")
 }
@@ -112,57 +114,61 @@ model Organization {
   projects                Project[]
   MembershipInvitation    MembershipInvitation[]
   ApiKey                  ApiKey[]
+  surveys                 Survey[]
 
   @@map("organizations")
 }
 
 model Project {
-  id                     String                    @id @default(cuid())
-  orgId                  String                    @map("org_id")
-  createdAt              DateTime                  @default(now()) @map("created_at")
-  updatedAt              DateTime                  @default(now()) @updatedAt @map("updated_at")
-  deletedAt              DateTime?                 @map("deleted_at")
-  name                   String
-  retentionDays          Int?                      @map("retention_days")
-  metadata               Json?
-  projectMembers         ProjectMembership[]
-  organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
-  apiKeys                ApiKey[]
-  dataset                Dataset[]
-  invitations            MembershipInvitation[]
-  sessions               TraceSession[]
-  Prompt                 Prompt[]
-  Model                  Model[]
-  EvalTemplate           EvalTemplate[]
-  JobConfiguration       JobConfiguration[]
-  JobExecution           JobExecution[]
-  LlmApiKeys             LlmApiKeys[]
-  PosthogIntegration     PosthogIntegration[]
-  BlobStorageIntegration BlobStorageIntegration[]
-  scoreConfig            ScoreConfig[]
-  BatchExport            BatchExport[]
-  comment                Comment[]
-  annotationQueue        AnnotationQueue[]
-  annotationQueueItem    AnnotationQueueItem[]
-  TraceMedia             TraceMedia[]
-  Media                  Media[]
-  ObservationMedia       ObservationMedia[]
-  LegacyTrace            LegacyPrismaTrace[]
-  LegacyObservation      LegacyPrismaObservation[]
-  LegacyScore            LegacyPrismaScore[]
-  PromptDependency       PromptDependency[]
-  LlmSchema              LlmSchema[]
-  LlmTool                LlmTool[]
-  PromptProtectedLabels  PromptProtectedLabels[]
-  Dashboard              Dashboard[]
-  DashboardWidget        DashboardWidget[]
-  TableViewPreset        TableViewPreset[]
-  actions                Action[]
-  triggers               Trigger[]
-  automationExecutions   AutomationExecution[]
-  Automation             Automation[]
-  DefaultLlmModel        DefaultLlmModel[]
-  Price                  Price[]
+  id                        String                      @id @default(cuid())
+  orgId                     String                      @map("org_id")
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  deletedAt                 DateTime?                   @map("deleted_at")
+  name                      String
+  retentionDays             Int?                        @map("retention_days")
+  metadata                  Json?
+  projectMembers            ProjectMembership[]
+  organization              Organization                @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+  apiKeys                   ApiKey[]
+  dataset                   Dataset[]
+  invitations               MembershipInvitation[]
+  sessions                  TraceSession[]
+  Prompt                    Prompt[]
+  Model                     Model[]
+  EvalTemplate              EvalTemplate[]
+  JobConfiguration          JobConfiguration[]
+  JobExecution              JobExecution[]
+  LlmApiKeys                LlmApiKeys[]
+  PosthogIntegration        PosthogIntegration[]
+  BlobStorageIntegration    BlobStorageIntegration[]
+  scoreConfig               ScoreConfig[]
+  BatchExport               BatchExport[]
+  comment                   Comment[]
+  annotationQueue           AnnotationQueue[]
+  annotationQueueItem       AnnotationQueueItem[]
+  TraceMedia                TraceMedia[]
+  Media                     Media[]
+  ObservationMedia          ObservationMedia[]
+  LegacyTrace               LegacyPrismaTrace[]
+  LegacyObservation         LegacyPrismaObservation[]
+  LegacyScore               LegacyPrismaScore[]
+  PromptDependency          PromptDependency[]
+  LlmSchema                 LlmSchema[]
+  LlmTool                   LlmTool[]
+  PromptProtectedLabels     PromptProtectedLabels[]
+  Dashboard                 Dashboard[]
+  DashboardWidget           DashboardWidget[]
+  TableViewPreset           TableViewPreset[]
+  actions                   Action[]
+  triggers                  Trigger[]
+  automationExecutions      AutomationExecution[]
+  Automation                Automation[]
+  DefaultLlmModel           DefaultLlmModel[]
+  Price                     Price[]
+  SlackIntegration          SlackIntegration?
+  PendingDeletion           PendingDeletion[]
+  AnnotationQueueAssignment AnnotationQueueAssignment[]
 
   @@index([orgId])
   @@map("projects")
@@ -489,15 +495,16 @@ enum ScoreDataType {
 }
 
 model AnnotationQueue {
-  id                  String                @id @default(cuid())
-  name                String
-  description         String?
-  scoreConfigIds      String[]              @default([]) @map("score_config_ids")
-  projectId           String                @map("project_id")
-  project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt           DateTime              @default(now()) @map("created_at")
-  updatedAt           DateTime              @default(now()) @updatedAt @map("updated_at")
-  annotationQueueItem AnnotationQueueItem[]
+  id                        String                      @id @default(cuid())
+  name                      String
+  description               String?
+  scoreConfigIds            String[]                    @default([]) @map("score_config_ids")
+  projectId                 String                      @map("project_id")
+  project                   Project                     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  annotationQueueItem       AnnotationQueueItem[]
+  annotationQueueAssignment AnnotationQueueAssignment[]
 
   @@unique([projectId, name])
   @@index([id, projectId])
@@ -539,6 +546,22 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
+}
+
+model AnnotationQueueAssignment {
+  id        String          @id @default(cuid())
+  projectId String          @map("project_id")
+  project   Project         @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  userId    String          @map("user_id")
+  user      User            @relation(fields: [userId], references: [id], onDelete: Cascade)
+  queueId   String          @map("queue_id")
+  queue     AnnotationQueue @relation(fields: [queueId], references: [id], onDelete: Cascade)
+  createdAt DateTime        @default(now()) @map("created_at")
+  updatedAt DateTime        @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([projectId, queueId, userId])
+  @@map("annotation_queue_assignments")
 }
 
 model CronJobs {
@@ -551,16 +574,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -1295,6 +1320,7 @@ model Automation {
 
 enum ActionType {
   WEBHOOK
+  SLACK
   // More action types can be added as needed
 }
 
@@ -1335,3 +1361,62 @@ model AutomationExecution {
   @@index([projectId])
   @@map("automation_executions")
 }
+
+// Slack Integration: Stores centralized Slack workspace connection for each project
+// One project can connect to one Slack workspace, supporting multiple channel automations
+model SlackIntegration {
+  id        String  @id @default(cuid())
+  projectId String  @unique @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // Installation details (encrypted using shared encryption utilities)
+  teamId    String @map("team_id") // Slack workspace ID
+  teamName  String @map("team_name") // Human-readable workspace name
+  botToken  String @map("bot_token") // Encrypted bot token for API calls
+  botUserId String @map("bot_user_id") // Bot user ID for workspace
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([teamId])
+  @@map("slack_integrations")
+}
+
+// Pending Deletions: Tracks objects (like traces) that are scheduled for batch deletion
+model PendingDeletion {
+  id        String  @id @default(cuid())
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  object    String  @map("object") // e.g., "trace", "observation", etc.
+  objectId  String  @map("object_id") // The ID of the object to be deleted
+  isDeleted Boolean @default(false) @map("is_deleted")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, object, isDeleted])
+  @@index([objectId, object])
+  @@map("pending_deletions")
+}
+
+model Survey {
+  id         String        @id @default(cuid())
+  createdAt  DateTime      @default(now()) @map("created_at")
+  surveyName SurveyName    @map("survey_name")
+  response   Json
+  userId     String?       @map("user_id")
+  userEmail  String?       @map("user_email")
+  orgId      String?       @map("org_id")
+  org        Organization? @relation(fields: [orgId], references: [id], onDelete: Cascade)
+  user       User?         @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+  @@map("surveys")
+}
+
+enum SurveyName {
+  ORG_ONBOARDING  @map("org_onboarding")
+  USER_ONBOARDING @map("user_onboarding")
+
+  @@map("SurveyName")
+}

packages/shared/scripts/seeder/seed-postgres.ts

@@ -23,6 +23,7 @@ import {
   SEED_TEXT_PROMPTS,
 } from "./utils/postgres-seed-constants";
 import {
+  generateDatasetItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -517,6 +518,7 @@ export async function createDatasets(
             description: data.description,
             projectId,
             metadata: data.metadata,
+            id: `${datasetName}-${projectId.slice(-8)}`,
           },
         }));
 
@@ -532,13 +534,23 @@ export async function createDatasets(
         const datasetItem = await prisma.datasetItem.upsert({
           where: {
             id_projectId: {
-              id: `${dataset.id}-${index}`,
+              id: generateDatasetItemId(
+                datasetName,
+                index,
+                projectId,
+                SEED_DATASETS.indexOf(data) || 0,
+              ),
               projectId,
             },
           },
           create: {
             projectId,
-            id: `${dataset.id}-${index}`,
+            id: generateDatasetItemId(
+              datasetName,
+              index,
+              projectId,
+              SEED_DATASETS.indexOf(data) || 0,
+            ),
             datasetId: dataset.id,
             sourceTraceId: sourceTraceId ?? null,
             sourceObservationId: null,
@@ -554,14 +566,14 @@ export async function createDatasets(
       for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.upsert({
           where: {
-            datasetId_projectId_name: {
-              datasetId: dataset.id,
+            id_projectId: {
+              id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
               projectId,
-              name: `demo-dataset-run-${datasetRunNumber}`,
             },
           },
           create: {
             projectId,
+            id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -2,6 +2,8 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItemsCh,
 } from "../../../src/server";
 import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
 import {
@@ -42,6 +44,16 @@ export class ClickHouseQueryBuilder {
     return await createObservationsCh(observations);
   }
 
+  /**
+   * Creates INSERT query for dataset run items data using VALUES syntax.
+   * Use for: Small datasets, dataset run items that link to postgres data (e.g. dataset runs)
+   */
+  async executeDatasetRunItemsInsert(
+    datasetRunItems: DatasetRunItemRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createDatasetRunItemsCh(datasetRunItems);
+  }
+
   /**
    * Creates INSERT query for score data using VALUES syntax.
    * Use for: Small datasets, scores with custom values and metadata.

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -6,6 +6,8 @@ import {
   REALISTIC_MODELS,
 } from "./clickhouse-seed-constants";
 import {
+  generateDatasetItemId,
+  generateDatasetRunItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -23,6 +25,8 @@ import {
   ObservationRecordInsertType,
   ScoreRecordInsertType,
   TraceRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItem,
 } from "../../../src/server";
 
 /**
@@ -60,6 +64,49 @@ export class DataGenerator {
     return Math.floor(Math.random() * (max - min + 1)) + min;
   }
 
+  /**
+   * Creates dataset run items for dataset runs.
+   * Use for: Dataset experiment scenarios.
+   */
+  generateDatasetRunItem(
+    input: DatasetItemInput & { runCreatedAt: number },
+    projectId: string,
+  ): DatasetRunItemRecordInsertType {
+    const datasetRunItemId = generateDatasetRunItemId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    // TODO: there are too many dataset run items in the postgres database?
+    return createDatasetRunItem({
+      id: datasetRunItemId,
+      project_id: projectId,
+      trace_id: generateDatasetRunTraceId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_id: `${input.datasetName}-${projectId.slice(-8)}`,
+      dataset_run_id: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_name: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_created_at: input.runCreatedAt,
+      dataset_run_description:
+        (input.runNumber || 0) % 2 === 0 ? "Dataset run description" : "",
+      dataset_run_metadata: { key: "value" },
+      dataset_item_id: generateDatasetItemId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_item_input: input.item.input,
+      dataset_item_expected_output: input.item.output,
+    });
+  }
+
   /**
    * Creates traces from dataset items for experiment runs.
    * Use for: Dataset experiments scenarios.

packages/shared/scripts/seeder/utils/markdown.txt

@@ -11,29 +11,29 @@
 ## 🎯 Getting Started
 
 ### Prerequisites
-\`\`\`bash
-npm install @langfuse/core
-pip install langfuse
+\`\`\`bash  
+npm install @langfuse/core  
+pip install langfuse  
 \`\`\`
 
 ### Quick Setup
 1. **Initialize your project**
-   \`\`\`typescript
-   import { Langfuse } from 'langfuse'
-   
-   const langfuse = new Langfuse({
-     secretKey: process.env.LANGFUSE_SECRET_KEY,
-     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-     baseUrl: 'https://cloud.langfuse.com'
-   })
+   \`\`\`typescript  
+   import { Langfuse } from 'langfuse'  
+
+   const langfuse = new Langfuse({  
+     secretKey: process.env.LANGFUSE_SECRET_KEY,  
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,  
+     baseUrl: 'https://cloud.langfuse.com'  
+   })  
    \`\`\`
 
 2. **Create your first trace**
-   \`\`\`python
-   from langfuse import Langfuse
-   
-   langfuse = Langfuse()
-   trace = langfuse.trace(name="chat-application")
+   \`\`\`python  
+   from langfuse import Langfuse  
+
+   langfuse = Langfuse()  
+   trace = langfuse.trace(name="chat-application")  
    \`\`\`
 
 ---
@@ -72,75 +72,75 @@ graph TD
 > **Note:** Traces are the foundation of observability in LLM applications.
 
 #### Creating Traces
-\`\`\`typescript
-// Basic trace creation
-const trace = langfuse.trace({
-  name: "user-query-processing",
-  userId: "user-123",
-  sessionId: "session-456",
-  metadata: {
-    environment: "production",
-    version: "2.1.0"
-  }
-})
+\`\`\`typescript  
+// Basic trace creation  
+const trace = langfuse.trace({  
+  name: "user-query-processing",  
+  userId: "user-123",  
+  sessionId: "session-456",  
+  metadata: {  
+    environment: "production",  
+    version: "2.1.0"  
+  }  
+})  
 
-// Nested observations
-const span = trace.span({
-  name: "document-retrieval",
-  input: { query: "What is machine learning?" },
-  metadata: { vectorStore: "pinecone" }
-})
+// Nested observations  
+const span = trace.span({  
+  name: "document-retrieval",  
+  input: { query: "What is machine learning?" },  
+  metadata: { vectorStore: "pinecone" }  
+})  
 
-const generation = span.generation({
-  name: "answer-generation",
-  model: "gpt-4",
-  input: retrievedDocs,
-  output: generatedAnswer,
-  usage: {
-    promptTokens: 1250,
-    completionTokens: 420,
-    totalTokens: 1670
-  }
-})
+const generation = span.generation({  
+  name: "answer-generation",  
+  model: "gpt-4",  
+  input: retrievedDocs,  
+  output: generatedAnswer,  
+  usage: {  
+    promptTokens: 1250,  
+    completionTokens: 420,  
+    totalTokens: 1670  
+  }  
+})  
 \`\`\`
 
 ### Advanced Features
 
 #### 🔄 Async Processing
-\`\`\`python
-import asyncio
-from langfuse import Langfuse
+\`\`\`python  
+import asyncio  
+from langfuse import Langfuse  
 
-async def process_batch():
-    langfuse = Langfuse()
-    
-    tasks = []
-    for item in batch_items:
-        task = asyncio.create_task(
-            process_item_with_tracing(langfuse, item)
-        )
-        tasks.append(task)
-    
-    results = await asyncio.gather(*tasks)
-    return results
+async def process_batch():  
+    langfuse = Langfuse()  
+
+    tasks = []  
+    for item in batch_items:  
+        task = asyncio.create_task(  
+            process_item_with_tracing(langfuse, item)  
+        )  
+        tasks.append(task)  
+
+    results = await asyncio.gather(*tasks)  
+    return results  
 \`\`\`
 
 #### 🎯 Custom Scoring
-\`\`\`typescript
-// Automated scoring
-trace.score({
-  name: "relevance",
-  value: 0.95,
-  comment: "Highly relevant response"
-})
+\`\`\`typescript  
+// Automated scoring  
+trace.score({  
+  name: "relevance",  
+  value: 0.95,  
+  comment: "Highly relevant response"  
+})  
 
-// Human feedback scoring
-trace.score({
-  name: "user-satisfaction", 
-  value: 1,
-  source: "user-feedback",
-  comment: "User rated 5/5 stars"
-})
+// Human feedback scoring  
+trace.score({  
+  name: "user-satisfaction",  
+  value: 1,  
+  source: "user-feedback",  
+  comment: "User rated 5/5 stars"  
+})  
 \`\`\`
 
 ---
@@ -156,20 +156,20 @@ trace.score({
 - **User Satisfaction**: Quality metrics
 
 #### Dashboard Setup
-\`\`\`yaml
-# monitoring-config.yml
-dashboards:
-  - name: "LLM Performance"
-    panels:
-      - type: "time-series"
-        title: "Response Latency"
-        query: "avg(response_time) by (model)"
-      - type: "stat"
-        title: "Daily Token Usage" 
-        query: "sum(tokens_used)"
-      - type: "table"
-        title: "Top Errors"
-        query: "topk(10, count by (error_type))"
+\`\`\`yaml  
+# monitoring-config.yml  
+dashboards:  
+  - name: "LLM Performance"  
+    panels:  
+      - type: "time-series"  
+        title: "Response Latency"  
+        query: "avg(response_time) by (model)"  
+      - type: "stat"  
+        title: "Daily Token Usage"  
+        query: "sum(tokens_used)"  
+      - type: "table"  
+        title: "Top Errors"  
+        query: "topk(10, count by (error_type))"  
 \`\`\`
 
 ### 🔐 Security Considerations
@@ -177,40 +177,40 @@ dashboards:
 > ⚠️ **Important**: Never log sensitive user data in traces
 
 #### Data Sanitization
-\`\`\`python
-def sanitize_input(data):
-    """Remove PII from trace data"""
-    sanitized = data.copy()
-    
-    # Remove email addresses
-    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
-                      '[EMAIL_REDACTED]', sanitized)
-    
-    # Remove phone numbers
-    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
-                      '[PHONE_REDACTED]', sanitized)
-    
-    return sanitized
+\`\`\`python  
+def sanitize_input(data):  
+    """Remove PII from trace data"""  
+    sanitized = data.copy()  
+
+    # Remove email addresses  
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b',  
+                      '[EMAIL_REDACTED]', sanitized)  
+
+    # Remove phone numbers  
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b',  
+                      '[PHONE_REDACTED]', sanitized)  
+
+    return sanitized  
 \`\`\`
 
 ### 🚀 Performance Optimization
 
 #### Batch Processing
-\`\`\`typescript
-// Efficient batch uploads
-const batchSize = 100
-const traces = []
+\`\`\`typescript  
+// Efficient batch uploads  
+const batchSize = 100  
+const traces = []  
 
-for (let i = 0; i < data.length; i += batchSize) {
-  const batch = data.slice(i, i + batchSize)
-  const processedBatch = await Promise.all(
-    batch.map(item => processWithLangfuse(item))
-  )
-  traces.push(...processedBatch)
-}
+for (let i = 0; i < data.length; i += batchSize) {  
+  const batch = data.slice(i, i + batchSize)  
+  const processedBatch = await Promise.all(  
+    batch.map(item => processWithLangfuse(item))  
+  )  
+  traces.push(...processedBatch)  
+}  
 
-// Flush all traces at once
-await langfuse.flushAsync()
+// Flush all traces at once  
+await langfuse.flushAsync()  
 \`\`\`
 
 ---
@@ -219,34 +219,34 @@ await langfuse.flushAsync()
 
 ### Multi-Agent System Tracing
 \`\`\`python
-class MultiAgentTracer:
-    def __init__(self):
-        self.langfuse = Langfuse()
-    
-    async def orchestrate_agents(self, task):
-        # Main orchestration trace
-        main_trace = self.langfuse.trace(
-            name="multi-agent-orchestration",
-            input={"task": task}
-        )
-        
-        # Agent 1: Research
-        research_span = main_trace.span(name="research-agent")
-        research_result = await self.research_agent.process(task)
-        research_span.end(output=research_result)
-        
+class MultiAgentTracer:  
+    def __init__(self):  
+        self.langfuse = Langfuse()  
+
+    async def orchestrate_agents(self, task):  
+        # Main orchestration trace  
+        main_trace = self.langfuse.trace(  
+            name="multi-agent-orchestration",  
+            input={"task": task}  
+        )  
+
+        # Agent 1: Research  
+        research_span = main_trace.span(name="research-agent")  
+        research_result = await self.research_agent.process(task)  
+        research_span.end(output=research_result)  
+
         # Agent 2: Analysis  
-        analysis_span = main_trace.span(name="analysis-agent")
-        analysis_result = await self.analysis_agent.process(research_result)
-        analysis_span.end(output=analysis_result)
-        
-        # Agent 3: Synthesis
-        synthesis_span = main_trace.span(name="synthesis-agent")
-        final_result = await self.synthesis_agent.process(analysis_result)
-        synthesis_span.end(output=final_result)
-        
-        main_trace.end(output=final_result)
-        return final_result
+        analysis_span = main_trace.span(name="analysis-agent")  
+        analysis_result = await self.analysis_agent.process(research_result)  
+        analysis_span.end(output=analysis_result)  
+
+        # Agent 3: Synthesis  
+        synthesis_span = main_trace.span(name="synthesis-agent")  
+        final_result = await self.synthesis_agent.process(analysis_result)  
+        synthesis_span.end(output=final_result)  
+
+        main_trace.end(output=final_result)  
+        return final_result  
 \`\`\`
 
 ---
@@ -256,11 +256,11 @@ class MultiAgentTracer:
 With proper implementation of Langfuse tracing, you can:
 
 - ✅ **Monitor** your LLM applications in real-time
-- ✅ **Debug** issues with detailed trace information  
+- ✅ **Debug** issues with detailed trace information
 - ✅ **Optimize** performance and costs
 - ✅ **Scale** your applications with confidence
 
 ### Next Steps
 1. Review the [official documentation](https://langfuse.com/docs)
 2. Join our [Discord community](https://discord.gg/langfuse)
-3. Check out [example projects](https://github.com/langfuse/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -1,3 +1,21 @@
+export const generateDatasetRunItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-run-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateDatasetItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
 export const generateDatasetRunTraceId = (
   datasetName: string,
   itemIndex: number,

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -92,12 +92,26 @@ export class SeederOrchestrator {
         logger.info(
           `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
         );
+        // const now = Date.now();
 
         const traces: TraceRecordInsertType[] = [];
         const observations: ObservationRecordInsertType[] = [];
+        // const datasetRunItems: DatasetRunItemRecordInsertType[] = [];
 
         for (const seedDataset of SEED_DATASETS) {
           for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // // Generate dataset run item data
+            // const datasetRunItem = this.dataGenerator.generateDatasetRunItem(
+            //   {
+            //     datasetName: seedDataset.name,
+            //     itemIndex,
+            //     item: datasetItem,
+            //     runNumber,
+            //     runCreatedAt: now,
+            //   },
+            //   projectId,
+            // );
+
             // Generate trace data
             const trace = this.dataGenerator.generateDatasetTrace(
               {
@@ -123,12 +137,14 @@ export class SeederOrchestrator {
 
             traces.push(trace);
             observations.push(observation);
+            // datasetRunItems.push(datasetRunItem);
           }
         }
 
         try {
           await this.queryBuilder.executeTracesInsert(traces);
           await this.queryBuilder.executeObservationsInsert(observations);
+          // await this.queryBuilder.executeDatasetRunItemsInsert(datasetRunItems);
         } catch (error) {
           logger.error(`✗ Insert failed:`, error);
           throw error;

packages/shared/src/domain/automations.ts

@@ -30,14 +30,14 @@ export type AutomationDomain = {
 };
 
 export type ActionDomain = Omit<Action, "config"> & {
-  config: SafeWebhookActionConfig;
+  config: SafeActionConfig;
 };
 
 export type ActionDomainWithSecrets = Omit<Action, "config"> & {
-  config: WebhookActionConfigWithSecrets;
+  config: ActionConfigWithSecrets;
 };
 
-export const ActionTypeSchema = z.enum(["WEBHOOK"]);
+export const ActionTypeSchema = z.enum(["WEBHOOK", "SLACK"]);
 
 export const AvailableWebhookApiSchema = z.record(
   z.enum(["prompt"]),
@@ -52,12 +52,13 @@ export const RequestHeaderSchema = z.object({
 export const WebhookActionConfigSchema = z.object({
   type: z.literal("WEBHOOK"),
   url: z.url(),
-  headers: z.record(z.string(), z.string()),
-  requestHeaders: z.record(z.string(), RequestHeaderSchema),
-  displayHeaders: z.record(z.string(), RequestHeaderSchema),
+  headers: z.record(z.string(), z.string()).optional(), // deprecated field, use requestHeaders instead
+  requestHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
+  displayHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
   apiVersion: AvailableWebhookApiSchema,
   secretKey: z.string(),
   displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
 });
 
 export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
@@ -77,18 +78,94 @@ export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
   displayHeaders: true,
 });
 
+export const SlackActionConfigSchema = z.object({
+  type: z.literal("SLACK"),
+  channelId: z.string(),
+  channelName: z.string(),
+  messageTemplate: z.string().optional(),
+});
+
+export type SlackActionConfig = z.infer<typeof SlackActionConfigSchema>;
+
 export const ActionConfigSchema = z.discriminatedUnion("type", [
   WebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export const ActionCreateSchema = z.discriminatedUnion("type", [
   WebhookActionCreateSchema,
+  SlackActionConfigSchema,
+]);
+
+export const SafeActionConfigSchema = z.discriminatedUnion("type", [
+  SafeWebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export type ActionTypes = z.infer<typeof ActionTypeSchema>;
 export type ActionConfig = z.infer<typeof ActionConfigSchema>;
 export type ActionCreate = z.infer<typeof ActionCreateSchema>;
+export type SafeActionConfig = z.infer<typeof SafeActionConfigSchema>;
 
+export type WebhookActionCreate = z.infer<typeof WebhookActionCreateSchema>;
 export type WebhookActionConfigWithSecrets = z.infer<
   typeof WebhookActionConfigSchema
 >;
+
+export type ActionConfigWithSecrets = z.infer<typeof ActionConfigSchema>;
+
+// Type Guards for Runtime Validation
+// Using existing Zod schemas to provide both compile-time and runtime type safety
+
+/**
+ * Type guard to check if a config is a valid webhook configuration with secrets
+ */
+export function isWebhookActionConfig(
+  config: unknown,
+): config is WebhookActionConfigWithSecrets {
+  return WebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if a config is a valid Slack configuration
+ */
+export function isSlackActionConfig(
+  config: unknown,
+): config is SlackActionConfig {
+  return SlackActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if an entire action has valid webhook configuration
+ */
+export function isWebhookAction(action: {
+  type: string;
+  config: unknown;
+}): action is { type: "WEBHOOK"; config: WebhookActionConfigWithSecrets } {
+  return action.type === "WEBHOOK" && isWebhookActionConfig(action.config);
+}
+
+/**
+ * Type guard for safe webhook config (without secrets)
+ */
+export function isSafeWebhookActionConfig(
+  config: unknown,
+): config is SafeWebhookActionConfig {
+  return SafeWebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Converts webhook config with secrets to safe config by only including allowed fields
+ */
+export function convertToSafeWebhookConfig(
+  webhookConfig: WebhookActionConfigWithSecrets,
+): SafeWebhookActionConfig {
+  return {
+    type: webhookConfig.type,
+    url: webhookConfig.url,
+    displayHeaders: webhookConfig.displayHeaders,
+    apiVersion: webhookConfig.apiVersion,
+    displaySecretKey: webhookConfig.displaySecretKey,
+    lastFailingExecutionId: webhookConfig.lastFailingExecutionId,
+  };
+}

packages/shared/src/domain/dataset-run-items.ts

@@ -0,0 +1,28 @@
+import z from "zod/v4";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const DatasetRunItemSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  datasetRunId: z.string(),
+  datasetItemId: z.string(),
+  datasetId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().nullable(),
+  error: z.string().nullable(),
+  // timestamps
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  // dataset run fields
+  datasetRunName: z.string(),
+  datasetRunDescription: z.string().nullable(),
+  datasetRunMetadata: MetadataDomain,
+  datasetRunCreatedAt: z.date(),
+  // dataset item fields
+  datasetItemInput: jsonSchema,
+  datasetItemExpectedOutput: jsonSchema,
+  datasetItemMetadata: MetadataDomain,
+});
+
+export type DatasetRunItemDomain = z.infer<typeof DatasetRunItemSchema>;

packages/shared/src/env.ts

@@ -15,7 +15,9 @@ const EnvSchema = z.object({
     .default(6379)
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
+  REDIS_USERNAME: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
+  REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
   REDIS_TLS_CA_PATH: z.string().optional(),
   REDIS_TLS_CERT_PATH: z.string().optional(),
@@ -31,8 +33,10 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
     )
     .optional(),
+  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
-  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(300),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(300), // 5 minutes
   CLICKHOUSE_URL: z.string().url(),
   CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
   CLICKHOUSE_DB: z.string().default("default"),
@@ -46,6 +50,14 @@ const EnvSchema = z.object({
     .nonnegative()
     .default(15_000),
   LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
+  LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT: z.coerce
+    .number()
+    .positive()
+    .default(1),
+  LANGFUSE_TRACE_DELETE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(5_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -84,6 +96,9 @@ const EnvSchema = z.object({
   LANGFUSE_S3_MEDIA_UPLOAD_SSE: z.enum(["AES256", "aws:kms"]).optional(),
   LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_AZURE_SKIP_CONTAINER_CHECK: z
+    .enum(["true", "false"])
+    .default("true"),
   LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
@@ -107,7 +122,13 @@ const EnvSchema = z.object({
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
   LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
   LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
-
+  // Dataset Run Items Migration Environment Variables
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH: z
+    .enum(["true", "false"])
+    .default("false"),
   LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES: z
     .enum(["true", "false"])
     .default("false"),
@@ -131,6 +152,60 @@ const EnvSchema = z.object({
   LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
+  SLACK_CLIENT_ID: z.string().optional(),
+  SLACK_CLIENT_SECRET: z.string().optional(),
+  SLACK_STATE_SECRET: z.string().optional(),
+  HTTPS_PROXY: z.string().optional(),
+
+  LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(1_000),
+
+  LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(600_000), // 10 minutes
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/annotation/types.ts

@@ -91,4 +91,11 @@ export const CreateQueueData = z.object({
   }),
 });
 
+export const CreateQueueWithAssignmentsData = CreateQueueData.extend({
+  newAssignmentUserIds: z.array(z.string()),
+});
+
 export type CreateQueue = z.infer<typeof CreateQueueData>;
+export type CreateQueueWithAssignments = z.infer<
+  typeof CreateQueueWithAssignmentsData
+>;

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,8 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
+  "observation-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/features/evals/utilities.ts

@@ -27,16 +27,42 @@ export const parseUnknownToString = (value: unknown): string => {
   return String(value);
 };
 
+/**
+ * Recursively parses JSON strings that may have been encoded multiple times.
+ * This handles cases where data has been JSON.stringify'd multiple times.
+ *
+ * @param value - The potentially multi-encoded JSON string
+ * @returns The final parsed object or the original value if parsing fails
+ */
+function parseMultiEncodedJson(value: unknown): unknown {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  try {
+    const parsed = JSON.parse(value);
+
+    // If result is still a string, it might be double-encoded - recurse
+    if (typeof parsed === "string") {
+      return parseMultiEncodedJson(parsed);
+    }
+
+    return parsed;
+  } catch {
+    // If parsing fails, return original value
+    return value;
+  }
+}
+
 function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
+  // selectedColumn should already be preprocessed by preprocessObjectWithJsonFields
+  // so we can directly use it with JSONPath
   const result = JSONPath({
     path: jsonSelector,
-    json:
-      typeof selectedColumn === "string"
-        ? JSON.parse(selectedColumn)
-        : selectedColumn,
+    json: selectedColumn as any, // JSONPath accepts unknown but types are strict
   });
 
-  return result.length > 0 ? result[0] : undefined;
+  return Array.isArray(result) && result.length > 0 ? result[0] : undefined;
 }
 
 export function extractValueFromObject(
@@ -44,7 +70,13 @@ export function extractValueFromObject(
   mapping: z.infer<typeof variableMapping>,
   parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown, // eslint-disable-line no-unused-vars
 ): { value: string; error: Error | null } {
-  const selectedColumn = obj[mapping.selectedColumnId];
+  let selectedColumn = obj[mapping.selectedColumnId];
+
+  // Simple preprocessing: attempt to parse to valid JSON object
+  if (typeof selectedColumn === "string") {
+    selectedColumn = parseMultiEncodedJson(selectedColumn);
+  }
+
   const jsonParser = parseJson || parseJsonDefault;
 
   let jsonSelectedColumn;

packages/shared/src/index.ts

@@ -18,6 +18,7 @@ export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
 export * from "./domain/webhooks";
+export * from "./domain/dataset-run-items";
 
 // llm api
 export * from "./server/llm/types";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -18,6 +18,21 @@ export const CloudConfigSchema = z.object({
 
   // custom rate limits for an organization
   rateLimitOverrides: CloudConfigRateLimit.optional(),
+
+  // billing alert configuration
+  usageAlerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      type: z.enum(["STRIPE"]).default("STRIPE"),
+      threshold: z.number().int().positive(),
+      alertId: z.string(), // Alert ID for tracking
+      meterId: z.string(), // Meter ID for usage tracking
+      notifications: z.object({
+        email: z.boolean().default(true),
+        recipients: z.array(z.string().email()).default([]),
+      }),
+    })
+    .optional(),
 });
 
 export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/server/auth/types.ts

@@ -1,7 +1,7 @@
 import z from "zod/v4";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
-import { ApiKeyScope } from "../../";
+import { ApiKeyScope, MakeOptional } from "../../";
 
 const ApiKeyBaseSchema = z.object({
   id: z.string(),
@@ -48,12 +48,25 @@ export type AuthHeaderValidVerificationResult = {
   scope: ApiAccessScope;
 };
 
-export type ApiAccessScope = {
+export type AuthHeaderValidVerificationResultIngestion = {
+  validKey: true;
+  scope: ApiAccessScopeIngestion;
+};
+
+type BaseApiAccessScope = {
   projectId: string | null;
   accessLevel: "organization" | "project" | "scores";
+};
+
+type ApiAccessScopeMetadata = {
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
   apiKeyId: string;
   publicKey: string;
 };
+
+export type ApiAccessScopeIngestion = BaseApiAccessScope &
+  MakeOptional<ApiAccessScopeMetadata>;
+
+export type ApiAccessScope = BaseApiAccessScope & ApiAccessScopeMetadata;

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -1,4 +1,4 @@
-import { instrumentAsync } from "../instrumentation";
+import { instrumentAsync, recordDistribution } from "../instrumentation";
 import * as opentelemetry from "@opentelemetry/api";
 import { env } from "../../env";
 import { logger } from "../logger";
@@ -19,12 +19,17 @@ const executionWrapper = async <T, Y>(
   return [res, duration];
 };
 
+/**
+ * Measures the execution time of two functions and returns the result based on the experiment configuration.
+ * This is used to compare the execution of AggregatingMergeTrees with the existing ReplacingMergeTree execution.
+ */
 export const measureAndReturn = async <T, Y>(args: {
   operationName: string;
   projectId: string;
   input: T;
   existingExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
   newExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
+  minStartTime?: Date;
 }): Promise<Y> => {
   return instrumentAsync(
     {
@@ -32,14 +37,34 @@ export const measureAndReturn = async <T, Y>(args: {
       spanKind: opentelemetry.SpanKind.CLIENT,
     },
     async (currentSpan) => {
-      const { input, existingExecution, newExecution } = args;
+      const { input, existingExecution, newExecution, minStartTime } = args;
 
       if (
         env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES !==
         "true"
       ) {
         currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
-        return existingExecution(input);
+
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newExecution(input);
+          }
+        }
+
+        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
+          ? newExecution(input)
+          : existingExecution(input);
       }
 
       // If not whitelisted, apply sampling logic
@@ -68,6 +93,14 @@ export const measureAndReturn = async <T, Y>(args: {
           durationDifference,
         );
 
+        recordDistribution(
+          "langfuse.experiment.amts.duration_difference_distribution",
+          durationDifference,
+          {
+            operation: args.operationName,
+          },
+        );
+
         if (
           env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
             (p) => p === args.projectId,
@@ -83,6 +116,23 @@ export const measureAndReturn = async <T, Y>(args: {
           );
         }
 
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newResult;
+          }
+        }
+
         return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
           ? newResult
           : existingResult;

packages/shared/src/server/clickhouse/schema.ts

@@ -2,6 +2,7 @@ export const ClickhouseTableNames = {
   traces: "traces",
   observations: "observations",
   scores: "scores",
+  dataset_run_items: "dataset_run_items",
 
   // Virtual tables for dashboards
   // TODO: Check if we can do this more elegantly

packages/shared/src/server/clickhouse/schemaUtils.ts

@@ -11,7 +11,8 @@ export type IngestionEntityTypes =
   | "trace"
   | "observation"
   | "score"
-  | "sdk_log";
+  | "sdk_log"
+  | "dataset_run_item";
 
 export const getClickhouseEntityType = (
   eventType: string,
@@ -29,6 +30,8 @@ export const getClickhouseEntityType = (
       return "observation";
     case eventTypes.SCORE_CREATE:
       return "score";
+    case eventTypes.DATASET_RUN_ITEM_CREATE:
+      return "dataset_run_item";
     case eventTypes.SDK_LOG:
       return "sdk_log";
     default:

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/dataset-run-items/datasetExecution.ts

@@ -0,0 +1,94 @@
+import { env } from "../../env";
+import { logger } from "../../server/logger";
+import {
+  DatasetRunItemsExecutionStrategy,
+  DatasetRunItemsOperationType,
+} from "./types";
+/**
+ * Returns the execution strategy for dataset run items based on environment variables.
+ *
+ * Two-phase migration approach:
+ * 1. Dual-write phase: DATASET_RUN_ITEMS_WRITE_TO_CLICKHOUSE=true (write to both databases)
+ * 2. Read migration phase: DATASET_RUN_ITEMS_READ_FROM_CLICKHOUSE=true (read from ClickHouse)
+ */
+function getDatasetRunItemsExecutionStrategy(): DatasetRunItemsExecutionStrategy {
+  return {
+    shouldWriteToClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH === "true",
+    shouldReadFromClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH === "true",
+  };
+}
+
+// Re-export the enum for backward compatibility
+
+/**
+ * Executes the appropriate database operation based on the execution strategy.
+ *
+ * @param postgresExecution - Function to execute PostgreSQL operation
+ * @param clickhouseExecution - Function to execute ClickHouse operation
+ * @param operationType - Type of operation ("read" or "write")
+ * @returns Result from the selected execution strategy
+ */
+export async function executeWithDatasetRunItemsStrategy<TInput, TOutput>({
+  input,
+  operationType,
+  postgresExecution,
+  clickhouseExecution,
+}: {
+  input: TInput;
+  operationType: DatasetRunItemsOperationType;
+  // eslint-disable-next-line no-unused-vars
+  postgresExecution: (input: TInput) => Promise<TOutput>;
+  // eslint-disable-next-line no-unused-vars
+  clickhouseExecution: (input: TInput) => Promise<TOutput>;
+}): Promise<TOutput> {
+  const strategy = getDatasetRunItemsExecutionStrategy();
+
+  if (operationType === DatasetRunItemsOperationType.WRITE) {
+    // For write operations, implement dual-write strategy
+    if (strategy.shouldWriteToClickHouse) {
+      // Dual-write phase: write to both databases
+      const postgresResult = await postgresExecution(input);
+
+      try {
+        await clickhouseExecution(input);
+        logger.debug("Successfully wrote to both PostgreSQL and ClickHouse", {
+          operation: `dataset_run_items_${operationType}`,
+        });
+      } catch (error) {
+        logger.error("ClickHouse write failed during dual-write phase", {
+          error: error instanceof Error ? error.message : String(error),
+          operation: `dataset_run_items_${operationType}`,
+        });
+        // Continue with PostgreSQL result since it succeeded
+      }
+
+      return postgresResult;
+    } else {
+      // Write only to PostgreSQL
+      return await postgresExecution(input);
+    }
+  } else {
+    // For read operations, rely on the strategy
+    const shouldExecuteClickhouse = strategy.shouldReadFromClickHouse;
+
+    if (shouldExecuteClickhouse) {
+      try {
+        return await clickhouseExecution(input);
+      } catch (error) {
+        logger.error(
+          "ClickHouse execution failed, falling back to PostgreSQL",
+          {
+            error: error instanceof Error ? error.message : String(error),
+            operation: `dataset_run_items_${operationType}`,
+          },
+        );
+        // Fallback to PostgreSQL for reliability
+        return await postgresExecution(input);
+      }
+    } else {
+      return await postgresExecution(input);
+    }
+  }
+}

packages/shared/src/server/dataset-run-items/types.ts

@@ -0,0 +1,16 @@
+/**
+ * Types and enums for dataset run items execution.
+ * This file is frontend-safe and doesn't import server-side dependencies.
+ */
+
+export enum DatasetRunItemsOperationType {
+  // eslint-disable-next-line no-unused-vars
+  READ = "read",
+  // eslint-disable-next-line no-unused-vars
+  WRITE = "write",
+}
+
+export type DatasetRunItemsExecutionStrategy = {
+  shouldWriteToClickHouse: boolean;
+  shouldReadFromClickHouse: boolean;
+};

packages/shared/src/server/index.ts

@@ -2,6 +2,7 @@ export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
+export * from "./services/email/billingAlert/sendBillingAlertEmail";
 export * from "./services/PromptService";
 export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
@@ -13,6 +14,7 @@ export * from "./llm/fetchLLMCompletion";
 export * from "./llm/utils";
 export * from "./llm/types";
 export * from "./llm/compileChatMessages";
+export * from "./llm/testModelCall";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
 export * from "./clickhouse/client";
@@ -20,6 +22,7 @@ export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
+export * from "../server/ingestion/modelMatch";
 export * from "./ingestion/processEventBatch";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
@@ -31,6 +34,7 @@ export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -55,6 +59,7 @@ export * from "./logger";
 export * from "./headerPropagation";
 export * from "./queries";
 export * from "./repositories";
+export * from "./utils/rendering";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
 export * from "./services/datasets-ui-table-service";
@@ -62,10 +67,17 @@ export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
 export * from "./clickhouse/measureAndReturn";
+export * from "./services/SlackService";
 
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
+// dataset run items
+export * from "./dataset-run-items/datasetExecution";
+export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
+
 // test utils
 export * from "./test-utils";
 export * from "./utils/headerUtils";
+export * from "./traceDeletionProcessor";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -1,19 +1,23 @@
-import { Model, Prisma } from "@langfuse/shared";
+import { Model, Prisma } from "../../";
 import {
   instrumentAsync,
   logger,
   recordIncrement,
-} from "@langfuse/shared/src/server";
-import { env } from "../env";
-import { redis } from "@langfuse/shared/src/server";
+  redis,
+  safeMultiDel,
+} from "../";
+import { type Cluster } from "ioredis";
+import { env } from "../../env";
 import { Decimal } from "decimal.js";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma } from "../../db";
 
 export type ModelMatchProps = {
   projectId: string;
   model: string;
 };
 
+const MODEL_MATCH_CACHE_LOCKED_KEY = "LOCK:model-match-clear";
+
 export async function findModel(p: ModelMatchProps): Promise<Model | null> {
   return instrumentAsync(
     {
@@ -76,12 +80,20 @@ const getModelFromRedis = async (
   }
 
   try {
+    if (await isModelMatchCacheLocked()) {
+      logger.info(
+        "Model match cache is locked. Skipping model lookup from Redis.",
+      );
+
+      return null;
+    }
+
     const key = getRedisModelKey(p);
     const redisModel = await redis?.get(key);
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;
@@ -178,6 +190,11 @@ export const getRedisModelKey = (p: ModelMatchProps) => {
 };
 
 const getModelMatchKeyPrefix = () => {
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // Use hash tags for Redis cluster compatibility
+    // This ensures all model cache keys are placed on the same hash slot
+    return "{model-match}";
+  }
   return "model-match";
 };
 
@@ -205,3 +222,99 @@ export const redisModelToPrismaModel = (redisModel: string): Model => {
         : null,
   };
 };
+
+export async function clearModelCacheForProject(
+  projectId: string,
+): Promise<void> {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    const pattern = `${getModelMatchKeyPrefix()}:${projectId}:*`;
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared ${keys.length} model cache entries for project ${projectId}`,
+      );
+    }
+  } catch (error) {
+    logger.error(
+      `Error clearing model cache for project ${projectId}: ${error}`,
+    );
+  }
+}
+
+export async function isModelMatchCacheLocked() {
+  try {
+    return Boolean(await redis?.exists(MODEL_MATCH_CACHE_LOCKED_KEY));
+  } catch (err) {
+    logger.error("Failed to check whether model match is locked", err);
+
+    return false;
+  }
+}
+
+export async function clearFullModelCache() {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    // Use lock to protect for concurrent executions
+    // This function is called on worker startup, so we want to avoid all workers triggering this delete
+    if (await isModelMatchCacheLocked()) {
+      logger.info("Model cache clearing already in progress; skipping.");
+
+      return;
+    }
+
+    const startTime = Date.now();
+    logger.info("Clearing full model cache...");
+
+    const tenMinutesInSeconds = 60 * 10;
+    await redis.setex(
+      MODEL_MATCH_CACHE_LOCKED_KEY,
+      tenMinutesInSeconds,
+      "locked",
+    );
+
+    const pattern = getModelMatchKeyPrefix() + "*";
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared full model cache with ${keys.length} keys in ${Date.now() - startTime}ms.`,
+      );
+    } else {
+      logger.info(`No keys found for match pattern '${pattern}'`);
+    }
+  } catch (error) {
+    logger.error(`Error clearing full model cache: ${error}`);
+  } finally {
+    await redis?.del(MODEL_MATCH_CACHE_LOCKED_KEY);
+  }
+}

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -8,7 +8,7 @@ import {
   LangfuseNotFoundError,
   UnauthorizedError,
 } from "../../errors";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { AuthHeaderValidVerificationResultIngestion } from "../auth/types";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import {
   getCurrentSpan,
@@ -30,6 +30,7 @@ import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -61,7 +62,7 @@ export type TokenCountDelegate = (p: {
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -73,6 +74,10 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
@@ -94,12 +99,12 @@ type ProcessEventBatchOptions = {
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
- * @param authCheck - AuthHeaderValidVerificationResult
+ * @param authCheck - AuthHeaderValidVerificationResultIngestion
  * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
-  authCheck: AuthHeaderValidVerificationResult,
+  authCheck: AuthHeaderValidVerificationResultIngestion,
   options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
@@ -124,8 +129,10 @@ export const processEventBatch = async (
     "langfuse.project.id",
     authCheck.scope.projectId ?? "",
   );
-  currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
-  currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
+  if (authCheck.scope.orgId)
+    currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
+  if (authCheck.scope.plan)
+    currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
@@ -256,11 +263,39 @@ export const processEventBatch = async (
       const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
       const queue = IngestionQueue.getInstance({ shardingKey });
 
-      const shouldSkipS3List =
-        getClickhouseEntityType(eventData.type) === "observation" &&
+      const isDatasetRunItemEvent =
+        getClickhouseEntityType(eventData.type) === "dataset_run_item";
+      const isObservationEvent =
+        getClickhouseEntityType(eventData.type) === "observation";
+
+      const isOtelOrSkipS3Project =
         authCheck.scope.projectId !== null &&
-        (projectIdsToSkipS3List.includes(authCheck.scope.projectId) ||
-          source === "otel");
+        (source === "otel" ||
+          projectIdsToSkipS3List.includes(authCheck.scope.projectId));
+
+      const shouldSkipS3List =
+        isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
+
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
 
       return queue
         ? queue.add(
@@ -285,7 +320,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -300,7 +335,7 @@ export const processEventBatch = async (
 
 const isAuthorized = (
   event: IngestionEventType,
-  authScope: AuthHeaderValidVerificationResult,
+  authScope: AuthHeaderValidVerificationResultIngestion,
 ): boolean => {
   if (event.type === eventTypes.SDK_LOG) {
     return true;

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/ingestion/types.ts

@@ -224,6 +224,7 @@ export const eventTypes = {
   GENERATION_CREATE: "generation-create",
   GENERATION_UPDATE: "generation-update",
   SDK_LOG: "sdk-log",
+  DATASET_RUN_ITEM_CREATE: "dataset-run-item-create",
   // LEGACY, only required for backwards compatibility
   OBSERVATION_CREATE: "observation-create",
   OBSERVATION_UPDATE: "observation-update",
@@ -343,9 +344,15 @@ export const SdkLogEvent = z.object({
 // As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 
 // Complete schema factory - single source of truth for ALL schemas
-const createAllIngestionSchemas = (
-  environmentSchema: z.ZodDefault<z.ZodString>,
-) => {
+const createAllIngestionSchemas = ({
+  isPublic = true,
+}: {
+  isPublic: boolean;
+}) => {
+  const environmentSchema = isPublic
+    ? PublicEnvironmentName
+    : InternalEnvironmentName;
+
   // Base schemas with environment
   const TraceBody = z.object({
     id: idSchema.nullish(),
@@ -504,6 +511,22 @@ const createAllIngestionSchemas = (
     ]),
   );
 
+  const DatasetRunItemBody = z.object({
+    // Core identifiers
+    id: idSchema.nullish(),
+    traceId: z.string(),
+    observationId: z.string().nullish(),
+    error: z.string().nullish(),
+    // Metadata (optional)
+    createdAt: stringDateTime.nullish(),
+    // Dataset identification
+    datasetId: z.string(),
+    // Run identification
+    runId: z.string(),
+    // Dataset item identification
+    datasetItemId: z.string(),
+  });
+
   // Event schemas
   const base = z.object({
     id: idSchema,
@@ -546,6 +569,17 @@ const createAllIngestionSchemas = (
     body: ScoreBody,
   });
 
+  const baseDatasetRunItemCreateEvent = base.extend({
+    type: z.literal(eventTypes.DATASET_RUN_ITEM_CREATE),
+    body: DatasetRunItemBody,
+  });
+
+  const datasetRunItemCreateEvent = isPublic
+    ? baseDatasetRunItemCreateEvent.refine(() => false, {
+        message: "Dataset run item creation is only allowed for internal usage",
+      })
+    : baseDatasetRunItemCreateEvent;
+
   const sdkLogEvent = base.extend({
     type: z.literal(eventTypes.SDK_LOG),
     body: SdkLogEvent,
@@ -570,6 +604,7 @@ const createAllIngestionSchemas = (
     generationCreateEvent,
     generationUpdateEvent,
     sdkLogEvent,
+    datasetRunItemCreateEvent,
     // LEGACY, only required for backwards compatibility
     legacyObservationCreateEvent,
     legacyObservationUpdateEvent,
@@ -595,6 +630,7 @@ const createAllIngestionSchemas = (
     generationCreateEvent,
     generationUpdateEvent,
     scoreEvent,
+    datasetRunItemCreateEvent,
     sdkLogEvent,
     legacyObservationCreateEvent,
     legacyObservationUpdateEvent,
@@ -604,8 +640,8 @@ const createAllIngestionSchemas = (
 };
 
 // Create both public and internal schema instances
-const publicSchemas = createAllIngestionSchemas(PublicEnvironmentName);
-const internalSchemas = createAllIngestionSchemas(InternalEnvironmentName);
+const publicSchemas = createAllIngestionSchemas({ isPublic: true });
+const internalSchemas = createAllIngestionSchemas({ isPublic: false });
 
 // Export individual schemas for backwards compatibility
 export const TraceBody = publicSchemas.TraceBody;
@@ -632,6 +668,8 @@ export const generationCreateEvent = publicSchemas.generationCreateEvent;
 export const generationUpdateEvent = publicSchemas.generationUpdateEvent;
 export const scoreEvent = publicSchemas.scoreEvent;
 export const sdkLogEvent = publicSchemas.sdkLogEvent;
+export const datasetRunItemCreateEvent =
+  publicSchemas.datasetRunItemCreateEvent;
 export const legacyObservationCreateEvent =
   publicSchemas.legacyObservationCreateEvent;
 export const legacyObservationUpdateEvent =
@@ -649,6 +687,7 @@ export const ingestionEvent = publicSchemas.ingestionEvent;
 export type IngestionEventType = z.infer<typeof ingestionEvent>;
 export type TraceEventType = z.infer<typeof traceEvent>;
 export type ScoreEventType = z.infer<typeof scoreEvent>;
+export type DatasetRunItemEventType = z.infer<typeof datasetRunItemCreateEvent>;
 
 /**
  * Creates an ingestion event schema with appropriate environment validation.

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -42,6 +42,7 @@ import {
 } from "./types";
 import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
+import { HttpsProxyAgent } from "https-proxy-agent";
 
 const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
 
@@ -217,6 +218,10 @@ export async function fetchLLMCompletion(
     (m) => m.content.length > 0 || "tool_calls" in m,
   );
 
+  // Common proxy configuration for all adapters
+  const proxyUrl = env.HTTPS_PROXY;
+  const proxyAgent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined;
+
   let chatModel:
     | ChatOpenAI
     | ChatAnthropic
@@ -232,7 +237,11 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
-      clientOptions: { maxRetries, timeout: 1000 * 60 * 2 }, // 2 minutes timeout
+      clientOptions: {
+        maxRetries,
+        timeout: 1000 * 60 * 2, // 2 minutes timeout
+        ...(proxyAgent && { httpAgent: proxyAgent }),
+      },
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
@@ -247,6 +256,7 @@ export async function fetchLLMCompletion(
       configuration: {
         baseURL,
         defaultHeaders: extraHeaders,
+        ...(proxyAgent && { httpAgent: proxyAgent }),
       },
       timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
@@ -264,6 +274,7 @@ export async function fetchLLMCompletion(
       timeout: 1000 * 60 * 2, // 2 minutes timeout
       configuration: {
         defaultHeaders: extraHeaders,
+        ...(proxyAgent && { httpAgent: proxyAgent }),
       },
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
@@ -316,22 +327,6 @@ export async function fetchLLMCompletion(
       maxRetries,
       apiKey,
     });
-  } else if (modelParams.adapter === LLMAdapter.Atla) {
-    // Atla models do not support:
-    // - temperature
-    // - max_tokens
-    // - top_p
-    chatModel = new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      callbacks: finalCallbacks,
-      maxRetries,
-      configuration: {
-        baseURL: baseURL,
-        defaultHeaders: extraHeaders,
-      },
-      timeout: 1000 * 60, // 1 minute timeout
-    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;
@@ -392,6 +387,7 @@ export async function fetchLLMCompletion(
           },
           configuration: {
             baseURL,
+            ...(proxyAgent && { httpAgent: proxyAgent }),
           },
           timeout: 1000 * 60 * 2, // 2 minutes timeout
         })

packages/shared/src/server/llm/testModelCall.ts

@@ -0,0 +1,52 @@
+import { z as zodV3 } from "zod/v3";
+import {
+  ChatMessageRole,
+  ChatMessageType,
+  LLMApiKeySchema,
+  type ModelConfig,
+} from "./types";
+import { decrypt } from "../../encryption";
+import { fetchLLMCompletion } from "./fetchLLMCompletion";
+import { decryptAndParseExtraHeaders } from "./utils";
+import z from "zod/v4";
+
+export const testModelCall = async ({
+  provider,
+  model,
+  apiKey,
+  prompt,
+  modelConfig,
+}: {
+  provider: string;
+  model: string;
+  apiKey: z.infer<typeof LLMApiKeySchema>;
+  prompt?: string;
+  modelConfig?: ModelConfig | null;
+}): Promise<void> => {
+  (
+    await fetchLLMCompletion({
+      streaming: false,
+      apiKey: decrypt(apiKey.secretKey), // decrypt the secret key
+      extraHeaders: decryptAndParseExtraHeaders(apiKey.extraHeaders),
+      baseURL: apiKey.baseURL ?? undefined,
+      messages: [
+        {
+          role: ChatMessageRole.User,
+          content: prompt ?? "mock content",
+          type: ChatMessageType.User,
+        },
+      ],
+      modelParams: {
+        provider: provider,
+        model: model,
+        adapter: apiKey.adapter,
+        ...modelConfig,
+      },
+      structuredOutputSchema: zodV3.object({
+        score: zodV3.string(),
+        reasoning: zodV3.string(),
+      }),
+      config: apiKey.config,
+    })
+  ).completion;
+};

packages/shared/src/server/llm/types.ts

@@ -113,6 +113,7 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
   Tool = "tool",
+  Model = "model", // Google Gemini assistant format
 }
 
 // Thought: should placeholder not semantically be part of this, because it can be
@@ -124,6 +125,7 @@ export enum ChatMessageType {
   AssistantText = "assistant-text",
   AssistantToolCall = "assistant-tool-call",
   ToolResult = "tool-result",
+  ModelText = "model-text",
   PublicAPICreated = "public-api-created",
   Placeholder = "placeholder",
 }
@@ -156,6 +158,13 @@ export const AssistantTextMessageSchema = z.object({
 });
 export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
 
+export const ModelMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ModelText),
+  role: z.literal(ChatMessageRole.Model),
+  content: z.string(),
+});
+export type ModelMessage = z.infer<typeof ModelMessageSchema>;
+
 export const AssistantToolCallMessageSchema = z.object({
   type: z.literal(ChatMessageType.AssistantToolCall),
   role: z.literal(ChatMessageRole.Assistant),
@@ -193,6 +202,7 @@ export const ChatMessageSchema = z.union([
   AssistantTextMessageSchema,
   AssistantToolCallMessageSchema,
   ToolResultMessageSchema,
+  ModelMessageSchema,
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
@@ -226,18 +236,12 @@ export type PromptVariable = { name: string; value: string; isUsed: boolean };
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
-  Atla = "atla",
   Azure = "azure",
   Bedrock = "bedrock",
   VertexAI = "google-vertex-ai",
   GoogleAIStudio = "google-ai-studio",
 }
 
-export const SYSTEM_ROLES: string[] = [
-  ChatMessageRole.System,
-  ChatMessageRole.Developer,
-];
-
 export const TextPromptContentSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
@@ -290,6 +294,12 @@ export const openAIModels = [
   "gpt-4.1-mini-2025-04-14",
   "gpt-4.1-nano",
   "gpt-4.1-nano-2025-04-14",
+  "gpt-5",
+  "gpt-5-2025-08-07",
+  "gpt-5-mini",
+  "gpt-5-mini-2025-08-07",
+  "gpt-5-nano",
+  "gpt-5-nano-2025-08-07",
   "o3",
   "o3-2025-04-16",
   "o4-mini",
@@ -327,6 +337,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const anthropicModels = [
   "claude-sonnet-4-20250514",
+  "claude-opus-4-1-20250805",
   "claude-opus-4-20250514",
   "claude-3-7-sonnet-20250219",
   "claude-3-5-sonnet-20241022",
@@ -373,8 +384,6 @@ export const googleAIStudioModels = [
   "gemini-1.5-flash-8b",
 ] as const;
 
-export const atlaModels = ["atla-selene", "atla-selene-20250214"] as const;
-
 export type AnthropicModel = (typeof anthropicModels)[number];
 export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
@@ -384,7 +393,6 @@ export const supportedModels = {
   [LLMAdapter.GoogleAIStudio]: googleAIStudioModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
-  [LLMAdapter.Atla]: atlaModels,
 } as const;
 
 export type LLMFunctionCall = {

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -6,6 +6,7 @@ export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
   tablePrefix?: string,
+  useTracesAmtCompatMode: boolean = false,
 ) => {
   const prefix = tablePrefix ? `${tablePrefix}.` : "";
 
@@ -13,9 +14,11 @@ export const clickhouseSearchCondition = (
     !searchType || searchType.includes("id")
       ? `${prefix}id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
-    searchType && searchType.includes("content")
+    searchType && searchType.includes("content") && !useTracesAmtCompatMode
       ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
-      : null,
+      : searchType && searchType.includes("content") && useTracesAmtCompatMode
+        ? `finalizeAggregation(${prefix}input) ILIKE {searchString: String} OR finalizeAggregation(${prefix}output) ILIKE {searchString: String}`
+        : null,
   ].filter(Boolean);
 
   return {

packages/shared/src/server/queries/createSessionsAllQuery.ts

@@ -1,88 +0,0 @@
-import { z } from "zod/v4";
-
-import { Prisma } from "@prisma/client";
-import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
-import { singleFilter } from "../../interfaces/filters";
-import { orderBy } from "../../interfaces/orderBy";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { sessionsViewCols } from "../../tableDefinitions";
-
-const GetSessionTableSQLParamsSchema = z.object({
-  projectId: z.string(),
-  filter: z.array(singleFilter).nullable(),
-  orderBy: orderBy,
-  page: z.number(),
-  limit: z.number(),
-});
-type GetSessionTableSQLParams = z.infer<typeof GetSessionTableSQLParamsSchema>;
-
-export const createSessionsAllQuery = (
-  select: Prisma.Sql,
-  params: GetSessionTableSQLParams,
-  options?: {
-    ignoreOrderBy?: boolean; // used by session.metrics and session.all.totalCount
-    sessionIdList?: string[]; // used by session.metrics
-  },
-): Prisma.Sql => {
-  const { projectId, filter, orderBy, page, limit } =
-    GetSessionTableSQLParamsSchema.parse(params);
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filter ?? [],
-    sessionsViewCols,
-    "sessions",
-  );
-  const orderByCondition = orderByToPrismaSql(orderBy, sessionsViewCols);
-
-  const sessionIdFilter = options?.sessionIdList
-    ? Prisma.sql`AND s.id IN (${Prisma.join(options?.sessionIdList)})`
-    : Prisma.sql``;
-
-  const sql = Prisma.sql`
-      SELECT
-        ${select}
-      FROM
-        trace_sessions AS s
-        LEFT JOIN LATERAL (
-          SELECT
-            t.session_id,
-            MAX(t. "timestamp") AS "max_timestamp",
-            MIN(t. "timestamp") AS "min_timestamp",
-            array_agg(t.id) AS "traceIds",
-            array_agg(DISTINCT t.user_id) AS "userIds",
-            count(t.id)::int AS "countTraces",
-            array_agg(DISTINCT u.tag) AS "tags"
-          FROM
-            traces t
-          LEFT JOIN LATERAL (
-            SELECT DISTINCT UNNEST(t.tags) AS tag) AS u ON TRUE
-          WHERE
-            t.project_id = ${projectId}
-            AND t.session_id = s.id
-          GROUP BY
-            t.session_id) AS t ON TRUE
-        LEFT JOIN LATERAL (
-          SELECT
-            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
-            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
-            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
-            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
-            SUM(o.prompt_tokens) AS "promptTokens",
-            SUM(o.completion_tokens) AS "completionTokens",
-            SUM(o.total_tokens) AS "totalTokens"
-          FROM
-            observations_view o
-          WHERE
-            o.project_id = ${projectId}
-            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
-      WHERE
-        s. "project_id" = ${projectId}
-        ${filterCondition}
-        ${sessionIdFilter}
-      ${options?.ignoreOrderBy ? Prisma.sql`` : orderByCondition}
-      LIMIT ${limit}
-      OFFSET ${page * limit}
-    `;
-
-  return sql;
-};

packages/shared/src/server/queries/index.ts

@@ -1,4 +1,3 @@
-export { createSessionsAllQuery } from "./createSessionsAllQuery";
 export {
   type FullObservations,
   type FullObservationsWithScores,

packages/shared/src/server/queues.ts

@@ -40,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -101,6 +116,24 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
+    z.object({
+      actionId: z.literal("observation-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -144,6 +177,7 @@ export const WebhookInputSchema = z.object({
   payload: WebhookOutboundEnvelopeSchema,
 });
 
+export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   z.object({
     entityType: z.literal("prompt-version"),
@@ -154,8 +188,6 @@ export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   }),
   // Add other entity types here in the future
 ]);
-
-export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export type EntityChangeEventType = z.infer<typeof EntityChangeEventSchema>;
 
 export type CreateEvalQueueEventType = z.infer<
@@ -165,6 +197,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -192,6 +225,13 @@ export type DeadLetterRetryQueueEventType = z.infer<
 
 export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
 
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -214,6 +254,7 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
   WebhookQueue = "webhook-queue",
   EntityChangeQueue = "entity-change-queue",
@@ -241,6 +282,7 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
   WebhookJob = "webhook-job",
   EntityChangeJob = "entity-change-job",
@@ -265,6 +307,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -282,6 +330,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -306,6 +355,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -34,7 +34,7 @@ export class ExperimentCreateQueue {
               attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 1000,
+                delay: 10_000, // 10 seconds
               },
             },
           },

packages/shared/src/server/redis/getQueue.ts

@@ -6,7 +6,6 @@ import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
 import { SecondaryIngestionQueue } from "./ingestionQueue";
-import { TraceUpsertQueue } from "./traceUpsert";
 import { TraceDeleteQueue } from "./traceDelete";
 import { ProjectDeleteQueue } from "./projectDelete";
 import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
@@ -23,11 +22,15 @@ import { ScoreDeleteQueue } from "./scoreDelete";
 import { DeadLetterRetryQueue } from "./dlqRetryQueue";
 import { WebhookQueue } from "./webhookQueue";
 import { EntityChangeQueue } from "./entityChangeQueue";
+import { DatasetDeleteQueue } from "./datasetDelete";
 
-// IngestionQueue is sharded and requires a sharding key
-// Use IngestionQueue.getInstance({ shardName: queueName }) directly instead
+// IngestionQueue and TraceUpsert are sharded and require a sharding key
+// Use IngestionQueue.getInstance({ shardName: queueName }) or TraceUpsertQueue.getInstance({ shardName: queueName }) directly instead
 export function getQueue(
-  queueName: Exclude<QueueName, QueueName.IngestionQueue>,
+  queueName: Exclude<
+    QueueName,
+    QueueName.IngestionQueue | QueueName.TraceUpsert
+  >,
 ): Queue | null {
   switch (queueName) {
     case QueueName.BatchExport:
@@ -36,12 +39,12 @@ export function getQueue(
       return CloudUsageMeteringQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.DatasetDelete:
+      return DatasetDeleteQueue.getInstance();
     case QueueName.EvaluationExecution:
       return EvalExecutionQueue.getInstance();
     case QueueName.ExperimentCreate:
       return ExperimentCreateQueue.getInstance();
-    case QueueName.TraceUpsert:
-      return TraceUpsertQueue.getInstance();
     case QueueName.TraceDelete:
       return TraceDeleteQueue.getInstance();
     case QueueName.ProjectDelete:

packages/shared/src/server/redis/redis.ts

@@ -6,6 +6,7 @@ import { logger } from "../logger";
 const defaultRedisOptions: Partial<RedisOptions> = {
   maxRetriesPerRequest: null,
   enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
+  keyPrefix: env.REDIS_KEY_PREFIX ?? undefined,
 };
 
 export const redisQueueRetryOptions: Partial<RedisOptions> = {
@@ -76,6 +77,7 @@ const createRedisClusterInstance = (
       callback(null, address);
     },
     redisOptions: {
+      username: env.REDIS_USERNAME || undefined,
       password: env.REDIS_AUTH || undefined,
       ...defaultRedisOptions,
       ...additionalOptions,
@@ -128,6 +130,7 @@ export const createNewRedisInstance = (
       ? new Redis({
           host: String(env.REDIS_HOST),
           port: Number(env.REDIS_PORT),
+          username: env.REDIS_USERNAME || undefined,
           password: String(env.REDIS_AUTH),
           ...defaultRedisOptions,
           ...additionalOptions,

packages/shared/src/server/redis/traceUpsert.ts

@@ -6,45 +6,92 @@ import {
   getQueuePrefix,
 } from "./redis";
 import { logger } from "../logger";
+import { getShardIndex } from "./sharding";
+import { env } from "../../env";
 
 export class TraceUpsertQueue {
-  private static instance: Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null =
-    null;
+  private static instances: Map<
+    number,
+    Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null
+  > = new Map();
 
-  public static getInstance(): Queue<
-    TQueueJobTypes[QueueName.TraceUpsert]
-  > | null {
-    if (TraceUpsertQueue.instance) return TraceUpsertQueue.instance;
+  public static getShardNames() {
+    return Array.from(
+      { length: env.LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT },
+      (_, i) => `${QueueName.TraceUpsert}${i > 0 ? `-${i}` : ""}`,
+    );
+  }
+
+  static getShardIndexFromShardName(
+    shardName: string | undefined,
+  ): number | null {
+    if (!shardName) return null;
+
+    // Extract shard index from shard name
+    const shardIndex =
+      shardName === QueueName.TraceUpsert
+        ? 0
+        : parseInt(shardName.replace(`${QueueName.TraceUpsert}-`, ""), 10);
+
+    if (isNaN(shardIndex)) return null;
+    return shardIndex;
+  }
+
+  /**
+   * Get the trace upsert queue instance for the given sharding key or shard name.
+   * @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-traceId`.
+   * @param shardName - Name of the shard. Should be `trace-upsert-queue-${shardIndex}` or plainly `trace-upsert-queue` for the first shard.
+   */
+  public static getInstance({
+    shardingKey,
+    shardName,
+  }: {
+    shardingKey?: string;
+    shardName?: string;
+  } = {}): Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null {
+    const shardIndex =
+      TraceUpsertQueue.getShardIndexFromShardName(shardName) ??
+      (env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
+        ? getShardIndex(
+            shardingKey,
+            env.LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT,
+          )
+        : 0);
+
+    // Check if we already have an instance for this shard
+    if (TraceUpsertQueue.instances.has(shardIndex)) {
+      return TraceUpsertQueue.instances.get(shardIndex) || null;
+    }
 
     const newRedis = createNewRedisInstance({
       enableOfflineQueue: false,
       ...redisQueueRetryOptions,
     });
 
-    TraceUpsertQueue.instance = newRedis
-      ? new Queue<TQueueJobTypes[QueueName.TraceUpsert]>(
-          QueueName.TraceUpsert,
-          {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.TraceUpsert),
-            defaultJobOptions: {
-              removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
-              removeOnFail: 100_000,
-              attempts: 5,
-              delay: 15_000, // 15 seconds
-              backoff: {
-                type: "exponential",
-                delay: 5000,
-              },
+    const name = `${QueueName.TraceUpsert}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
+    const queueInstance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.TraceUpsert]>(name, {
+          connection: newRedis,
+          prefix: getQueuePrefix(name),
+          defaultJobOptions: {
+            removeOnComplete: 100,
+            removeOnFail: 100_000,
+            attempts: 5,
+            delay: 15_000, // 15 seconds
+            backoff: {
+              type: "exponential",
+              delay: 5000,
             },
           },
-        )
+        })
       : null;
 
-    TraceUpsertQueue.instance?.on("error", (err) => {
-      logger.error("TraceUpsertQueue error", err);
+    queueInstance?.on("error", (err) => {
+      logger.error(`TraceUpsertQueue shard ${shardIndex} error`, err);
     });
 
-    return TraceUpsertQueue.instance;
+    TraceUpsertQueue.instances.set(shardIndex, queueInstance);
+
+    return queueInstance;
   }
 }

packages/shared/src/server/redis/webhookQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
 import { logger } from "../logger";
 
 export class WebhookQueue {
@@ -23,6 +27,7 @@ export class WebhookQueue {
           QueueName.WebhookQueue,
           {
             connection: newRedis,
+            prefix: getQueuePrefix(QueueName.WebhookQueue),
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,

packages/shared/src/server/repositories/automation-repository.ts

@@ -2,17 +2,22 @@ import {
   Action,
   ActionExecutionStatus,
   JobConfigState,
+  Prisma,
   prisma,
   Trigger,
 } from "../../db";
 import {
   TriggerEventSource,
-  WebhookActionConfigWithSecrets,
   TriggerDomain,
   TriggerEventAction,
   ActionDomain,
   AutomationDomain,
-  SafeWebhookActionConfig,
+  ActionDomainWithSecrets,
+  SafeActionConfig,
+  isWebhookActionConfig,
+  WebhookActionConfigWithSecrets,
+  isSafeWebhookActionConfig,
+  convertToSafeWebhookConfig,
 } from "../../domain/automations";
 import { FilterState } from "../../types";
 import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";
@@ -23,7 +28,7 @@ export const getActionByIdWithSecrets = async ({
 }: {
   projectId: string;
   actionId: string;
-}) => {
+}): Promise<ActionDomainWithSecrets | null> => {
   const actionConfig = await prisma.action.findFirst({
     where: {
       id: actionId,
@@ -35,31 +40,41 @@ export const getActionByIdWithSecrets = async ({
     return null;
   }
 
-  const config = actionConfig.config as WebhookActionConfigWithSecrets;
+  if (isWebhookActionConfig(actionConfig.config)) {
+    const config = actionConfig.config; // Type guard ensures this is WebhookActionConfigWithSecrets
 
-  // Decrypt secret headers for webhook execution using new structure
-  const decryptedHeaders = config.requestHeaders
-    ? decryptSecretHeaders(mergeHeaders(config.headers, config.requestHeaders))
-    : Object.entries(config.headers).reduce(
-        (acc, [key, value]) => {
-          acc[key] = { secret: false, value };
-          return acc;
-        },
-        {} as Record<string, { secret: boolean; value: string }>,
-      );
+    // Decrypt secret headers for webhook execution using new structure
+    const decryptedHeaders = config.requestHeaders
+      ? decryptSecretHeaders(
+          mergeHeaders(config.headers, config.requestHeaders),
+        )
+      : config.headers
+        ? Object.entries(config.headers).reduce(
+            (acc, [key, value]) => {
+              acc[key] = { secret: false, value };
+              return acc;
+            },
+            {} as Record<string, { secret: boolean; value: string }>,
+          )
+        : {};
 
-  return {
-    ...actionConfig,
-    config: {
-      type: config.type,
-      url: config.url,
-      requestHeaders: decryptedHeaders,
-      displayHeaders: config.displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-      secretKey: config.secretKey,
-    },
-  };
+    return {
+      ...actionConfig,
+      config: {
+        type: config.type,
+        url: config.url,
+        requestHeaders: decryptedHeaders,
+        displayHeaders: getDisplayHeaders(config),
+        apiVersion: config.apiVersion,
+        displaySecretKey: config.displaySecretKey,
+        secretKey: config.secretKey,
+        lastFailingExecutionId: config.lastFailingExecutionId,
+      },
+    };
+  }
+
+  // For SLACK and others, return as stored (already safe)
+  return actionConfig as ActionDomainWithSecrets;
 };
 
 export const getActionById = async ({
@@ -128,10 +143,7 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {
   };
 };
 
-const convertActionToDomain = (action: Action): ActionDomain => {
-  const config = action.config as WebhookActionConfigWithSecrets;
-
-  // Handle legacy headers - convert them to displayHeaders format if displayHeaders is undefined
+const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {
   let displayHeaders = config.displayHeaders;
   if (!displayHeaders && config.headers) {
     // Convert legacy headers to displayHeaders format
@@ -143,17 +155,25 @@ const convertActionToDomain = (action: Action): ActionDomain => {
       {} as Record<string, { secret: boolean; value: string }>,
     );
   }
+  return displayHeaders;
+};
 
+const convertActionToDomain = (action: Action): ActionDomain => {
+  if (isWebhookActionConfig(action.config)) {
+    const config = action.config;
+    config.displayHeaders = getDisplayHeaders(config);
+
+    return {
+      ...action,
+      config: convertToSafeWebhookConfig(config),
+    };
+  }
+
+  // For SLACK (or future types) return config as-is
   return {
     ...action,
-    config: {
-      type: config.type,
-      url: config.url,
-      displayHeaders,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-    } as SafeWebhookActionConfig,
-  };
+    config: action.config as SafeActionConfig,
+  } as ActionDomain;
 };
 
 export const getAutomationById = async ({
@@ -225,28 +245,49 @@ export const getConsecutiveAutomationFailures = async ({
   automationId: string;
   projectId: string;
 }): Promise<number> => {
-  // First get the automation to extract triggerId and actionId
-  const automation = await prisma.automation.findFirst({
-    where: {
-      id: automationId,
-      projectId,
-    },
+  const automation = await getAutomationById({
+    automationId,
+    projectId,
   });
 
   if (!automation) {
     return 0;
   }
 
-  const { triggerId, actionId } = automation;
-  const executions = await prisma.automationExecution.findMany({
-    where: {
-      triggerId,
-      actionId,
-      projectId,
-      status: {
-        in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
-      },
+  // Build where clause - if lastFailingExecutionId is set, only consider executions newer than it
+  const whereClause: Prisma.AutomationExecutionWhereInput = {
+    triggerId: automation.trigger.id,
+    actionId: automation.action.id,
+    projectId,
+    status: {
+      in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
     },
+  };
+
+  // If there's a lastFailingExecutionId, we need to get executions that are newer than that execution
+  if (
+    isSafeWebhookActionConfig(automation.action.config) &&
+    automation.action.config.lastFailingExecutionId
+  ) {
+    // First get the timestamp of the last failing execution
+    const lastFailingExecution = await prisma.automationExecution.findUnique({
+      where: {
+        id: automation.action.config.lastFailingExecutionId,
+      },
+      select: {
+        createdAt: true,
+      },
+    });
+
+    if (lastFailingExecution) {
+      whereClause.createdAt = {
+        gt: lastFailingExecution.createdAt,
+      };
+    }
+  }
+
+  const executions = await prisma.automationExecution.findMany({
+    where: whereClause,
     orderBy: {
       createdAt: "desc",
     },

packages/shared/src/server/repositories/clickhouse.ts

@@ -36,7 +36,7 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces" | "observations";
+  table: "scores" | "traces" | "observations" | "traces_null";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>; // eslint-disable-line no-unused-vars
   tags?: Record<string, string>;

packages/shared/src/server/repositories/dataset-run-items-converters.ts

@@ -0,0 +1,86 @@
+import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
+import { DatasetRunItemRecordReadType } from "./definitions";
+
+export const convertToDatasetRunMetrics = (row: any) => {
+  return {
+    id: row.dataset_run_id,
+    projectId: row.project_id,
+    createdAt: new Date(row.created_at),
+    updatedAt: new Date(row.updated_at),
+    name: row.datasetRunName,
+    description: row.datasetRunDescription ?? "",
+    metadata: row.datasetRunMetadata,
+    countRunItems: row.count_run_items,
+    avgTotalCost: undefined,
+    avgLatency: undefined,
+    scores: undefined,
+    datasetId: row.dataset_id,
+  };
+};
+
+export const convertDatasetRunItemDomainToClickhouse = (
+  datasetRunItem: DatasetRunItemDomain,
+): DatasetRunItemRecordReadType => {
+  return {
+    id: datasetRunItem.id,
+    project_id: datasetRunItem.projectId,
+    trace_id: datasetRunItem.traceId,
+    observation_id: datasetRunItem.observationId,
+    dataset_id: datasetRunItem.datasetId,
+    dataset_run_id: datasetRunItem.datasetRunId,
+    dataset_run_name: datasetRunItem.datasetRunName,
+    dataset_run_description: datasetRunItem.datasetRunDescription,
+    dataset_run_metadata: datasetRunItem.datasetRunMetadata as Record<
+      string,
+      string
+    >,
+    dataset_item_id: datasetRunItem.datasetItemId,
+    dataset_item_input: datasetRunItem.datasetItemInput as string,
+    dataset_item_expected_output:
+      datasetRunItem.datasetItemExpectedOutput as string,
+    dataset_item_metadata: datasetRunItem.datasetItemMetadata as Record<
+      string,
+      string
+    >,
+    created_at: convertDateToClickhouseDateTime(datasetRunItem.createdAt),
+    updated_at: convertDateToClickhouseDateTime(datasetRunItem.updatedAt),
+    event_ts: convertDateToClickhouseDateTime(new Date()),
+    is_deleted: 0,
+    dataset_run_created_at: convertDateToClickhouseDateTime(
+      datasetRunItem.datasetRunCreatedAt,
+    ),
+    error: datasetRunItem.error,
+  };
+};
+
+export const convertDatasetRunItemClickhouseToDomain = (
+  row: DatasetRunItemRecordReadType,
+): DatasetRunItemDomain => {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    traceId: row.trace_id,
+    observationId: row.observation_id ?? null,
+    datasetRunId: row.dataset_run_id,
+    datasetRunName: row.dataset_run_name,
+    datasetRunDescription: row.dataset_run_description ?? null,
+    datasetRunCreatedAt: parseClickhouseUTCDateTimeFormat(
+      row.dataset_run_created_at,
+    ),
+    datasetRunMetadata:
+      parseMetadataCHRecordToDomain(row.dataset_run_metadata) ?? null,
+    datasetItemId: row.dataset_item_id,
+    datasetItemInput: row.dataset_item_input,
+    datasetItemExpectedOutput: row.dataset_item_expected_output,
+    datasetItemMetadata: parseMetadataCHRecordToDomain(
+      row.dataset_item_metadata,
+    ),
+    createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
+    updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
+    datasetId: row.dataset_id,
+    error: row.error ?? null,
+  };
+};

packages/shared/src/server/repositories/dataset-run-items.ts

@@ -0,0 +1,458 @@
+import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
+import { type OrderByState } from "../../interfaces/orderBy";
+import { datasetRunItemsTableUiColumnDefinitions } from "../../tableDefinitions";
+import { FilterState } from "../../types";
+import {
+  createFilterFromFilterState,
+  FilterList,
+  orderByToClickhouseSql,
+  StringFilter,
+} from "../queries";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "./clickhouse";
+import { convertDatasetRunItemClickhouseToDomain } from "./dataset-run-items-converters";
+import { DatasetRunItemRecordReadType } from "./definitions";
+import { env } from "../../env";
+import { commandClickhouse } from "./clickhouse";
+import Decimal from "decimal.js";
+
+type DatasetRunItemsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  filter: FilterState;
+  orderBy?: OrderByState | OrderByState[];
+  limit?: number;
+  offset?: number;
+};
+
+type DatasetRunsMetricsTableQuery = {
+  projectId: string;
+  datasetId: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+};
+
+export type DatasetRunsMetrics = {
+  id: string;
+  projectId: string;
+  createdAt: Date;
+  datasetId: string;
+  countRunItems: number;
+  avgTotalCost: Decimal;
+  avgLatency: number;
+};
+
+type DatasetRunsMetricsRecordType = {
+  dataset_run_id: string;
+  project_id: string;
+  dataset_run_created_at: string;
+  dataset_id: string;
+  count_run_items: number;
+  avg_latency_seconds: number;
+  avg_total_cost: number;
+};
+
+const convertDatasetRunsMetricsRecord = (
+  record: DatasetRunsMetricsRecordType,
+): DatasetRunsMetrics => {
+  return {
+    id: record.dataset_run_id,
+    projectId: record.project_id,
+    createdAt: parseClickhouseUTCDateTimeFormat(record.dataset_run_created_at),
+    datasetId: record.dataset_id,
+    countRunItems: record.count_run_items,
+    avgTotalCost: record.avg_total_cost
+      ? new Decimal(record.avg_total_cost)
+      : new Decimal(0),
+    avgLatency: record.avg_latency_seconds ?? 0,
+  };
+};
+
+const getProjectDatasetIdDefaultFilter = (
+  projectId: string,
+  datasetId: string,
+) => {
+  return {
+    datasetRunItemsFilter: new FilterList([
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "project_id",
+        operator: "=",
+        value: projectId,
+      }),
+      new StringFilter({
+        clickhouseTable: "dataset_run_items",
+        field: "dataset_id",
+        operator: "=",
+        value: datasetId,
+      }),
+    ]),
+  };
+};
+
+const getDatasetRunsTableInternal = async <T>(
+  opts: DatasetRunsMetricsTableQuery & {
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, orderBy, limit, offset } = opts;
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    orderByArray.push(orderBy);
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    WITH observations_filtered AS (
+      SELECT
+        o.id,
+        o.trace_id,
+        o.project_id,
+        o.start_time,
+        o.end_time,
+        o.total_cost
+      FROM observations o FINAL
+      WHERE o.project_id = {projectId: String}
+        AND o.start_time >= (
+          SELECT min(dri.dataset_run_created_at) - INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+        AND o.start_time <= (
+          SELECT max(dri.dataset_run_created_at) + INTERVAL 1 DAY 
+          FROM dataset_run_items dri 
+          WHERE dri.project_id = {projectId: String} 
+            AND dri.dataset_id = {datasetId: String}
+        )
+    ),
+    traces_aggregated AS (
+      SELECT
+        of.trace_id,
+        of.project_id,
+        dateDiff('millisecond', min(of.start_time), max(of.end_time)) as latency_ms,
+        sum(of.total_cost) as total_cost
+      FROM observations_filtered of
+      JOIN dataset_run_items dri ON dri.trace_id = of.trace_id 
+        AND dri.project_id = of.project_id
+        AND dri.observation_id IS NULL  -- Only for trace-level dataset run items
+      WHERE dri.dataset_id = {datasetId: String}
+      GROUP BY of.trace_id, of.project_id
+    ),
+    observations_direct AS (
+      SELECT
+        dri.observation_id,
+        dri.project_id,
+        dri.trace_id,
+        of.total_cost,
+        dateDiff('millisecond', of.start_time, of.end_time) as latency_ms
+      FROM dataset_run_items dri
+      JOIN observations_filtered of ON dri.observation_id = of.id
+        AND dri.project_id = of.project_id
+        AND dri.trace_id = of.trace_id
+      WHERE dri.dataset_id = {datasetId: String}
+        AND dri.observation_id IS NOT NULL  -- Only for observation-level dataset run items
+    )
+    SELECT DISTINCT
+      dri.dataset_run_id as dataset_run_id,
+      dri.project_id as project_id,
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count_run_items,
+      
+      -- Latency metrics (priority: observation > trace)
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.latency_ms IS NOT NULL
+        THEN od.latency_ms / 1000.0
+        ELSE COALESCE(ta.latency_ms / 1000.0, 0)
+      END) as avg_latency_seconds,
+      
+      -- Cost metrics (priority: observation > trace)  
+      AVG(CASE
+        WHEN dri.observation_id IS NOT NULL AND od.total_cost IS NOT NULL
+        THEN od.total_cost
+        ELSE COALESCE(ta.total_cost, 0)
+      END) as avg_total_cost
+    FROM dataset_run_items dri 
+    LEFT JOIN traces_aggregated ta
+      ON dri.trace_id = ta.trace_id
+      AND dri.project_id = ta.project_id
+    LEFT JOIN observations_direct od
+      ON dri.observation_id = od.observation_id
+      AND dri.project_id = od.project_id
+      AND dri.trace_id = od.trace_id
+    WHERE ${appliedFilter.query}
+    GROUP BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_run_created_at
+    ORDER BY dri.dataset_run_created_at DESC
+    ${orderByClause}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      projectId,
+      datasetId,
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunsTableMetricsCh = async (
+  opts: DatasetRunsMetricsTableQuery,
+): Promise<DatasetRunsMetrics[]> => {
+  // First get the metrics (latency, cost, counts)
+  const rows = await getDatasetRunsTableInternal<DatasetRunsMetricsRecordType>({
+    ...opts,
+    tags: { kind: "list" },
+  });
+
+  return rows.map(convertDatasetRunsMetricsRecord);
+};
+
+const getDatasetRunItemsTableInternal = async <T>(
+  opts: DatasetRunItemsTableQuery & {
+    select: "count" | "rows";
+    tags: Record<string, string>;
+  },
+): Promise<Array<T>> => {
+  const { projectId, datasetId, filter, orderBy, limit, offset } = opts;
+
+  let selectString = "";
+
+  switch (opts.select) {
+    case "count":
+      selectString =
+        "count(DISTINCT dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id) as count";
+      break;
+    case "rows":
+      selectString = `
+      dri.id as id, 
+      dri.project_id as project_id, 
+      dri.trace_id as trace_id, 
+      dri.observation_id as observation_id, 
+      dri.dataset_id as dataset_id,
+      dri.dataset_run_id as dataset_run_id, 
+      dri.dataset_item_id as dataset_item_id, 
+      dri.error as error,
+      dri.created_at as created_at, 
+      dri.updated_at as updated_at,
+      dri.dataset_run_name as dataset_run_name,
+      dri.dataset_run_description as dataset_run_description,
+      dri.dataset_run_metadata as dataset_run_metadata,
+      dri.dataset_run_created_at as dataset_run_created_at,
+      dri.dataset_item_input as dataset_item_input,
+      dri.dataset_item_expected_output as dataset_item_expected_output,
+      dri.dataset_item_metadata as dataset_item_metadata,
+      dri.is_deleted as is_deleted,
+      dri.event_ts as event_ts`;
+      break;
+    default:
+      throw new Error(`Unknown select type: ${opts.select}`);
+  }
+
+  const { datasetRunItemsFilter } = getProjectDatasetIdDefaultFilter(
+    projectId,
+    datasetId,
+  );
+
+  datasetRunItemsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      datasetRunItemsTableUiColumnDefinitions,
+    ),
+  );
+  const appliedFilter = datasetRunItemsFilter.apply();
+
+  // Build ORDER BY array - conditionally add event_ts DESC for rows
+  const orderByArray: OrderByState[] = [];
+
+  // Add user ordering if provided
+  if (orderBy) {
+    if (Array.isArray(orderBy)) {
+      orderByArray.push(...orderBy);
+    } else {
+      orderByArray.push(orderBy);
+    }
+  }
+
+  // Add event_ts DESC for row queries (for deduplication)
+  if (opts.select === "rows") {
+    orderByArray.push({
+      column: "eventTs",
+      order: "DESC",
+    });
+  }
+
+  const orderByClause = orderByToClickhouseSql(
+    orderByArray,
+    datasetRunItemsTableUiColumnDefinitions,
+  );
+
+  const query = `
+    SELECT
+      ${selectString}
+    FROM dataset_run_items dri 
+    WHERE ${appliedFilter.query}
+    ${orderByClause}
+    ${opts.select === "rows" ? "LIMIT 1 BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.dataset_item_id" : ""}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedFilter.params,
+    },
+    tags: {
+      ...(opts.tags ?? {}),
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+export const getDatasetRunItemsByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<DatasetRunItemDomain[]> => {
+  const rows =
+    await getDatasetRunItemsTableInternal<DatasetRunItemRecordReadType>({
+      ...opts,
+      select: "rows",
+      tags: { kind: "list" },
+    });
+
+  return rows.map(convertDatasetRunItemClickhouseToDomain);
+};
+
+export const getDatasetRunItemsCountByDatasetIdCh = async (
+  opts: DatasetRunItemsTableQuery,
+): Promise<number> => {
+  const rows = await getDatasetRunItemsTableInternal<{ count: string }>({
+    ...opts,
+    select: "count",
+    tags: { kind: "list" },
+  });
+
+  return Number(rows[0]?.count);
+};
+
+export const deleteDatasetRunItemsByProjectId = async ({
+  projectId,
+}: {
+  projectId: string;
+}) => {
+  const query = `
+      DELETE FROM dataset_run_items
+      WHERE project_id = {projectId: String};
+    `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetId = async ({
+  projectId,
+  datasetId,
+}: {
+  projectId: string;
+  datasetId: string;
+}) => {
+  const query = `
+  DELETE FROM dataset_run_items
+  WHERE project_id = {projectId: String}
+  AND dataset_id = {datasetId: String}
+`;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetRunIds = async ({
+  projectId,
+  datasetRunIds,
+  datasetId,
+}: {
+  projectId: string;
+  datasetRunIds: string[];
+  datasetId: string;
+}) => {
+  const query = `
+    DELETE FROM dataset_run_items
+    WHERE project_id = {projectId: String}
+    AND dataset_id = {datasetId: String}
+    AND dataset_run_id IN ({datasetRunIds: Array(String)})
+  `;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};

packages/shared/src/server/repositories/definitions.ts

@@ -122,7 +122,7 @@ export const traceRecordInsertSchema = traceRecordBaseSchema.extend({
 });
 export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;
 
-export const traceMtRecordInsertSchema = z.object({
+export const traceNullRecordInsertSchema = z.object({
   // Identifiers
   project_id: z.string(),
   id: z.string(),
@@ -157,7 +157,9 @@ export const traceMtRecordInsertSchema = z.object({
   updated_at: z.number(),
   event_ts: z.number(),
 });
-export type TraceMtRecordInsertType = z.infer<typeof traceMtRecordInsertSchema>;
+export type TraceNullRecordInsertType = z.infer<
+  typeof traceNullRecordInsertSchema
+>;
 
 export const scoreRecordBaseSchema = z.object({
   id: z.string(),
@@ -196,6 +198,45 @@ export const scoreRecordInsertSchema = scoreRecordBaseSchema.extend({
 });
 export type ScoreRecordInsertType = z.infer<typeof scoreRecordInsertSchema>;
 
+const datasetRunItemRecordBaseSchema = z.object({
+  id: z.string(),
+  project_id: z.string(),
+  trace_id: z.string(),
+  observation_id: z.string().nullish(),
+  dataset_id: z.string(),
+  dataset_run_id: z.string(),
+  dataset_item_id: z.string(),
+  dataset_run_name: z.string(),
+  dataset_run_description: z.string().nullish(),
+  dataset_run_metadata: z.record(z.string(), z.string()),
+  dataset_item_input: z.string(),
+  dataset_item_expected_output: z.string(),
+  dataset_item_metadata: z.record(z.string(), z.string()),
+  is_deleted: z.number(),
+  error: z.string().nullish(),
+});
+
+const datasetRunItemRecordReadSchema = datasetRunItemRecordBaseSchema.extend({
+  dataset_run_created_at: clickhouseStringDateSchema,
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+  event_ts: clickhouseStringDateSchema,
+});
+export type DatasetRunItemRecordReadType = z.infer<
+  typeof datasetRunItemRecordReadSchema
+>;
+
+export const datasetRunItemRecordInsertSchema =
+  datasetRunItemRecordBaseSchema.extend({
+    created_at: z.number(),
+    updated_at: z.number(),
+    event_ts: z.number(),
+    dataset_run_created_at: z.number(),
+  });
+export type DatasetRunItemRecordInsertType = z.infer<
+  typeof datasetRunItemRecordInsertSchema
+>;
+
 export const blobStorageFileLogRecordBaseSchema = z.object({
   id: z.string(),
   project_id: z.string(),
@@ -308,6 +349,50 @@ export const convertPostgresTraceToInsert = (
   };
 };
 
+export const convertPostgresDatasetRunItemToInsert = (
+  datasetRunItem: Record<string, any>,
+): DatasetRunItemRecordInsertType => {
+  return {
+    id: datasetRunItem.id,
+    project_id: datasetRunItem.project_id,
+    dataset_run_id: datasetRunItem.dataset_run_id,
+    dataset_item_id: datasetRunItem.dataset_item_id,
+    dataset_id: datasetRunItem.dataset_id,
+    trace_id: datasetRunItem.trace_id,
+    observation_id: datasetRunItem.observation_id,
+    error: datasetRunItem.error,
+    created_at: datasetRunItem.created_at?.getTime(),
+    updated_at: datasetRunItem.updated_at?.getTime(),
+    // denormalized run data
+    dataset_run_name: datasetRunItem.dataset_run_name,
+    dataset_run_description: datasetRunItem.dataset_run_description,
+    dataset_run_metadata:
+      typeof datasetRunItem.dataset_run_metadata === "string" ||
+      typeof datasetRunItem.dataset_run_metadata === "number" ||
+      typeof datasetRunItem.dataset_run_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_run_metadata }
+        : Array.isArray(datasetRunItem.dataset_run_metadata)
+          ? { metadata: datasetRunItem.dataset_run_metadata }
+          : (datasetRunItem.dataset_run_metadata ?? {}),
+    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
+    // denormalized item data
+    dataset_item_input: JSON.stringify(datasetRunItem.dataset_item_input),
+    dataset_item_expected_output: JSON.stringify(
+      datasetRunItem.dataset_item_expected_output,
+    ),
+    dataset_item_metadata:
+      typeof datasetRunItem.dataset_item_metadata === "string" ||
+      typeof datasetRunItem.dataset_item_metadata === "number" ||
+      typeof datasetRunItem.dataset_item_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_item_metadata }
+        : Array.isArray(datasetRunItem.dataset_item_metadata)
+          ? { metadata: datasetRunItem.dataset_item_metadata }
+          : (datasetRunItem.dataset_item_metadata ?? {}),
+    event_ts: datasetRunItem.created_at?.getTime(),
+    is_deleted: 0,
+  };
+};
+
 /**
  * Expects a single record from a
  * `select o.*,
@@ -412,9 +497,9 @@ export const convertPostgresScoreToInsert = (
   };
 };
 
-export const convertTraceToTraceMt = (
+export const convertTraceToTraceNull = (
   traceRecord: TraceRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: traceRecord.project_id,
@@ -452,13 +537,13 @@ export const convertTraceToTraceMt = (
   };
 };
 
-export const convertObservationToTraceMt = (
+export const convertObservationToTraceNull = (
   observationRecord: ObservationRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: observationRecord.project_id,
-    // Use trace_id as the id in traces_mt. Always set given the conditions around calling the function
+    // Use trace_id as the id in traces_null. Always set given the conditions around calling the function
     id: observationRecord.trace_id || "",
     start_time: observationRecord.start_time,
     end_time: observationRecord.end_time || null,
@@ -496,13 +581,13 @@ export const convertObservationToTraceMt = (
   };
 };
 
-export const convertScoreToTraceMt = (
+export const convertScoreToTraceNull = (
   scoreRecord: ScoreRecordInsertType,
-): TraceMtRecordInsertType => {
+): TraceNullRecordInsertType => {
   return {
     // Identifiers
     project_id: scoreRecord.project_id,
-    // Use trace_id as the id in traces_mt. Always set given the conditions around calling the function
+    // Use trace_id as the id in traces_null. Always set given the conditions around calling the function
     id: scoreRecord.trace_id || "",
     start_time: scoreRecord.timestamp,
     end_time: null, // scores don't have end_time

packages/shared/src/server/repositories/index.ts

@@ -13,3 +13,5 @@ export * from "./scores-utils";
 export * from "./blobStorageLog";
 export * from "./environments";
 export * from "./automation-repository";
+export * from "./dataset-run-items-converters";
+export * from "./dataset-run-items";

packages/shared/src/server/repositories/observations.ts

@@ -23,7 +23,8 @@ import {
   observationsTableUiColumnDefinitions,
 } from "../../tableDefinitions";
 import { OrderByState } from "../../interfaces/orderBy";
-import { getTracesByIds } from "./traces";
+import { getTimeframesTracesAMT, getTracesByIds } from "./traces";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertObservation } from "./observations_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -36,6 +37,7 @@ import { TracingSearchType } from "../../interfaces/search";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { ObservationType } from "../../domain";
 import { recordDistribution } from "../instrumentation";
+import { DEFAULT_RENDERING_PROPS, RenderingProps } from "../utils/rendering";
 
 /**
  * Checks if observation exists in clickhouse.
@@ -288,7 +290,7 @@ export const getObservationForTraceIdByName = async (
     },
   });
 
-  return records.map(convertObservation);
+  return records.map((record) => convertObservation(record));
 };
 
 export const getObservationById = async ({
@@ -298,6 +300,7 @@ export const getObservationById = async ({
   startTime,
   type,
   traceId,
+  renderingProps = DEFAULT_RENDERING_PROPS,
 }: {
   id: string;
   projectId: string;
@@ -305,6 +308,7 @@ export const getObservationById = async ({
   startTime?: Date;
   type?: ObservationType;
   traceId?: string;
+  renderingProps?: RenderingProps;
 }) => {
   const records = await getObservationByIdInternal({
     id,
@@ -313,8 +317,11 @@ export const getObservationById = async ({
     startTime,
     type,
     traceId,
+    renderingProps,
   });
-  const mapped = records.map(convertObservation);
+  const mapped = records.map((record) =>
+    convertObservation(record, renderingProps),
+  );
 
   mapped.forEach((observation) => {
     recordDistribution(
@@ -384,7 +391,7 @@ export const getObservationsById = async (
     query,
     params: { ids, projectId },
   });
-  return records.map(convertObservation);
+  return records.map((record) => convertObservation(record));
 };
 
 const getObservationByIdInternal = async ({
@@ -394,6 +401,7 @@ const getObservationByIdInternal = async ({
   startTime,
   type,
   traceId,
+  renderingProps = DEFAULT_RENDERING_PROPS,
 }: {
   id: string;
   projectId: string;
@@ -401,6 +409,7 @@ const getObservationByIdInternal = async ({
   startTime?: Date;
   type?: ObservationType;
   traceId?: string;
+  renderingProps?: RenderingProps;
 }) => {
   const query = `
   SELECT
@@ -417,7 +426,7 @@ const getObservationByIdInternal = async ({
     level,
     status_message,
     version,
-    ${fetchWithInputOutput ? "input, output," : ""}
+    ${fetchWithInputOutput ? (renderingProps.truncated ? `left(input, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as input, left(output, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as output,` : "input, output,") : ""}
     provided_model_name,
     internal_model_id,
     model_parameters,
@@ -612,12 +621,14 @@ const getObservationsTableInternal = async <T>(
   } = opts;
 
   const selectString = selectIOAndMetadata
-    ? `
-    ${select},
-    ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
-  `
+    ? `${select}, o.input, o.output, o.metadata`
     : select;
 
+  const timeFilter = filter.find(
+    (f) =>
+      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
+  );
+
   const scoresFilter = new FilterList([
     new StringFilter({
       clickhouseTable: "scores",
@@ -627,33 +638,22 @@ const getObservationsTableInternal = async <T>(
     }),
   ]);
 
-  const timeFilter = opts.filter.find(
-    (f) =>
-      f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
-  );
-
-  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
-  const traceTableFilter = opts.filter.filter(
-    (f) =>
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(f.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(f.column),
-  );
-
   const hasScoresFilter = filter.some((f) =>
     f.column.toLowerCase().includes("scores"),
   );
 
-  const orderByTraces = opts.orderBy
-    ? observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableId)
-        .includes(opts.orderBy.column) ||
-      observationsTableTraceUiColumnDefinitions
-        .map((c) => c.uiTableName)
-        .includes(opts.orderBy.column)
+  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
+  const traceTableFilter = filter.filter((f) =>
+    observationsTableTraceUiColumnDefinitions.some(
+      (c) => c.uiTableId === f.column || c.uiTableName === f.column,
+    ),
+  );
+
+  const orderByTraces = orderBy
+    ? observationsTableTraceUiColumnDefinitions.some(
+        (c) =>
+          c.uiTableId === orderBy.column || c.uiTableName === orderBy.column,
+      )
     : undefined;
 
   timeFilter
@@ -759,7 +759,7 @@ const getObservationsTableInternal = async <T>(
       SELECT
        ${selectString}
       FROM observations o 
-        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN __TRACE_TABLE__ t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
         ${hasScoresFilter ? `LEFT JOIN scores_agg AS s ON s.trace_id = o.trace_id and s.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
         
@@ -769,30 +769,52 @@ const getObservationsTableInternal = async <T>(
       ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-  const res = await queryClickhouse<T>({
-    query,
-    params: {
-      ...appliedScoresFilter.params,
-      ...appliedObservationsFilter.params,
-      ...(timeFilter
-        ? {
-            tracesTimestampFilter: convertDateToClickhouseDateTime(
-              timeFilter.value as Date,
-            ),
-          }
-        : {}),
-      ...search.params,
+  return measureAndReturn({
+    operationName: "getObservationsTableInternal",
+    projectId,
+    minStartTime: (timeFilter?.value as Date) || undefined,
+    input: {
+      params: {
+        ...appliedScoresFilter.params,
+        ...appliedObservationsFilter.params,
+        ...(timeFilter
+          ? {
+              tracesTimestampFilter: convertDateToClickhouseDateTime(
+                timeFilter.value as Date,
+              ),
+            }
+          : {}),
+        ...search.params,
+      },
+      tags: {
+        ...(opts.tags ?? {}),
+        feature: "tracing",
+        type: "observation",
+        projectId,
+        kind: opts.select,
+        operation_name: "getObservationsTableInternal",
+      },
     },
-    tags: {
-      ...(opts.tags ?? {}),
-      feature: "tracing",
-      type: "observation",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(
+        (timeFilter?.value as Date) || undefined,
+      );
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };
 
 export const getObservationsGroupedByModel = async (
@@ -1473,6 +1495,9 @@ export const getObservationsForBlobStorageExport = function (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
+    },
   });
 
   return records;
@@ -1483,6 +1508,15 @@ export const getGenerationsForPostHog = async function* (
   minTimestamp: Date,
   maxTimestamp: Date,
 ) {
+  // Determine which trace table to use based on experiment flag
+  const useAMT = env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true";
+  // Subtract 7d from minTimestamp to account for shift in query
+  const traceTable = useAMT
+    ? getTimeframesTracesAMT(
+        new Date(minTimestamp.getTime() - 7 * 24 * 60 * 60 * 1000),
+      )
+    : "traces";
+
   const query = `
     SELECT
       o.name as name,
@@ -1507,7 +1541,7 @@ export const getGenerationsForPostHog = async function* (
       t.tags as trace_tags,
       t.metadata['$posthog_session_id'] as posthog_session_id
     FROM observations o FINAL
-    LEFT JOIN traces t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
+    LEFT JOIN ${traceTable} t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
     WHERE o.project_id = {projectId: String}
     AND t.project_id = {projectId: String}
     AND o.start_time >= {minTimestamp: DateTime64(3)}
@@ -1529,9 +1563,10 @@ export const getGenerationsForPostHog = async function* (
       type: "observation",
       kind: "analytic",
       projectId,
+      experiment_amt: useAMT ? "new" : "original",
     },
     clickhouseConfigs: {
-      request_timeout: 300_000, // 5 minutes
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
       clickhouse_settings: {
         join_algorithm: "grace_hash",
         grace_hash_join_initial_buckets: "32",
@@ -1545,6 +1580,7 @@ export const getGenerationsForPostHog = async function* (
       timestamp: record.start_time,
       langfuse_generation_name: record.name,
       langfuse_trace_name: record.trace_name,
+      langfuse_trace_id: record.trace_id,
       langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
       langfuse_id: record.id,
       langfuse_cost_usd: record.total_cost,

packages/shared/src/server/repositories/observations_converters.ts

@@ -1,15 +1,20 @@
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
-import { parseJsonPrioritised } from "../../utils/json";
 import {
   Observation,
   ObservationLevelType,
   ObservationType,
 } from "../../domain";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import {
+  RenderingProps,
+  DEFAULT_RENDERING_PROPS,
+  applyInputOutputRendering,
+} from "../utils/rendering";
 
 export const convertObservation = (
   record: ObservationRecordReadType,
+  renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,
 ): Observation => {
   const reducedCostDetails = reduceUsageOrCostDetails(record.cost_details);
   const reducedUsageDetails = reduceUsageOrCostDetails(record.usage_details);
@@ -30,10 +35,8 @@ export const convertObservation = (
     level: record.level as ObservationLevelType,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
-    output: record.output
-      ? (parseJsonPrioritised(record.output) ?? null)
-      : null,
+    input: applyInputOutputRendering(record.input, renderingProps),
+    output: applyInputOutputRendering(record.output, renderingProps),
     modelParameters: record.model_parameters
       ? (JSON.parse(record.model_parameters) ?? null)
       : null,

packages/shared/src/server/repositories/scores.ts

@@ -32,6 +32,8 @@ import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { recordDistribution } from "../instrumentation";
 import { prisma } from "../../db";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
+import { getTimeframesTracesAMT } from "./traces";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -284,6 +286,53 @@ export const getScoresForDatasetRuns = async <
   return rows.map(convertToScore);
 };
 
+export const getTraceScoresForDatasetRuns = async (
+  projectId: string,
+  datasetRunIds: string[],
+): Promise<Array<{ dataset_run_id: string } & any>> => {
+  if (datasetRunIds.length === 0) return [];
+
+  const query = `
+    SELECT 
+      s.* EXCEPT (metadata),
+      length(mapKeys(s.metadata)) > 0 AS has_metadata,
+      dri.dataset_run_id as run_id
+    FROM dataset_run_items dri 
+    JOIN scores s FINAL ON dri.trace_id = s.trace_id 
+      AND dri.project_id = s.project_id
+    WHERE dri.project_id = {projectId: String}
+      AND dri.dataset_run_id IN {datasetRunIds: Array(String)}
+      AND s.project_id = {projectId: String}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id, dri.dataset_run_id
+  `;
+
+  const rows = await queryClickhouse<
+    Omit<ScoreRecordReadType, "metadata"> & {
+      has_metadata: 0 | 1;
+      run_id: string;
+    }
+  >({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+    },
+    tags: {
+      feature: "dataset-run-items",
+      type: "trace-scores",
+      kind: "list",
+      projectId,
+    },
+  });
+
+  return rows.map((row) => ({
+    ...convertToScore({ ...row, metadata: {} }),
+    datasetRunId: row.run_id,
+    hasMetadata: !!row.has_metadata,
+  }));
+};
+
 // Used in multiple places, including the public API, hence the non-default exclusion of metadata via excludeMetadata flag
 export const getScoresForTraces = async <
   ExcludeMetadata extends boolean,
@@ -895,31 +944,49 @@ const getScoresUiGeneric = async <T>(props: {
       SELECT 
           ${select}
       FROM scores s final
-      ${performTracesJoin ? "LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
+      ${performTracesJoin ? "LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId: projectId,
-      ...(scoresFilterRes ? scoresFilterRes.params : {}),
-      limit: limit,
-      offset: offset,
+  return measureAndReturn({
+    operationName: "getScoresUiGeneric",
+    projectId,
+    input: {
+      params: {
+        projectId: projectId,
+        ...(scoresFilterRes ? scoresFilterRes.params : {}),
+        limit: limit,
+        offset: offset,
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "score",
+        projectId,
+        select: props.select,
+        operation_name: "getScoresUiGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "score",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces_all_amt"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return rows;
 };
 
 export const getScoreNames = async (
@@ -1086,7 +1153,7 @@ export const getNumericScoreHistogram = async (
   const query = `
     select s.value
     from scores s
-    ${traceFilter ? `LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
+    ${traceFilter ? `LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
     WHERE s.project_id = {projectId: String}
     ${traceFilter ? `AND t.project_id = {projectId: String}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
@@ -1095,18 +1162,45 @@ export const getNumericScoreHistogram = async (
     ${limit !== undefined ? `limit {limit: Int32}` : ""}
   `;
 
-  return queryClickhouse<{ value: number }>({
-    query,
-    params: {
-      projectId,
-      limit,
-      ...(chFilterRes ? chFilterRes.params : {}),
+  // Extract timestamp from filter for AMT table selection
+  const timestampFilter = chFilter.find(
+    (f) => f.clickhouseTable === "traces" && f.field === "timestamp",
+  ) as TimeFilter | undefined;
+  const timestamp = timestampFilter?.value;
+
+  return measureAndReturn({
+    operationName: "getNumericScoreHistogram",
+    projectId,
+    minStartTime: timestamp,
+    input: {
+      params: {
+        projectId,
+        limit,
+        ...(chFilterRes ? chFilterRes.params : {}),
+      },
+      tags: {
+        feature: "tracing",
+        type: "score",
+        kind: "analytic",
+        projectId,
+        operation_name: "getNumericScoreHistogram",
+      },
+      timestamp,
     },
-    tags: {
-      feature: "tracing",
-      type: "score",
-      kind: "analytic",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<{ value: number }>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+      });
+    },
+    newExecution: async (input) => {
+      const traceAmt = getTimeframesTracesAMT(input.timestamp);
+      return queryClickhouse<{ value: number }>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+      });
     },
   });
 };
@@ -1320,6 +1414,9 @@ export const getScoresForBlobStorageExport = function (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
+    },
   });
 
   return records;
@@ -1330,21 +1427,34 @@ export const getScoresForPostHog = async function* (
   minTimestamp: Date,
   maxTimestamp: Date,
 ) {
+  // Determine which trace table to use based on experiment flag
+  const useAMT = env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true";
+  // Subtract 7d from minTimestamp to account for shift in query
+  const traceTable = useAMT
+    ? getTimeframesTracesAMT(
+        new Date(minTimestamp.getTime() - 7 * 24 * 60 * 60 * 1000),
+      )
+    : "traces";
+
   const query = `    SELECT
       s.id as id,
       s.timestamp as timestamp,
       s.name as name,
       s.value as value,
+      s.string_value as string_value,
+      s.data_type as data_type,
       s.comment as comment,
       s.environment as environment,
+      t.id as trace_id,
       t.name as trace_name,
       t.session_id as trace_session_id,
       t.user_id as trace_user_id,
       t.release as trace_release,
       t.tags as trace_tags,
+      s.metadata as metadata,
       t.metadata['$posthog_session_id'] as posthog_session_id
     FROM scores s FINAL
-    LEFT JOIN traces t FINAL ON s.trace_id = t.id AND s.project_id = t.project_id
+    LEFT JOIN ${traceTable} t FINAL ON s.trace_id = t.id AND s.project_id = t.project_id
     WHERE s.project_id = {projectId: String}
     AND t.project_id = {projectId: String}
     AND s.timestamp >= {minTimestamp: DateTime64(3)}
@@ -1365,9 +1475,10 @@ export const getScoresForPostHog = async function* (
       type: "score",
       kind: "analytic",
       projectId,
+      experiment_amt: useAMT ? "new" : "original",
     },
     clickhouseConfigs: {
-      request_timeout: 300_000, // 5 minutes
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
       clickhouse_settings: {
         join_algorithm: "grace_hash",
         grace_hash_join_initial_buckets: "32",
@@ -1382,7 +1493,11 @@ export const getScoresForPostHog = async function* (
       langfuse_score_name: record.name,
       langfuse_score_value: record.value,
       langfuse_score_comment: record.comment,
+      langfuse_score_metadata: record.metadata,
+      langfuse_score_string_value: record.string_value,
+      langfuse_score_data_type: record.data_type,
       langfuse_trace_name: record.trace_name,
+      langfuse_trace_id: record.trace_id,
       langfuse_id: record.id,
       langfuse_session_id: record.trace_session_id,
       langfuse_project_id: projectId,

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,9 +1,13 @@
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import { parseJsonPrioritised } from "../../utils/json";
 import { TraceDomain } from "../../domain";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import {
+  RenderingProps,
+  DEFAULT_RENDERING_PROPS,
+  applyInputOutputRendering,
+} from "../utils/rendering";
 
 export const convertTraceDomainToClickhouse = (
   trace: TraceDomain,
@@ -33,6 +37,7 @@ export const convertTraceDomainToClickhouse = (
 
 export const convertClickhouseToDomain = (
   record: TraceRecordReadType,
+  renderingProps: RenderingProps = DEFAULT_RENDERING_PROPS,
 ): TraceDomain => {
   return {
     id: record.id,
@@ -47,10 +52,8 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
-    output: record.output
-      ? (parseJsonPrioritised(record.output) ?? null)
-      : null,
+    input: applyInputOutputRendering(record.input, renderingProps),
+    output: applyInputOutputRendering(record.output, renderingProps),
     metadata: parseMetadataCHRecordToDomain(record.metadata),
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),

packages/shared/src/server/services/DashboardService/types.ts

@@ -35,6 +35,12 @@ export const HistogramChartConfig = BaseTotalValueChartConfig.extend({
 
 export const PivotTableChartConfig = BaseTotalValueChartConfig.extend({
   type: z.literal("PIVOT_TABLE"),
+  defaultSort: z
+    .object({
+      column: z.string(),
+      order: z.enum(["ASC", "DESC"]),
+    })
+    .optional(),
 });
 
 // Define dimension schema

packages/shared/src/server/services/DefaultEvaluationModelService/DefaultEvalModelService.ts

@@ -1,7 +1,12 @@
 import z from "zod/v4";
 import { prisma } from "../../../db";
-import { LangfuseNotFoundError, QUEUE_ERROR_MESSAGES } from "../../../errors";
+import {
+  ForbiddenError,
+  LangfuseNotFoundError,
+  QUEUE_ERROR_MESSAGES,
+} from "../../../errors";
 import { LLMApiKeySchema, ZodModelConfig } from "../../llm/types";
+import { testModelCall } from "../../llm/testModelCall";
 
 type ValidConfig = {
   provider: string;
@@ -47,6 +52,23 @@ export class DefaultEvalModelService {
       );
     }
 
+    try {
+      if (LLMApiKeySchema.safeParse(llmApiKey).success) {
+        // Make a test structured output call to validate the LLM key
+        await testModelCall({
+          provider,
+          model,
+          apiKey: llmApiKey as z.infer<typeof LLMApiKeySchema>,
+          modelConfig: modelParams,
+        });
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "Unknown error";
+      throw new ForbiddenError(
+        `Model configuration not valid for evaluation. ${message}`,
+      );
+    }
+
     // Create or update the default model
     return prisma.defaultLlmModel.upsert({
       where: {

packages/shared/src/server/services/SlackService.ts

@@ -0,0 +1,386 @@
+/**
+ * Slack Integration Service
+ *
+ * Simplified service that properly uses the official Slack SDK libraries:
+ * - @slack/oauth InstallProvider for OAuth flow management
+ * - @slack/web-api WebClient for Slack API operations
+ * - Metadata-based project-to-team mapping
+ */
+
+import { WebClient } from "@slack/web-api";
+import { InstallProvider } from "@slack/oauth";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { prisma } from "../../db";
+import { encrypt, decrypt } from "../../encryption";
+
+// Types for Slack integration
+export interface SlackChannel {
+  id: string;
+  name: string;
+  isPrivate: boolean;
+  isMember: boolean;
+}
+
+export interface SlackMessageParams {
+  client: WebClient;
+  channelId: string;
+  blocks: any[];
+  text?: string;
+}
+
+export interface SlackMessageResponse {
+  messageTs: string;
+  channel: string;
+}
+
+// Interface for Slack installation metadata
+export interface SlackInstallationMetadata {
+  projectId: string;
+}
+
+/**
+ * Type guard to validate Slack installation metadata
+ */
+function isSlackInstallationMetadata(
+  metadata: unknown,
+): metadata is SlackInstallationMetadata {
+  return (
+    typeof metadata === "object" &&
+    metadata !== null &&
+    "projectId" in metadata &&
+    typeof metadata.projectId === "string" &&
+    metadata.projectId.length > 0
+  );
+}
+
+/**
+ * Helper function to safely parse and validate Slack installation metadata
+ */
+export function parseSlackInstallationMetadata(
+  metadata: unknown,
+): SlackInstallationMetadata {
+  if (typeof metadata !== "string") {
+    throw new Error("Installation metadata must be a string");
+  }
+
+  let parsedMetadata: unknown;
+  try {
+    parsedMetadata = JSON.parse(metadata);
+  } catch {
+    throw new Error("Failed to parse installation metadata as JSON");
+  }
+
+  if (!isSlackInstallationMetadata(parsedMetadata)) {
+    throw new Error(
+      "Invalid installation metadata: missing or invalid projectId",
+    );
+  }
+
+  return parsedMetadata;
+}
+
+/**
+ * Slack Service Class
+ *
+ * Uses InstallProvider for OAuth flow and metadata-based project mapping.
+ * Much simpler than the previous implementation while maintaining all functionality.
+ */
+export class SlackService {
+  private static instance: SlackService | null = null;
+  private installer: InstallProvider;
+
+  private constructor() {
+    this.installer = new InstallProvider({
+      clientId: env.SLACK_CLIENT_ID!,
+      clientSecret: env.SLACK_CLIENT_SECRET!,
+      stateSecret: env.SLACK_STATE_SECRET!,
+      installUrlOptions: {
+        scopes: ["channels:read", "chat:write", "chat:write.public"],
+      },
+      installationStore: {
+        storeInstallation: async (installation) => {
+          try {
+            const metadata = parseSlackInstallationMetadata(
+              installation.metadata,
+            );
+            const projectId = metadata.projectId;
+
+            logger.info("Storing Slack installation for project", {
+              projectId,
+              teamId: installation.team?.id,
+              teamName: installation.team?.name,
+            });
+
+            // Store by projectId (one integration per project)
+            await prisma.slackIntegration.upsert({
+              where: { projectId },
+              create: {
+                projectId,
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+              update: {
+                teamId: installation.team?.id!,
+                teamName: installation.team?.name!,
+                botToken: encrypt(installation.bot?.token!),
+                botUserId: installation.bot?.userId!,
+              },
+            });
+
+            logger.info("Slack installation stored successfully", {
+              projectId,
+              teamId: installation.team?.id,
+            });
+          } catch (error) {
+            logger.error("Failed to store Slack installation", { error });
+            throw error;
+          }
+        },
+
+        fetchInstallation: async (installQuery) => {
+          try {
+            // Handle both teamId and projectId lookups
+            // When SDK calls with teamId, we treat it as projectId
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided");
+            }
+
+            const integration = await prisma.slackIntegration.findFirst({
+              where: {
+                OR: [
+                  { teamId: lookupId }, // Actual team ID lookup
+                  { projectId: lookupId }, // Project ID lookup (our custom usage)
+                ],
+              },
+            });
+
+            if (!integration) {
+              throw new Error("Slack integration not found");
+            }
+
+            // Return full Installation interface as expected by SDK
+            return {
+              team: {
+                id: integration.teamId,
+                name: integration.teamName,
+              },
+              bot: {
+                id: integration.botUserId,
+                token: decrypt(integration.botToken),
+                userId: integration.botUserId,
+                scopes: [],
+              },
+              enterprise: undefined,
+              user: {
+                token: undefined,
+                refreshToken: undefined,
+                expiresAt: undefined,
+                scopes: undefined,
+                id: integration.botUserId,
+              },
+            };
+          } catch (error) {
+            logger.error("Failed to fetch Slack installation", { error });
+            throw error;
+          }
+        },
+
+        deleteInstallation: async (installQuery) => {
+          try {
+            const lookupId = installQuery.teamId;
+
+            if (!lookupId) {
+              throw new Error("No lookup ID provided for deletion");
+            }
+
+            await prisma.slackIntegration.deleteMany({
+              where: {
+                OR: [{ teamId: lookupId }, { projectId: lookupId }],
+              },
+            });
+
+            logger.info("Slack installation deleted successfully", {
+              lookupId,
+            });
+          } catch (error) {
+            logger.error("Failed to delete Slack installation", { error });
+            throw error;
+          }
+        },
+      },
+    });
+  }
+
+  /**
+   * Get singleton instance of SlackService
+   */
+  static getInstance(): SlackService {
+    if (!SlackService.instance) {
+      SlackService.instance = new SlackService();
+    }
+    return SlackService.instance;
+  }
+
+  /**
+   * Get the configured InstallProvider instance for OAuth handling
+   */
+  getInstaller(): InstallProvider {
+    return this.installer;
+  }
+
+  /**
+   * Reset the singleton instance (useful for testing)
+   */
+  static resetInstance(): void {
+    SlackService.instance = null;
+  }
+
+  /**
+   * Delete Slack integration for a project
+   */
+  async deleteIntegration(projectId: string): Promise<void> {
+    try {
+      if (!this.installer.installationStore?.deleteInstallation) {
+        throw new Error("Installation store not configured");
+      }
+
+      await this.installer.installationStore.deleteInstallation({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      logger.info("Slack integration deleted for project", { projectId });
+    } catch (error) {
+      logger.error("Failed to delete Slack integration", { error, projectId });
+      throw new Error(
+        `Failed to delete integration: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get WebClient for a specific project
+   */
+  async getWebClientForProject(projectId: string): Promise<WebClient> {
+    try {
+      // Use projectId as the teamId parameter (handled by our fetchInstallation)
+      const auth = await this.installer.authorize({
+        teamId: projectId,
+        isEnterpriseInstall: false,
+        enterpriseId: undefined,
+      });
+
+      if (!auth.botToken) {
+        throw new Error("No bot token found for project");
+      }
+
+      const client = new WebClient(auth.botToken);
+      logger.debug("Created WebClient for project", { projectId });
+
+      return client;
+    } catch (error) {
+      logger.error("Failed to create WebClient for project", {
+        error,
+        projectId,
+      });
+      throw new Error(
+        `Failed to create WebClient: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Get channels accessible to the bot
+   */
+  async getChannels(client: WebClient): Promise<SlackChannel[]> {
+    try {
+      const result = await client.conversations.list({
+        exclude_archived: true,
+        types: "public_channel",
+        limit: 200,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Slack API error: ${result.error}`);
+      }
+
+      const channels: SlackChannel[] = (result.channels || []).map(
+        (channel) => ({
+          id: channel.id!,
+          name: channel.name!,
+          isPrivate: channel.is_private || false,
+          isMember: channel.is_member || false,
+        }),
+      );
+
+      logger.debug("Retrieved channels from Slack", {
+        channelCount: channels.length,
+      });
+
+      return channels;
+    } catch (error) {
+      logger.error("Failed to fetch channels", { error });
+      throw new Error(
+        `Failed to fetch channels: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Send a message to a Slack channel
+   */
+  async sendMessage(params: SlackMessageParams): Promise<SlackMessageResponse> {
+    try {
+      const result = await params.client.chat.postMessage({
+        channel: params.channelId,
+        blocks: params.blocks,
+        text: params.text || "Langfuse Notification",
+        unfurl_links: false,
+        unfurl_media: false,
+      });
+
+      if (!result.ok) {
+        throw new Error(`Failed to send message: ${result.error}`);
+      }
+
+      const response = {
+        messageTs: result.ts!,
+        channel: result.channel!,
+      };
+
+      logger.info("Message sent successfully to Slack", {
+        channel: params.channelId,
+        messageTs: response.messageTs,
+      });
+
+      return response;
+    } catch (error) {
+      logger.error("Failed to send message", {
+        error,
+        channelId: params.channelId,
+      });
+      throw new Error(
+        `Failed to send message: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  /**
+   * Validate a WebClient instance
+   */
+  async validateClient(client: WebClient): Promise<boolean> {
+    try {
+      const result = await client.auth.test();
+      return result.ok || false;
+    } catch (error) {
+      logger.warn("Client validation failed", { error });
+      return false;
+    }
+  }
+}

packages/shared/src/server/services/StorageService.ts

@@ -104,6 +104,7 @@ export class StorageServiceFactory {
   }
 }
 
+let azureContainersExists: Record<string, boolean> = {};
 class AzureBlobStorageService implements StorageService {
   private client: ContainerClient;
   private container: string;
@@ -139,8 +140,18 @@ class AzureBlobStorageService implements StorageService {
   }
 
   private async createContainerIfNotExists(): Promise<void> {
+    // Skip container existence check if environment variable is set
+    if (env.LANGFUSE_AZURE_SKIP_CONTAINER_CHECK === "true") {
+      return;
+    }
+
     try {
+      if (azureContainersExists[this.container]) {
+        return; // Container already exists, no need to create it again
+      }
       await this.client.createIfNotExists();
+      azureContainersExists[this.container] = true; // Mark container as created
+      logger.info(`Azure Blob Storage container ${this.container} created`);
     } catch (err) {
       logger.error(
         `Failed to create Azure Blob Storage container ${this.container}`,

packages/shared/src/server/services/datasets-ui-table-service.ts

@@ -9,7 +9,7 @@ type FetchDatasetItemsTableProps = {
   filter: FilterState;
 };
 
-const getDatasetRunItemsTableGeneric = async <T>(
+const getDatasetRunItemsTableGenericPg = async <T>(
   props: FetchDatasetItemsTableProps,
 ) => {
   const { select, projectId, filter } = props;
@@ -45,11 +45,11 @@ const getDatasetRunItemsTableGeneric = async <T>(
   return res;
 };
 
-export const getDatasetRunItemsTableCount = async (props: {
+export const getDatasetRunItemsTableCountPg = async (props: {
   projectId: string;
   filter: FilterState;
 }) => {
-  const res = await getDatasetRunItemsTableGeneric<Array<{ count: bigint }>>({
+  const res = await getDatasetRunItemsTableGenericPg<Array<{ count: bigint }>>({
     select: "count",
     projectId: props.projectId,
     filter: props.filter,

packages/shared/src/server/services/email/billingAlert/BillingAlertEmailTemplate.tsx

@@ -0,0 +1,134 @@
+import React from "react";
+import {
+  Body,
+  Button,
+  Container,
+  Head,
+  Heading,
+  Hr,
+  Html,
+  Img,
+  Preview,
+  Section,
+  Text,
+  Tailwind,
+  Row,
+  Column,
+} from "@react-email/components";
+
+interface BillingAlertEmailProps {
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const BillingAlertEmailTemplate = ({
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  return (
+    <Html>
+      <Head />
+      <Preview>
+        Your Langfuse Cloud usage is {`${currentUsage}`} events for the current
+        billing period
+      </Preview>
+      <Tailwind>
+        <Body className="bg-background my-auto mx-auto font-sans">
+          <Container className="mx-auto my-10 w-[465px] rounded border border-solid border-[#eaeaea] p-5">
+            <Section className="mt-8">
+              <Img
+                src="https://static.langfuse.com/langfuse_logo_transactional_email.png"
+                width="40"
+                height="40"
+                alt="Langfuse"
+                className="mx-auto my-0"
+              />
+            </Section>
+
+            <Section>
+              <Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
+                Usage Threshold Exceeded
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                Your organization &quot;{organizationName}&quot; has exceeded
+                the configured billing threshold
+              </Text>
+            </Section>
+
+            <Section className="mt-8">
+              <div className="bg-gray-50 border border-gray-200 rounded-lg p-4">
+                <Row>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Current Usage (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {currentUsage}
+                    </Text>
+                  </Column>
+                  <Column className="text-center">
+                    <Text className="text-gray-600 text-sm font-medium m-0 mb-1">
+                      Alert Threshold (# Events)
+                    </Text>
+                    <Text className="text-2xl font-bold text-gray-900 m-0">
+                      {threshold}
+                    </Text>
+                  </Column>
+                </Row>
+              </div>
+            </Section>
+
+            <Section className="mt-8 text-center">
+              <Button
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
+                href={billingUrl}
+              >
+                View Billing Page and Manage Alerts
+              </Button>
+            </Section>
+
+            <Section className="mt-8">
+              <Heading className="text-black text-[18px] font-semibold">
+                What happens next?
+              </Heading>
+              <Text className="text-gray-700 text-sm leading-6">
+                • Your current billing cycle continues normally
+                <br />
+                • Charges will appear on your next invoice
+                <br />
+                • You can adjust usage or modify alert thresholds
+                <br />• Contact support if you have questions about your bill
+              </Text>
+            </Section>
+
+            <Hr className="border border-solid border-[#eaeaea] my-[26px] mx-0 w-full" />
+
+            <Section>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                This email was sent to {receiverEmail} regarding billing alerts
+                for &quot;{organizationName}&quot;.
+              </Text>
+              <Text className="text-[#666666] text-[12px] leading-[24px]">
+                Questions? Contact us at{" "}
+                <a
+                  href="mailto:support@langfuse.com"
+                  className="text-blue-600 no-underline"
+                >
+                  support@langfuse.com
+                </a>
+              </Text>
+            </Section>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  );
+};
+
+export default BillingAlertEmailTemplate;

packages/shared/src/server/services/email/billingAlert/sendBillingAlertEmail.ts

@@ -0,0 +1,59 @@
+import { createTransport } from "nodemailer";
+import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
+import { render } from "@react-email/render";
+import { BillingAlertEmailTemplate } from "./BillingAlertEmailTemplate";
+import { logger } from "../../../logger";
+
+export interface BillingAlertEmailProps {
+  env: Partial<
+    Record<"EMAIL_FROM_ADDRESS" | "SMTP_CONNECTION_URL", string | undefined>
+  >;
+  organizationName: string;
+  currentUsage: number;
+  threshold: number;
+  billingUrl: string;
+  receiverEmail: string;
+}
+
+export const sendBillingAlertEmail = async ({
+  env,
+  organizationName,
+  currentUsage,
+  threshold,
+  billingUrl,
+  receiverEmail,
+}: BillingAlertEmailProps) => {
+  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
+    logger.error(
+      "Missing environment variables for sending billing alert email.",
+    );
+    return;
+  }
+
+  try {
+    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
+
+    const emailSubject = `Langfuse Cloud Billing Alert: ${organizationName} usage exceeded ${threshold} events`;
+    const emailHtml = await render(
+      BillingAlertEmailTemplate({
+        organizationName,
+        currentUsage,
+        threshold,
+        billingUrl,
+        receiverEmail,
+      }),
+    );
+
+    await mailer.sendMail({
+      to: receiverEmail,
+      from: {
+        address: env.EMAIL_FROM_ADDRESS,
+        name: "Langfuse",
+      },
+      subject: emailSubject,
+      html: emailHtml,
+    });
+  } catch (error) {
+    logger.error(`Failed to send billing alert email`, error);
+  }
+};

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -3,6 +3,7 @@ import { OrderByState } from "../../interfaces/orderBy";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { FilterState } from "../../types";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
 import {
   getProjectIdDefaultFilter,
@@ -11,6 +12,7 @@ import {
 import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
   queryClickhouse,
+  getTimeframesTracesAMT,
 } from "../repositories";
 
 export type SessionDataReturnType = {
@@ -22,6 +24,8 @@ export type SessionDataReturnType = {
   trace_count: number;
   trace_tags: string[];
   trace_environment?: string;
+  scores_avg?: Array<Array<[string, number]>>;
+  score_categories?: Array<Array<string>>;
 };
 
 export type SessionWithMetricsReturnType = SessionDataReturnType & {
@@ -157,7 +161,9 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         session_total_cost,
         session_input_usage,
         session_output_usage,
-        session_total_usage`;
+        session_total_usage,
+        scores_avg,
+        score_categories`;
       break;
     default: {
       const exhaustiveCheckDefault: never = select;
@@ -165,7 +171,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     }
   }
 
-  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+  const { tracesFilter, scoresFilter } = getProjectIdDefaultFilter(projectId, {
     tracesPrefix: "s",
   });
 
@@ -174,6 +180,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
   const tracesFilterRes = tracesFilter
     .filter((f) => f.field !== "environment")
     .apply();
+  const scoresFilterRes = scoresFilter.apply();
 
   const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
     (f) =>
@@ -193,20 +200,25 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
     );
   }
 
-  const additionalSingleTraceFilter = tracesFilter.find(
-    (f) =>
-      f.field === "bookmarked" ||
-      f.field === "session_id" ||
-      f.field === "environment",
-  );
-
-  if (additionalSingleTraceFilter) {
-    filters.push(additionalSingleTraceFilter);
-  }
+  tracesFilter
+    .filter(
+      (f) =>
+        f.field === "bookmarked" ||
+        f.field === "session_id" ||
+        f.field === "environment",
+    )
+    .forEach((f) => filters.push(f));
 
   const singleTraceFilter =
     filters.length > 0 ? new FilterList(filters).apply() : undefined;
 
+  const requiresScoresJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "scores") !== undefined ||
+    sessionCols.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "scores";
+
   const hasMetricsFilter =
     tracesFilter.find((f) =>
       [
@@ -217,6 +229,8 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         "session_total_usage",
         "session_output_usage",
         "session_input_usage",
+        "scores_avg",
+        "score_categories",
       ].includes(f.field),
     ) ||
     (orderBy &&
@@ -233,115 +247,188 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
 
   const selectMetrics = select === "metrics" || hasMetricsFilter;
 
+  const scoresCte = `scores_agg AS (
+    SELECT
+      project_id,
+      session_id AS score_session_id,
+      -- For numeric scores, use tuples of (name, avg_value)
+      groupArrayIf(
+        tuple(name, avg_value),
+        data_type IN ('NUMERIC', 'BOOLEAN')
+      ) AS scores_avg,
+      -- For categorical scores, use name:value format for improved query performance
+      groupArrayIf(
+        concat(name, ':', string_value),
+        data_type = 'CATEGORICAL' AND notEmpty(string_value)
+      ) AS score_categories
+    FROM (
+      SELECT
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value,
+        avg(value) avg_value
+      FROM scores s FINAL
+      WHERE 
+        project_id = {projectId: String}
+        ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
+      GROUP BY
+        project_id,
+        session_id,
+        name,
+        data_type,
+        string_value
+      ) tmp
+    GROUP BY
+      project_id, session_id
+  )`;
+
   // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
   const query = `
-    WITH deduplicated_traces AS (
-      SELECT * EXCEPT input, output, metadata
-      FROM traces t
-      WHERE t.session_id IS NOT NULL 
-        AND t.project_id = {projectId: String}
-        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    deduplicated_observations AS (
-        SELECT * 
-        FROM observations o
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        AND o.trace_id IN (
-          SELECT id
-          FROM deduplicated_traces
-        )
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    observations_agg AS (
-      SELECT o.trace_id,
-            count(*) as obs_count,
-            min(o.start_time) as min_start_time,
-            max(o.end_time) as max_end_time,
-            sumMap(usage_details) as sum_usage_details,
-            sumMap(cost_details) as sum_cost_details,
-            anyLast(project_id) as project_id
-      FROM deduplicated_observations o
-      WHERE o.project_id = {projectId: String}
-      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-      GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            anyLast(t.environment) as trace_environment
-            -- Aggregate observations data at session level
-            ${
-              selectMetrics
-                ? `
-            ,
-            sum(o.obs_count) as total_observations,
-            -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
-            -- LLM session started on that date so this behaviour should yield better results.
-            date_diff('millisecond', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage`
-                : ""
-            }
-        FROM deduplicated_traces t
-        ${
-          selectMetrics
-            ? `LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id`
-            : ""
-        }
-        WHERE t.session_id IS NOT NULL
+        WITH ${select === "metrics" || requiresScoresJoin ? `${scoresCte},` : ""}
+        deduplicated_traces AS (
+          SELECT * EXCEPT input, output, metadata
+          FROM __TRACE_TABLE__ t
+          WHERE t.session_id IS NOT NULL 
             AND t.project_id = {projectId: String}
             ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${sqlSelect}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
+            LIMIT 1 BY id, project_id
+        ),
+        deduplicated_observations AS (
+            SELECT * 
+            FROM observations o
+            WHERE o.project_id = {projectId: String}
+            ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+            AND o.trace_id IN (
+              SELECT id
+              FROM deduplicated_traces
+            )
+            ORDER BY event_ts DESC
+            LIMIT 1 BY id, project_id
+        ),
+        observations_agg AS (
+          SELECT o.trace_id,
+                count(*) as obs_count,
+                min(o.start_time) as min_start_time,
+                max(o.end_time) as max_end_time,
+                sumMap(usage_details) as sum_usage_details,
+                sumMap(cost_details) as sum_cost_details,
+                anyLast(project_id) as project_id
+          FROM deduplicated_observations o
+          WHERE o.project_id = {projectId: String}
+          ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+          GROUP BY o.trace_id
+        ),
+        session_data AS (
+            SELECT
+                t.session_id,
+                anyLast(t.project_id) as project_id,
+                max(t.timestamp) as max_timestamp,
+                min(t.timestamp) as min_timestamp,
+                groupArray(t.id) AS trace_ids,
+                groupUniqArray(t.user_id) AS user_ids,
+                count(*) as trace_count,
+                groupUniqArrayArray(t.tags) as trace_tags,
+                anyLast(t.environment) as trace_environment
+                -- Aggregate observations data at session level
+                ${
+                  selectMetrics
+                    ? `,
+                      sum(o.obs_count) as total_observations,
+                      -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
+                      -- LLM session started on that date so this behaviour should yield better results.
+                      date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
+                      sumMap(o.sum_usage_details) as session_usage_details,
+                      sumMap(o.sum_cost_details) as session_cost_details,
+                      ${
+                        select === "metrics" || requiresScoresJoin
+                          ? `groupUniqArrayArray(s.scores_avg) as scores_avg,
+                      groupUniqArrayArray(s.score_categories) as score_categories,`
+                          : ""
+                      }
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+                      sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+                      sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                    : ""
+                }
+            FROM deduplicated_traces t
+            ${
+              selectMetrics
+                ? `LEFT JOIN observations_agg o
+                   ON t.id = o.trace_id AND t.project_id = o.project_id`
+                : ""
+            }
+           ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_agg s on s.project_id = t.project_id and t.session_id = s.score_session_id` : ""}
+            WHERE t.session_id IS NOT NULL
+                AND t.project_id = {projectId: String}
+                ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+            GROUP BY t.session_id
+        )
+        SELECT ${sqlSelect}
+        FROM session_data s
+        WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+        ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+        `;
 
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
+  return measureAndReturn({
+    operationName: "getSessionsTableGeneric",
+    projectId,
+    minStartTime: filter?.find(
+      (f) =>
+        f.column === "min_timestamp" &&
+        (f.operator === ">=" || f.operator === ">"),
+    )?.value as Date | undefined,
+    input: {
+      params: {
+        projectId,
+        limit: limit,
+        offset: limit && page ? limit * page : 0,
+        ...tracesFilterRes.params,
+        ...singleTraceFilter?.params,
+        ...scoresFilterRes.params,
+        ...(traceTimestampFilter
+          ? {
+              observationsStartTime: convertDateToClickhouseDateTime(
+                traceTimestampFilter.value,
+              ),
+            }
+          : {}),
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "sessions-table",
+        projectId,
+        operation_name: "getSessionsTableGeneric",
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "sessions-table",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "original" },
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      // Extract the timestamp from filter for AMT table selection
+      const fromTimestamp = filter?.find(
+        (f) =>
+          f.column === "min_timestamp" &&
+          (f.operator === ">=" || f.operator === ">"),
+      )?.value as Date | undefined;
+      const traceAmt = getTimeframesTracesAMT(fromTimestamp);
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: { ...input.tags, experiment_amt: "new" },
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -21,7 +21,9 @@ import {
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
   reduceUsageOrCostDetails,
+  getTimeframesTracesAMT,
 } from "../repositories";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { TracingSearchType } from "../../interfaces/search";
 import { ObservationLevelType, TraceDomain } from "../../domain";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
@@ -217,54 +219,6 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
     clickhouseConfigs,
   } = props;
 
-  let sqlSelect: string;
-  switch (select) {
-    case "count":
-      sqlSelect = "count(*) as count";
-      break;
-    case "metrics":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as project_id,
-        t.timestamp as timestamp,
-        os.latency_milliseconds / 1000 as latency,
-        os.cost_details as cost_details,
-        os.usage_details as usage_details,
-        os.aggregated_level as level,
-        os.error_count as error_count,
-        os.warning_count as warning_count,
-        os.default_count as default_count,
-        os.debug_count as debug_count,
-        os.observation_count as observation_count,
-        s.scores_avg as scores_avg,
-        s.score_categories as score_categories,
-        t.public as public`;
-      break;
-    case "rows":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as project_id,
-        t.timestamp as timestamp,
-        t.tags as tags,
-        t.bookmarked as bookmarked,
-        t.name as name,
-        t.release as release,
-        t.version as version,
-        t.user_id as user_id,
-        t.environment as environment,
-        t.session_id as session_id,
-        t.public as public`;
-      break;
-    case "identifiers":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as projectId,
-        t.timestamp as timestamp`;
-      break;
-    default:
-      throw new Error(`Unknown select type: ${select}`);
-  }
-
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
 
@@ -329,149 +283,319 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
   const scoresFilterRes = scoresFilter.apply();
   const observationFilterRes = observationsFilter.apply();
 
-  const search = clickhouseSearchCondition(searchQuery, searchType, "t");
-
-  const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
-  const orderByCols = [
-    ...tracesTableUiColumnDefinitions,
-    {
-      clickhouseSelect: "toDate(t.timestamp)",
-      uiTableName: "timestamp_to_date",
-      uiTableId: "timestamp_to_date",
-      clickhouseTableName: "traces",
-    },
-    {
-      clickhouseSelect: "t.event_ts",
-      uiTableName: "event_ts",
-      uiTableId: "event_ts",
-      clickhouseTableName: "traces",
-    },
-  ];
-  const chOrderBy = orderByToClickhouseSql(
-    [
-      defaultOrder
-        ? [
-            {
-              column: "timestamp_to_date",
-              order: orderBy.order,
-            },
-            { column: "timestamp", order: orderBy.order },
-            { column: "event_ts", order: "DESC" as "DESC" },
-          ]
-        : null,
-      orderBy ?? null,
-    ].flat(),
-    orderByCols,
-  );
-
-  // complex query ahead:
-  // - we only join scores and observations if we really need them to speed up default views
-  // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
-  // - we filter the observations and scores as much as possible before joining them to traces.
-  // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
-  //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
-
-  const query = `
+  const observationsAndScoresCTE = `
     WITH observations_stats AS (
       SELECT
         COUNT(*) AS observation_count,
-          sumMap(usage_details) as usage_details,
-          SUM(total_cost) AS total_cost,
-          date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
-          countIf(level = 'ERROR') as error_count,
-          countIf(level = 'WARNING') as warning_count,
-          countIf(level = 'DEFAULT') as default_count,
-          countIf(level = 'DEBUG') as debug_count,
-          multiIf(
-            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
-            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
-            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
-            'DEBUG'
-          ) AS aggregated_level,
-          sumMap(cost_details) as cost_details,
-          trace_id,
-          project_id
-      FROM observations o FINAL 
+        sumMap(usage_details) as usage_details,
+        SUM(total_cost) AS total_cost,
+        date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+        countIf(level = 'ERROR') as error_count,
+        countIf(level = 'WARNING') as warning_count,
+        countIf(level = 'DEFAULT') as default_count,
+        countIf(level = 'DEBUG') as debug_count,
+        multiIf(
+          arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
+          arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
+          arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
+          'DEBUG'
+        ) AS aggregated_level,
+        sumMap(cost_details) as cost_details,
+        trace_id,
+        project_id
+      FROM observations o FINAL
       WHERE o.project_id = {projectId: String}
-      ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
-      ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
+        ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
       GROUP BY trace_id, project_id
     ),
-    scores_avg AS (
-      SELECT
-        project_id,
-        trace_id,
-        -- For numeric scores, use tuples of (name, avg_value)
-        groupArrayIf(
-          tuple(name, avg_value),
-          data_type IN ('NUMERIC', 'BOOLEAN')
-        ) AS scores_avg,
-        -- For categorical scores, use name:value format for improved query performance
-        groupArrayIf(
-          concat(name, ':', string_value),
-          data_type = 'CATEGORICAL' AND notEmpty(string_value)
-        ) AS score_categories
-      FROM (
-        SELECT 
-          project_id,
-          trace_id,
-          name,
-          data_type,
-          string_value,
-          avg(value) as avg_value
-        FROM scores s FINAL 
-        WHERE 
-          project_id = {projectId: String}
-          ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
-          ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
-        GROUP BY 
-          project_id,
-          trace_id,
-          name,
-          data_type,
-          string_value
-      ) tmp
-      GROUP BY project_id, trace_id
-    )
-    SELECT ${sqlSelect}
-    -- FINAL is used for non default ordering and count.
-    FROM traces t  ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "" : "FINAL"}
-    ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
-    ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
-    WHERE t.project_id = {projectId: String}
-    ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
-    ${search.query}
-    ${chOrderBy}
-    -- This is used for metrics and row queries. Count has only one result.
-    -- This is only used for default ordering. Otherwise, we use final.
-    ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+         scores_avg AS (
+           SELECT
+             project_id,
+             trace_id,
+             -- For numeric scores, use tuples of (name, avg_value)
+             groupArrayIf(
+               tuple(name, avg_value),
+               data_type IN ('NUMERIC', 'BOOLEAN')
+             ) AS scores_avg,
+             -- For categorical scores, use name:value format for improved query performance
+             groupArrayIf(
+               concat(name, ':', string_value),
+               data_type = 'CATEGORICAL' AND notEmpty(string_value)
+             ) AS score_categories
+           FROM (
+                  SELECT
+                    project_id,
+                    trace_id,
+                    name,
+                    data_type,
+                    string_value,
+                    avg(value) as avg_value
+                  FROM scores s FINAL
+                  WHERE
+                    project_id = {projectId: String}
+                    ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+                    ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
+                  GROUP BY
+                    project_id,
+                    trace_id,
+                    name,
+                    data_type,
+                    string_value
+                ) tmp
+           GROUP BY project_id, trace_id
+         )
   `;
 
-  const res = await queryClickhouse<
-    SelectReturnTypeMap[keyof SelectReturnTypeMap]
-  >({
-    query: query,
-    params: {
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      traceTimestamp: timeStampFilter?.value.getTime(),
-      projectId: projectId,
-      ...tracesFilterRes.params,
-      ...observationFilterRes.params,
-      ...scoresFilterRes.params,
-      ...search.params,
-    },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "traces-table",
-      projectId,
-    },
-    clickhouseConfigs,
-  });
+  return measureAndReturn({
+    operationName: "getTracesTableGeneric",
+    projectId: props.projectId,
+    minStartTime: select !== "metrics" ? timeStampFilter?.value : undefined,
+    input: props,
+    existingExecution: async (props) => {
+      let sqlSelect: string;
+      switch (select) {
+        case "count":
+          sqlSelect = "count(*) as count";
+          break;
+        case "metrics":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            os.latency_milliseconds / 1000 as latency,
+            os.cost_details as cost_details,
+            os.usage_details as usage_details,
+            os.aggregated_level as level,
+            os.error_count as error_count,
+            os.warning_count as warning_count,
+            os.default_count as default_count,
+            os.debug_count as debug_count,
+            os.observation_count as observation_count,
+            s.scores_avg as scores_avg,
+            s.score_categories as score_categories,
+            t.public as public`;
+          break;
+        case "rows":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            t.tags as tags,
+            t.bookmarked as bookmarked,
+            t.name as name,
+            t.release as release,
+            t.version as version,
+            t.user_id as user_id,
+            t.environment as environment,
+            t.session_id as session_id,
+            t.public as public`;
+          break;
+        case "identifiers":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as projectId,
+            t.timestamp as timestamp`;
+          break;
+        default:
+          throw new Error(`Unknown select type: ${select}`);
+      }
 
-  return res;
+      const search = clickhouseSearchCondition(searchQuery, searchType, "t");
+
+      const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
+      const orderByCols = [
+        ...tracesTableUiColumnDefinitions,
+        {
+          clickhouseSelect: "toDate(t.timestamp)",
+          uiTableName: "timestamp_to_date",
+          uiTableId: "timestamp_to_date",
+          clickhouseTableName: "traces",
+        },
+        {
+          clickhouseSelect: "t.event_ts",
+          uiTableName: "event_ts",
+          uiTableId: "event_ts",
+          clickhouseTableName: "traces",
+        },
+      ];
+      const chOrderBy = orderByToClickhouseSql(
+        [
+          defaultOrder
+            ? [
+                {
+                  column: "timestamp_to_date",
+                  order: orderBy.order,
+                },
+                { column: "timestamp", order: orderBy.order },
+                { column: "event_ts", order: "DESC" as "DESC" },
+              ]
+            : null,
+          orderBy ?? null,
+        ].flat(),
+        orderByCols,
+      );
+
+      // complex query ahead:
+      // - we only join scores and observations if we really need them to speed up default views
+      // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
+      // - we filter the observations and scores as much as possible before joining them to traces.
+      // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
+      //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
+
+      const query = `
+        ${observationsAndScoresCTE}        
+
+        SELECT ${sqlSelect}
+        -- FINAL is used for non default ordering and count.
+        FROM traces t  ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "" : "FINAL"}
+        ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+        ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+        WHERE t.project_id = {projectId: String}
+        ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
+        ${search.query}
+        ${chOrderBy}
+        -- This is used for metrics and row queries. Count has only one result.
+        -- This is only used for default ordering. Otherwise, we use final.
+        ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+      `;
+
+      const res = await queryClickhouse<
+        SelectReturnTypeMap[keyof SelectReturnTypeMap]
+      >({
+        query: query,
+        params: {
+          limit: limit,
+          offset: limit && page ? limit * page : 0,
+          traceTimestamp: timeStampFilter?.value.getTime(),
+          projectId: projectId,
+          ...tracesFilterRes.params,
+          ...observationFilterRes.params,
+          ...scoresFilterRes.params,
+          ...search.params,
+        },
+        tags: {
+          ...(props.tags ?? {}),
+          feature: "tracing",
+          type: "traces-table",
+          projectId,
+          experiment_amt: "original",
+          operation_name: "getTracesTableGeneric",
+        },
+        clickhouseConfigs,
+      });
+
+      return res;
+    },
+    newExecution: async () => {
+      let sqlSelect: string;
+      switch (select) {
+        case "count":
+          sqlSelect = "count(*) as count";
+          break;
+        case "metrics":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            os.latency_milliseconds / 1000 as latency,
+            os.cost_details as cost_details,
+            os.usage_details as usage_details,
+            os.aggregated_level as level,
+            os.error_count as error_count,
+            os.warning_count as warning_count,
+            os.default_count as default_count,
+            os.debug_count as debug_count,
+            os.observation_count as observation_count,
+            s.scores_avg as scores_avg,
+            s.score_categories as score_categories,
+            t.public as public`;
+          break;
+        case "rows":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            t.tags as tags,
+            finalizeAggregation(t.bookmarked) as bookmarked,
+            t.name as name,
+            t.release as release,
+            t.version as version,
+            t.user_id as user_id,
+            t.environment as environment,
+            t.session_id as session_id,
+            finalizeAggregation(t.public) as public`;
+          break;
+        case "identifiers":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as projectId,
+            t.timestamp as timestamp`;
+          break;
+        default:
+          throw new Error(`Unknown select type: ${select}`);
+      }
+
+      const search = clickhouseSearchCondition(
+        searchQuery,
+        searchType,
+        "t",
+        true,
+      );
+
+      const chOrderBy = orderByToClickhouseSql(
+        [orderBy ?? null].flat(),
+        tracesTableUiColumnDefinitions,
+      );
+
+      const tracesAmt =
+        select === "metrics"
+          ? "traces_all_amt"
+          : getTimeframesTracesAMT(timeStampFilter?.value);
+
+      const query = `
+        ${observationsAndScoresCTE}
+
+        SELECT ${sqlSelect}
+        FROM ${tracesAmt} t FINAL
+        ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+        ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+        WHERE t.project_id = {projectId: String}
+        ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
+        ${search.query}
+        ${chOrderBy}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+      `;
+
+      const res = await queryClickhouse<
+        SelectReturnTypeMap[keyof SelectReturnTypeMap]
+      >({
+        query: query,
+        params: {
+          limit: limit,
+          offset: limit && page ? limit * page : 0,
+          traceTimestamp: timeStampFilter?.value.getTime(),
+          projectId: projectId,
+          ...tracesFilterRes.params,
+          ...observationFilterRes.params,
+          ...scoresFilterRes.params,
+          ...search.params,
+        },
+        tags: {
+          ...(props.tags ?? {}),
+          feature: "tracing",
+          type: "traces-table",
+          projectId,
+          experiment_amt: "new",
+          operation_name: "getTracesTableGeneric",
+        },
+        clickhouseConfigs,
+      });
+
+      return res;
+    },
+  });
 }
 
 export const getTracesTableCount = async (props: {