chore: release v3.46.0

* feat: add support for GCS buckets

* chore: cleanup

* chore: trim gcs credentials before parsing

* chore: add logging for redis shutdown in tests

* chore: update timeout for trace delete test

* chore: bump trace delete concurrency for tests

.cursor/rules/authorization-and-rbac.mdc

@@ -0,0 +1,6 @@
+---
+description: How to manage authorization and RBAC within full-stack langfuse features
+globs: web/
+alwaysApply: true
+---
+See [README.md](mdc:web/src/features/rbac/README.md) for details

.cursor/rules/entitlements.mdc

@@ -0,0 +1,7 @@
+---
+description: How to enforce entitlements across full-stack features
+globs: web/**
+alwaysApply: true
+---
+
+Check [README.md](mdc:web/src/features/entitlements/README.md) to learn more about entitlements

.cursor/rules/frontend-features.mdc

@@ -0,0 +1,33 @@
+---
+description: How we build new full-stack features including frontend APIs, file structure, and components
+globs: web/**
+alwaysApply: true
+---
+## Framework
+
+- Next.js
+- Pages router
+
+## File Structure
+
+- We generally put all code related to a net-new feature into a folder within web/src/features.
+- Checkout other features to learn about the common structure.
+
+## API for frontend features
+
+- We use TRPC.io to power full-stack features of the Langfuse Frontend
+  - Entry point for all trpc routes: [root.ts](mdc:web/src/server/api/root.ts)
+- Authentication, see [authorization-and-rbac.mdc](mdc:.cursor/rules/authorization-and-rbac.mdc)
+- Entitlements, see [entitlements.mdc](mdc:.cursor/rules/entitlements.mdc)
+
+## Components
+
+- We use Shadcn/ui
+  - Components in `@/src/components/ui`
+  - If a component is not installed yet, ask the user to install it for you
+- When creating new custom components that generalize, we add them with a generalizable naming to `@/src/components`
+
+## Styling
+
+- We use Tailwind CSS
+- We use a standard color palette which automatically handles light/dark mode, see [globals.css](mdc:web/src/styles/globals.css)

.cursor/rules/public-api.mdc

@@ -0,0 +1,16 @@
+---
+description: How to create new public api routes for Langfuse
+globs: 
+alwaysApply: false
+---
+
+# Implementation
+
+- All public api routes are in /web/src/pages/api/public
+- New api routes should follow these guidelines:
+  - Use [withMiddlewares.ts](mdc:web/src/features/public-api/server/withMiddlewares.ts) as a wrapper
+  - Define types of api request and response in /web/src/features/public-api/types, use strict() for all zod objects
+  - Add end-to-end test, similar to [datasets-api.servertest.ts](mdc:web/src/__tests__/async/datasets-api.servertest.ts)
+  - Add fern configuration in /fern, learn more about structure here: https://buildwithfern.com/learn/api-definition/fern/overview
+  - Prompt user to regenerate the OpenAPI spec via the fern CLI
+  - Pagination starts at 1, query typing defined in publicApiPaginationZod, return meta in paginationMetaResponseZod

.env.prod.example

@@ -84,6 +84,7 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITLAB_ISSUER=
 # AUTH_GITLAB_CLIENT_AUTH_METHOD=
 # AUTH_GITLAB_CHECKS=
+# AUTH_GITLAB_URL=
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
@@ -114,6 +115,11 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_KEYCLOAK_CLIENT_AUTH_METHOD=
 # AUTH_KEYCLOAK_CHECKS=
+# AUTH_WORKOS_CLIENT_ID=
+# AUTH_WORKOS_CLIENT_SECRET=
+# AUTH_WORKOS_ALLOW_ACCOUNT_LINKING=false
+# AUTH_WORKOS_ORGANIZATION_ID=
+# AUTH_WORKOS_CONNECTION_ID=
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
@@ -166,7 +172,32 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_INIT_USER_NAME=User Name
 # LANGFUSE_INIT_USER_PASSWORD=password
 
+# Redis configuration
+# REDIS_HOST=
+# REDIS_PORT=
+# REDIS_AUTH=
+# REDIS_CONNECTION_STRING=
+# REDIS_ENABLE_AUTO_PIPELINING=
 
+# Cache configuration
+# LANGFUSE_CACHE_API_KEY_ENABLED=
+# LANGFUSE_CACHE_API_KEY_TTL_SECONDS=
+# LANGFUSE_CACHE_PROMPT_ENABLED=
+# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
+
+# Clickhouse configuration
+# CLICKHOUSE_URL=
+# CLICKHOUSE_CLUSTER_NAME=default
+# CLICKHOUSE_DB=default
+# CLICKHOUSE_USER=
+# CLICKHOUSE_PASSWORD=
+# CLICKHOUSE_CLUSTER_ENABLED=true
+
+# Ingestion configuration
+# LANGFUSE_INGESTION_QUEUE_DELAY_MS=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
+# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
 
 ### START Enterprise Edition Configuration
 
@@ -188,17 +219,6 @@ OTEL_SERVICE_NAME="langfuse"
 ### END Enterprise Edition Configuration
 
 
-
-### START Envs to be deprecated in Langfuse v3.0
-
-# Disable the expensive analytics queries and related features
-# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
-
-### END Envs to be deprecated in Langfuse v3.0
-
-
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
@@ -235,19 +255,8 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Admin API
 # ADMIN_API_KEY=
-
-# Redis
-# REDIS_HOST=
-# REDIS_PORT=
-# REDIS_AUTH=
-# REDIS_CONNECTION_STRING=
-# REDIS_ENABLE_AUTO_PIPELINING=
-
-# Cache configuration
-# LANGFUSE_CACHE_API_KEY_ENABLED=
-# LANGFUSE_CACHE_API_KEY_TTL_SECONDS=
-# LANGFUSE_CACHE_PROMPT_ENABLED=
-# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
+# LANGFUSE_CACHE_MODEL_MATCH_ENABLED=
+# LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS=
 
 # Rate limiting
 # LANGFUSE_RATE_LIMITS_ENABLED=
@@ -256,22 +265,15 @@ OTEL_SERVICE_NAME="langfuse"
 # STRIPE_SECRET_KEY=
 # STRIPE_WEBHOOK_SIGNING_SECRET=
 
+# Betterstack Status Page
+# BETTERSTACK_UPTIME_API_KEY=
+# BETTERSTACK_UPTIME_STATUS_PAGE_ID=
+
 ### END Langfuse Cloud Config
 
 
-## START Langfuse V3 Ingestion 
+### START Langfuse CI Config
 
-# Clickhouse
-# CLICKHOUSE_MIGRATION_URL=
-# CLICKHOUSE_URL=
-# CLICKHOUSE_USER=
-# CLICKHOUSE_PASSWORD=
-# CLICKHOUSE_DB=
+# LANGFUSE_INIT_ORG_CLOUD_PLAN=
 
-# Ingestion
-# LANGFUSE_INGESTION_QUEUE_DELAY_MS=
-# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
-# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
-# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
-
-## END Langfuse V3 Ingestion
+### END Langfuse CI Config

.github/workflows/_deploy_ecs_service.yml

@@ -56,6 +56,7 @@ jobs:
             --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
             --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
       - name: Render AWS ECS Task Definition

.github/workflows/pipeline.yml

@@ -134,14 +134,14 @@ jobs:
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
-      - name: Run + migrate
+      - name: Run dev containers
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
           POSTGRES_VERSION: ${{ matrix.postgres-version }}
-      - name: Seed DB
+      - name: Migrate DB
         run: |
           pnpm run db:migrate
           pnpm --filter=shared ch:up
@@ -152,6 +152,7 @@ jobs:
         env:
           LANGFUSE_INIT_ORG_ID: "seed-org-id"
           LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_ORG_CLOUD_PLAN: "Team"
           LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
           LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
           LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
@@ -211,14 +212,15 @@ jobs:
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
-      - name: Run + migrate
+          echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100" >> .env
+      - name: Run dev containers
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
           POSTGRES_VERSION: ${{ matrix.postgres-version }}
-      - name: Seed DB
+      - name: Migrate DB
         run: |
           pnpm run db:migrate
           pnpm --filter=shared ch:up
@@ -229,6 +231,7 @@ jobs:
         env:
           LANGFUSE_INIT_ORG_ID: "seed-org-id"
           LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_ORG_CLOUD_PLAN: "Team"
           LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
           LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
           LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"

.github/workflows/stale_issues.yml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale. Please reopen if the issue persists."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -53,117 +53,10 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
-**Langfuse v2**
-
-```mermaid
-flowchart TB
-    subgraph s4["Clients"]
-        subgraph s2["langfuse/langfuse-python"]
-            Python["Python low-level SDK"]
-            Decorator["observe() decorator"] -->|extends| Python
-            OAI["OpenAI drop-in replacement"] -->|extends| Python
-            Llamaindex["LlamaIndex Integration"] -->|extends| Python
-            LCPYTHON["Langchain Python Integration"] -->|extends| Python
-            Langflow -->|uses| LCPYTHON
-            LiteLLM -->|uses| Python
-        end
-        subgraph s3["langfuse/langfuse-js"]
-            JS["JS SDK"]
-            LCJS["Langchain JS Integration"]  -->|extends| JS
-            Flowise -->|uses| LCJS
-        end
-    end
-
-    DB[Postgres Database]
-    Redis[Redis]
-
-    subgraph s1["Application (langfuse/langfuse/web)"]
-        API[Public HTTP API]
-        G[TRPC API]
-        I[NextAuth]
-        H[React Frontend]
-        Prisma[Prisma ORM]
-        H --> G
-        H --> I
-        G --> I
-        G --- Prisma
-        API --- Prisma
-        I --- Prisma
-    end
-
-    Prisma --- DB
-    JS --- API
-    Python --- API
-```
-
-**Langfuse v3 (work in progress, not released yet)**
-
-> [!NOTE]
-> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
-
-```mermaid
-flowchart TB
-    subgraph s4["Clients"]
-        subgraph s2["langfuse/langfuse-python"]
-            Python["Python low-level SDK"]
-            Decorator["observe() decorator"] -->|extends| Python
-            OAI["OpenAI drop-in replacement"] -->|extends| Python
-            Llamaindex["LlamaIndex Integration"] -->|extends| Python
-            LCPYTHON["Langchain Python Integration"] -->|extends| Python
-            Langflow -->|uses| LCPYTHON
-            LiteLLM -->|uses| Python
-        end
-        subgraph s3["langfuse/langfuse-js"]
-            JS["JS SDK"]
-            LCJS["Langchain JS Integration"]  -->|extends| JS
-            Flowise -->|uses| LCJS
-        end
-    end
-
-    subgraph s9 ["VPC (US and EU separated)"]
-        DB[Postgres Database]
-        Redis[Redis Cache/Queue]
-        Clickhouse[Clickhouse Database]
-
-        subgraph s1["Application (langfuse/langfuse/web)"]
-            API[Public HTTP API]
-            G[TRPC API]
-            I[NextAuth]
-            H[React Frontend]
-            ORM
-            H --> G
-            H --> I
-            G --> I
-            G --- ORM
-            API --- ORM
-            I --- ORM
-        end
-
-        subgraph s5["Application (langfuse/langfuse/worker)"]
-            Worker
-        end
-
-        Worker --- DB
-        Worker --- Redis
-        Worker --- Clickhouse
-
-        ORM --- DB
-        ORM --- Redis
-        ORM --- Clickhouse
-
-
-    end
-
-    JS --- API
-    Python --- API
-```
+See this [diagram](https://langfuse.com/self-hosting#architecture) for an overview of the architecture.
 
 ### Network Overview
 
-> [!NOTE]
-> This will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-
 ```mermaid
 flowchart LR
    Browser ---|Web UI & TRPC API| App
@@ -209,11 +102,12 @@ Requirements
 
 **Steps**
 
-1. Fork the repository and clone it locally
-2. Run the development database
+1. Install [golang-migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate#migrate-cli) as CLI
+2. Fork the repository and clone it locally
 
    ```bash
-   pnpm run infra:dev:up
+   git clone https://github.com/langfuse/langfuse.git
+   cd langfuse
    ```
 
 3. Create an env file
@@ -222,40 +116,23 @@ Requirements
     cp .env.dev.example .env
    ```
 
-4. Install dependencies
+4. Run the entire infrastructure in dev mode
 
    ```bash
-   pnpm install
+   pnpm run dx
    ```
 
-5. Run the migrations
+   You will be asked whether you want to reset Postgres and ClickHouse. Confirm both with 'Y' and press enter.
 
-   All database migrations and configs are in the `shared` package.
+5. Open the web app in your browser to start using Langfuse:
 
-   ```bash
-   pnpm --filter=shared run db:migrate
+   - [Sign up page, http://localhost:3000](http://localhost:3000)
+   - [Demo project, http://localhost:3000/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a](http://localhost:3000/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a)
 
-   # Optional: seed the database
-   # pnpm run db:seed
-   # pnpm run db:seed:examples
-   # pnpm --filter=shared run db:seed:load
-   ```
+6. Log in as a test user:
 
-6. Start the development server
-
-   ```bash
-    pnpm run dev
-   ```
-
-7. Open the web app in the browser:
-
-   http://localhost:3000
-
-8. Log in as a test user (after you ran `db:seed` command):
-
-   Username: demo@langfuse.com
-
-   Password: password
+   - Username: `demo@langfuse.com`
+   - Password: `password`
 
 ## Monorepo quickstart
 

docker-compose.dev.yml

@@ -10,8 +10,8 @@ services:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
     ports:
-      - "8123:8123"
-      - "9000:9000"
+      - 127.0.0.1:8123:8123
+      - 127.0.0.1:9000:9000
     depends_on:
       - postgres
 
@@ -24,8 +24,8 @@ services:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: miniosecret
     ports:
-      - "9090:9000"
-      - "9091:9001"
+      - 127.0.0.1:9090:9000
+      - 127.0.0.1:9091:9001
     volumes:
       - langfuse_minio_data:/data
     healthcheck:
@@ -41,7 +41,7 @@ services:
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
     ports:
-      - 6379:6379
+      - 127.0.0.1:6379:6379
 
   postgres:
     image: postgres:${POSTGRES_VERSION:-latest}
@@ -57,7 +57,7 @@ services:
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=postgres
     ports:
-      - 5432:5432
+      - 127.0.0.1:5432:5432
     volumes:
       - langfuse_postgres_data:/var/lib/postgresql/data
 

docker-compose.yml

@@ -1,3 +1,8 @@
+# Make sure to update the credential placeholders with your own secrets.
+# We mark them with # CHANGEME in the file below.
+# In addition, we recommend to restrict inbound traffic on the host to langfuse-web (port 3000) and minio (port 9090) only.
+# All other components are bound to localhost (127.0.0.1) to only accept connections from the local machine.
+# External connections from other machines will not be able to reach these services directly.
 services:
   langfuse-worker:
     image: langfuse/langfuse-worker:3
@@ -12,37 +17,37 @@ services:
       clickhouse:
         condition: service_healthy
     ports:
-      - "3030:3030"
+      - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      SALT: "mysalt"
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
+      SALT: "mysalt" # CHANGEME
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
       TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
       LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
       CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
       CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
       CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
       LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
       LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
       LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
       LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
       LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
       LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090}
       LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
       LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
       LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
       LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
       REDIS_HOST: ${REDIS_HOST:-redis}
       REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret} # CHANGEME
       REDIS_TLS_ENABLED: ${REDIS_TLS_ENABLED:-false}
       REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
       REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
@@ -53,11 +58,11 @@ services:
     restart: always
     depends_on: *langfuse-depends-on
     ports:
-      - "3000:3000"
+      - 3000:3000
     environment:
       <<: *langfuse-worker-env
       NEXTAUTH_URL: http://localhost:3000
-      NEXTAUTH_SECRET: mysecret
+      NEXTAUTH_SECRET: mysecret # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
       LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
@@ -75,13 +80,13 @@ services:
     environment:
       CLICKHOUSE_DB: default
       CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse # CHANGEME
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
     ports:
-      - "8123:8123"
-      - "9000:9000"
+      - 127.0.0.1:8123:8123
+      - 127.0.0.1:9000:9000
     healthcheck:
       test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
       interval: 5s
@@ -97,10 +102,10 @@ services:
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
     environment:
       MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
+      MINIO_ROOT_PASSWORD: miniosecret # CHANGEME
     ports:
-      - "9090:9000"
-      - "9091:9001"
+      - 9090:9000
+      - 127.0.0.1:9091:9001
     volumes:
       - langfuse_minio_data:/data
     healthcheck:
@@ -113,10 +118,11 @@ services:
   redis:
     image: redis:7
     restart: always
+    # CHANGEME: row below to secure redis password
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
     ports:
-      - 6379:6379
+      - 127.0.0.1:6379:6379
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 3s
@@ -133,10 +139,10 @@ services:
       retries: 10
     environment:
       POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
+      POSTGRES_PASSWORD: postgres # CHANGEME
       POSTGRES_DB: postgres
     ports:
-      - 5432:5432
+      - 127.0.0.1:5432:5432
     volumes:
       - langfuse_postgres_data:/var/lib/postgresql/data
 

ee/package.json

@@ -26,9 +26,9 @@
   "dependencies": {
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "axios": "^1.7.7",
+    "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
-    "next": "^14.2.21",
+    "next": "^14.2.25",
     "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },

fern/apis/server/definition/annotation-queues.yml

@@ -0,0 +1,165 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    listQueues:
+      docs: Get all annotation queues
+      method: GET
+      path: /annotation-queues
+      request:
+        name: GetAnnotationQueuesRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedAnnotationQueues
+
+    getQueue:
+      docs: Get an annotation queue by ID
+      method: GET
+      path: /annotation-queues/{queueId}
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      response: AnnotationQueue
+
+    listQueueItems:
+      docs: Get items for a specific annotation queue
+      method: GET
+      path: /annotation-queues/{queueId}/items
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request:
+        name: GetAnnotationQueueItemsRequest
+        query-parameters:
+          status:
+            type: optional<AnnotationQueueStatus>
+            docs: Filter by status
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedAnnotationQueueItems
+
+    getQueueItem:
+      docs: Get a specific item from an annotation queue
+      method: GET
+      path: /annotation-queues/{queueId}/items/{itemId}
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+        itemId:
+          type: string
+          docs: The unique identifier of the annotation queue item
+      response: AnnotationQueueItem
+
+    createQueueItem:
+      docs: Add an item to an annotation queue
+      method: POST
+      path: /annotation-queues/{queueId}/items
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: CreateAnnotationQueueItemRequest
+      response: AnnotationQueueItem
+
+    updateQueueItem:
+      docs: Update an annotation queue item
+      method: PATCH
+      path: /annotation-queues/{queueId}/items/{itemId}
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+        itemId:
+          type: string
+          docs: The unique identifier of the annotation queue item
+      request: UpdateAnnotationQueueItemRequest
+      response: AnnotationQueueItem
+
+    deleteQueueItem:
+      docs: Remove an item from an annotation queue
+      method: DELETE
+      path: /annotation-queues/{queueId}/items/{itemId}
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+        itemId:
+          type: string
+          docs: The unique identifier of the annotation queue item
+      response: DeleteAnnotationQueueItemResponse
+
+types:
+  AnnotationQueueStatus:
+    enum:
+      - PENDING
+      - COMPLETED
+
+  AnnotationQueueObjectType:
+    enum:
+      - TRACE
+      - OBSERVATION
+
+  AnnotationQueue:
+    properties:
+      id: string
+      name: string
+      description: optional<string>
+      scoreConfigIds: list<string>
+      createdAt: datetime
+      updatedAt: datetime
+
+  AnnotationQueueItem:
+    properties:
+      id: string
+      queueId: string
+      objectId: string
+      objectType: AnnotationQueueObjectType
+      status: AnnotationQueueStatus
+      completedAt: optional<datetime>
+      createdAt: datetime
+      updatedAt: datetime
+
+  PaginatedAnnotationQueues:
+    properties:
+      data: list<AnnotationQueue>
+      meta: pagination.MetaResponse
+
+  PaginatedAnnotationQueueItems:
+    properties:
+      data: list<AnnotationQueueItem>
+      meta: pagination.MetaResponse
+
+  CreateAnnotationQueueItemRequest:
+    properties:
+      objectId: string
+      objectType: AnnotationQueueObjectType
+      status:
+        type: optional<AnnotationQueueStatus>
+        docs: Defaults to PENDING for new queue items
+
+  UpdateAnnotationQueueItemRequest:
+    properties:
+      status:
+        type: optional<AnnotationQueueStatus>
+
+  DeleteAnnotationQueueItemResponse:
+    properties:
+      success: boolean
+      message: string

fern/apis/server/definition/ingestion.yml

@@ -394,16 +394,27 @@ types:
       successes: list<IngestionSuccess>
       errors: list<IngestionError>
 
-  OpenAIUsageSchema:
+  OpenAICompletionUsageSchema:
+    docs: OpenAI Usage schema from (Chat-)Completion APIs
     properties:
       prompt_tokens: integer
       completion_tokens: integer
       total_tokens: integer
-      prompt_tokens_details: optional<map<string, integer>>
-      completion_tokens_details: optional<map<string, integer>>
+      prompt_tokens_details: optional<map<string, optional<integer>>>
+      completion_tokens_details: optional<map<string, optional<integer>>>
+
+  OpenAIResponseUsageSchema:
+    docs: OpenAI Usage schema from Response API
+    properties:
+      input_tokens: integer
+      output_tokens: integer
+      total_tokens: integer
+      input_tokens_details: optional<map<string, optional<integer>>>
+      output_tokens_details: optional<map<string, optional<integer>>>
 
   UsageDetails:
     discriminated: false
     union:
       - map<string, integer>
-      - OpenAIUsageSchema
+      - OpenAICompletionUsageSchema
+      - OpenAIResponseUsageSchema

fern/apis/server/definition/prompts.yml

@@ -128,6 +128,9 @@ types:
       commitMessage:
         type: optional<string>
         docs: Commit message for this prompt version.
+      resolutionGraph:
+        type: optional<map<string, unknown>>
+        docs: The dependency resolution graph for the current prompt. Null if prompt has no dependencies.
 
   ChatMessage:
     properties:

fern/apis/server/definition/trace.yml

@@ -15,6 +15,15 @@ service:
           type: string
           docs: The unique langfuse identifier of a trace
       response: commons.TraceWithFullDetails
+    delete:
+      docs: Delete a specific trace
+      method: DELETE
+      path: /traces/{traceId}
+      path-parameters:
+        traceId:
+          type: string
+          docs: The unique langfuse identifier of the trace to delete
+      response: DeleteTraceResponse
     list:
       docs: Get list of traces
       method: GET
@@ -55,12 +64,27 @@ service:
             allow-multiple: true
             docs: Optional filter for traces where the environment is one of the provided values.
       response: Traces
+    deleteMultiple:
+      docs: Delete multiple traces
+      method: DELETE
+      path: /traces
+      request:
+        name: DeleteTracesRequest
+        body:
+          properties:
+            traceIds:
+              type: list<string>
+              docs: List of trace IDs to delete
+      response: DeleteTraceResponse
 
 types:
   Traces:
     properties:
       data: list<commons.TraceWithDetails>
       meta: pagination.MetaResponse
+  DeleteTraceResponse:
+    properties:
+      message: string
   Sort:
     properties:
       id: string

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.35.1",
+  "version": "3.46.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/clickhouse/migrations/clustered/0009_add_project_environments.down.sql

@@ -0,0 +1,7 @@
+-- Drop materialized views
+DROP VIEW IF EXISTS project_environments_traces_mv ON CLUSTER default;
+DROP VIEW IF EXISTS project_environments_observations_mv ON CLUSTER default;
+DROP VIEW IF EXISTS project_environments_scores_mv ON CLUSTER default;
+
+-- Drop the project_environments table
+DROP TABLE IF EXISTS project_environments ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0009_add_project_environments.up.sql

@@ -0,0 +1,30 @@
+-- Create the project_environments table with AggregatingMergeTree engine
+CREATE TABLE project_environments ON CLUSTER default (
+    `project_id` String,
+    `environments` SimpleAggregateFunction(groupUniqArrayArray, Array(String))
+) ENGINE = ReplicatedAggregatingMergeTree
+ORDER BY (project_id);
+
+-- Create materialized view for traces
+CREATE MATERIALIZED VIEW project_environments_traces_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM traces
+GROUP BY project_id;
+
+-- Create materialized view for observations
+CREATE MATERIALIZED VIEW project_environments_observations_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM observations
+GROUP BY project_id;
+
+-- Create materialized view for scores
+CREATE MATERIALIZED VIEW project_environments_scores_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM scores
+GROUP BY project_id;

packages/shared/clickhouse/migrations/unclustered/0009_add_project_environments.down.sql

@@ -0,0 +1,7 @@
+-- Drop materialized views
+DROP VIEW IF EXISTS project_environments_traces_mv;
+DROP VIEW IF EXISTS project_environments_observations_mv;
+DROP VIEW IF EXISTS project_environments_scores_mv;
+
+-- Drop the project_environments table
+DROP TABLE IF EXISTS project_environments;

packages/shared/clickhouse/migrations/unclustered/0009_add_project_environments.up.sql

@@ -0,0 +1,30 @@
+-- Create the project_environments table with AggregatingMergeTree engine
+CREATE TABLE project_environments (
+    `project_id` String,
+    `environments` SimpleAggregateFunction(groupUniqArrayArray, Array(String))
+) ENGINE = AggregatingMergeTree
+ORDER BY (project_id);
+
+-- Create materialized view for traces
+CREATE MATERIALIZED VIEW project_environments_traces_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM traces
+GROUP BY project_id;
+
+-- Create materialized view for observations
+CREATE MATERIALIZED VIEW project_environments_observations_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM observations
+GROUP BY project_id;
+
+-- Create materialized view for scores
+CREATE MATERIALIZED VIEW project_environments_scores_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM scores
+GROUP BY project_id;

packages/shared/clickhouse/scripts/seed.ts

@@ -1,6 +1,5 @@
-import { ObservationRecordReadType } from "@langfuse/shared/src/server";
 import { prisma } from "../../src/db";
-import { redis } from "@langfuse/shared/src/server";
+import { ObservationRecordReadType, redis } from "../../src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
 import { createDatasets } from "../../prisma/seed";
 import { queryClickhouse } from "../../src/server/repositories/clickhouse";

packages/shared/package.json

@@ -47,7 +47,7 @@
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
     "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed",
-    "ch:seed": "dotenv -e ../../.env -- tsx clickhouse/scripts/seed.ts",
+    "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' clickhouse/scripts/seed.ts",
     "load:setup": "dotenv -e ../../.env -- tsx scripts/load-seed.ts"
   },
   "prisma": {
@@ -61,6 +61,7 @@
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
+    "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.12",
     "@langchain/aws": "^0.1.3",
     "@langchain/core": "^0.3.37",
@@ -72,7 +73,7 @@
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
-    "axios": "^1.7.7",
+    "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.34.10",
     "dd-trace": "^5.36.0",
@@ -84,6 +85,7 @@
     "langchain": "^0.3.15",
     "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
+    "lossless-json": "^4.0.2",
     "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",

packages/shared/prisma/generated/types.ts

@@ -59,6 +59,11 @@ export const CommentObjectType = {
     PROMPT: "PROMPT"
 } as const;
 export type CommentObjectType = (typeof CommentObjectType)[keyof typeof CommentObjectType];
+export const AuditLogRecordType = {
+    USER: "USER",
+    API_KEY: "API_KEY"
+} as const;
+export type AuditLogRecordType = (typeof AuditLogRecordType)[keyof typeof AuditLogRecordType];
 export const JobType = {
     EVAL: "EVAL"
 } as const;
@@ -75,6 +80,12 @@ export const JobExecutionStatus = {
     CANCELLED: "CANCELLED"
 } as const;
 export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
+export const BlobStorageIntegrationType = {
+    S3: "S3",
+    S3_COMPATIBLE: "S3_COMPATIBLE",
+    AZURE_BLOB_STORAGE: "AZURE_BLOB_STORAGE"
+} as const;
+export type BlobStorageIntegrationType = (typeof BlobStorageIntegrationType)[keyof typeof BlobStorageIntegrationType];
 export type Account = {
     id: string;
     user_id: string;
@@ -132,9 +143,11 @@ export type AuditLog = {
     id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
-    user_id: string;
+    type: Generated<AuditLogRecordType>;
+    api_key_id: string | null;
+    user_id: string | null;
     org_id: string;
-    user_org_role: string;
+    user_org_role: string | null;
     project_id: string | null;
     user_project_role: string | null;
     resource_type: string;
@@ -181,6 +194,23 @@ export type BillingMeterBackup = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type BlobStorageIntegration = {
+    project_id: string;
+    type: BlobStorageIntegrationType;
+    bucket_name: string;
+    prefix: string;
+    access_key_id: string;
+    secret_access_key: string;
+    region: string;
+    endpoint: string | null;
+    force_path_style: boolean;
+    next_sync_at: Timestamp | null;
+    last_sync_at: Timestamp | null;
+    enabled: boolean;
+    export_frequency: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Comment = {
     id: string;
     project_id: string;
@@ -371,6 +401,24 @@ export type LlmApiKeys = {
     config: unknown | null;
     project_id: string;
 };
+export type LlmSchema = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    schema: unknown;
+};
+export type LlmTool = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    parameters: unknown;
+};
 export type Media = {
     id: string;
     sha_256_hash: string;
@@ -485,12 +533,15 @@ export type Prompt = {
     labels: Generated<string[]>;
     commit_message: string | null;
 };
-export type QueueBackUp = {
+export type PromptDependency = {
     id: string;
-    project_id: string | null;
-    queue_name: string;
-    content: unknown;
     created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    parent_id: string;
+    child_name: string;
+    child_label: string | null;
+    child_version: number | null;
 };
 export type ScoreConfig = {
     id: string;
@@ -562,6 +613,7 @@ export type DB = {
     background_migrations: BackgroundMigration;
     batch_exports: BatchExport;
     billing_meter_backups: BillingMeterBackup;
+    blob_storage_integrations: BlobStorageIntegration;
     comments: Comment;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
@@ -572,6 +624,8 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    llm_schemas: LlmSchema;
+    llm_tools: LlmTool;
     media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
@@ -583,8 +637,8 @@ export type DB = {
     prices: Price;
     project_memberships: ProjectMembership;
     projects: Project;
+    prompt_dependencies: PromptDependency;
     prompts: Prompt;
-    queue_backups: QueueBackUp;
     score_configs: ScoreConfig;
     scores: LegacyPrismaScore;
     Session: Session;

packages/shared/prisma/migrations/20250303144044_add_prompt_dependencies_table/migration.sql

@@ -0,0 +1,18 @@
+CREATE TABLE "prompt_dependencies" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "parent_id" TEXT NOT NULL,
+    "child_name" TEXT NOT NULL,
+    "child_label" TEXT,
+    "child_version" INTEGER,
+
+    CONSTRAINT "prompt_dependencies_pkey" PRIMARY KEY ("id")
+);
+
+ALTER TABLE "prompt_dependencies" ADD CONSTRAINT "prompt_dependencies_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "prompt_dependencies" ADD CONSTRAINT "prompt_dependencies_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "prompts"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+CREATE INDEX "prompt_dependencies_project_id_parent_id" ON "prompt_dependencies"("project_id", "parent_id");
+CREATE INDEX "prompt_dependencies_project_id_child_name" ON "prompt_dependencies"("project_id", "child_name");

packages/shared/prisma/migrations/20250310100328_add_api_key_to_audit_log/migration.sql

@@ -0,0 +1,11 @@
+-- CreateEnum
+CREATE TYPE "AuditLogRecordType" AS ENUM ('USER', 'API_KEY');
+
+-- AlterTable
+ALTER TABLE "audit_logs" ADD COLUMN     "api_key_id" TEXT,
+ADD COLUMN     "type" "AuditLogRecordType" NOT NULL DEFAULT 'USER',
+ALTER COLUMN "user_id" DROP NOT NULL,
+ALTER COLUMN "user_org_role" DROP NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "audit_logs_api_key_id_idx" ON "audit_logs"("api_key_id");

packages/shared/prisma/migrations/20250321102240_drop_queue_backup_table/migration.sql

@@ -0,0 +1,2 @@
+-- DropTable
+DROP TABLE "queue_backups";

packages/shared/prisma/migrations/20250324110557_add_blobstorage_integration_table/migration.sql

@@ -0,0 +1,26 @@
+-- CreateEnum
+CREATE TYPE "BlobStorageIntegrationType" AS ENUM ('S3', 'S3_COMPATIBLE', 'AZURE_BLOB_STORAGE');
+
+-- CreateTable
+CREATE TABLE "blob_storage_integrations" (
+    "project_id" TEXT NOT NULL,
+    "type" "BlobStorageIntegrationType" NOT NULL,
+    "bucket_name" TEXT NOT NULL,
+    "prefix" TEXT NOT NULL,
+    "access_key_id" TEXT NOT NULL,
+    "secret_access_key" TEXT NOT NULL,
+    "region" TEXT NOT NULL,
+    "endpoint" TEXT,
+    "force_path_style" BOOLEAN NOT NULL,
+    "next_sync_at" TIMESTAMP(3),
+    "last_sync_at" TIMESTAMP(3),
+    "enabled" BOOLEAN NOT NULL,
+    "export_frequency" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "blob_storage_integrations_pkey" PRIMARY KEY ("project_id")
+);
+
+-- AddForeignKey
+ALTER TABLE "blob_storage_integrations" ADD CONSTRAINT "blob_storage_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250326180640_add_llm_tools_and_schemas_tables/migration.sql

@@ -0,0 +1,37 @@
+-- CreateTable
+CREATE TABLE "llm_schemas" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "schema" JSON NOT NULL,
+
+    CONSTRAINT "llm_schemas_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "llm_tools" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "parameters" JSON NOT NULL,
+
+    CONSTRAINT "llm_tools_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_schemas_project_id_name_key" ON "llm_schemas"("project_id", "name");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_tools_project_id_name_key" ON "llm_tools"("project_id", "name");
+
+-- AddForeignKey
+ALTER TABLE "llm_schemas" ADD CONSTRAINT "llm_schemas_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "llm_tools" ADD CONSTRAINT "llm_tools_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -109,37 +109,41 @@ model Organization {
 }
 
 model Project {
-  id                  String                    @id @default(cuid())
-  orgId               String                    @map("org_id")
-  createdAt           DateTime                  @default(now()) @map("created_at")
-  updatedAt           DateTime                  @default(now()) @updatedAt @map("updated_at")
-  deletedAt           DateTime?                 @map("deleted_at")
-  name                String
-  retentionDays       Int?                      @map("retention_days")
-  projectMembers      ProjectMembership[]
-  organization        Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
-  apiKeys             ApiKey[]
-  dataset             Dataset[]
-  invitations         MembershipInvitation[]
-  sessions            TraceSession[]
-  Prompt              Prompt[]
-  Model               Model[]
-  EvalTemplate        EvalTemplate[]
-  JobConfiguration    JobConfiguration[]
-  JobExecution        JobExecution[]
-  LlmApiKeys          LlmApiKeys[]
-  PosthogIntegration  PosthogIntegration[]
-  scoreConfig         ScoreConfig[]
-  BatchExport         BatchExport[]
-  comment             Comment[]
-  annotationQueue     AnnotationQueue[]
-  annotationQueueItem AnnotationQueueItem[]
-  TraceMedia          TraceMedia[]
-  Media               Media[]
-  ObservationMedia    ObservationMedia[]
-  LegacyTrace         LegacyPrismaTrace[]
-  LegacyObservation   LegacyPrismaObservation[]
-  LegacyScore         LegacyPrismaScore[]
+  id                     String                    @id @default(cuid())
+  orgId                  String                    @map("org_id")
+  createdAt              DateTime                  @default(now()) @map("created_at")
+  updatedAt              DateTime                  @default(now()) @updatedAt @map("updated_at")
+  deletedAt              DateTime?                 @map("deleted_at")
+  name                   String
+  retentionDays          Int?                      @map("retention_days")
+  projectMembers         ProjectMembership[]
+  organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+  apiKeys                ApiKey[]
+  dataset                Dataset[]
+  invitations            MembershipInvitation[]
+  sessions               TraceSession[]
+  Prompt                 Prompt[]
+  Model                  Model[]
+  EvalTemplate           EvalTemplate[]
+  JobConfiguration       JobConfiguration[]
+  JobExecution           JobExecution[]
+  LlmApiKeys             LlmApiKeys[]
+  PosthogIntegration     PosthogIntegration[]
+  BlobStorageIntegration BlobStorageIntegration[]
+  scoreConfig            ScoreConfig[]
+  BatchExport            BatchExport[]
+  comment                Comment[]
+  annotationQueue        AnnotationQueue[]
+  annotationQueueItem    AnnotationQueueItem[]
+  TraceMedia             TraceMedia[]
+  Media                  Media[]
+  ObservationMedia       ObservationMedia[]
+  LegacyTrace            LegacyPrismaTrace[]
+  LegacyObservation      LegacyPrismaObservation[]
+  LegacyScore            LegacyPrismaScore[]
+  PromptDependency       PromptDependency[]
+  LlmSchema              LlmSchema[]
+  LlmTool                LlmTool[]
 
   @@index([orgId])
   @@map("projects")
@@ -293,6 +297,7 @@ model LegacyPrismaTrace {
   release    String?
   version    String?
   projectId  String   @map("project_id")
+  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
   public     Boolean  @default(false)
   bookmarked Boolean  @default(false)
   tags       String[] @default([])
@@ -301,7 +306,6 @@ model LegacyPrismaTrace {
   sessionId  String?  @map("session_id")
   createdAt  DateTime @default(now()) @map("created_at")
   updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")
-  project    Project  @relation(fields: [projectId], references: [id])
 
   @@index([projectId, timestamp])
   @@index([sessionId])
@@ -318,6 +322,7 @@ model LegacyPrismaObservation {
   id                   String                       @id @default(cuid())
   traceId              String?                      @map("trace_id")
   projectId            String                       @map("project_id")
+  project              Project                      @relation(fields: [projectId], references: [id], onDelete: Cascade)
   type                 LegacyPrismaObservationType
   startTime            DateTime                     @default(now()) @map("start_time")
   endTime              DateTime?                    @map("end_time")
@@ -349,7 +354,6 @@ model LegacyPrismaObservation {
   calculatedOutputCost Decimal?                     @map("calculated_output_cost")
   calculatedTotalCost  Decimal?                     @map("calculated_total_cost")
   completionStartTime  DateTime?                    @map("completion_start_time")
-  project              Project                      @relation(fields: [projectId], references: [id], onDelete: Cascade)
   promptId             String?                      @map("prompt_id") // no fk constraint, prompt can be deleted
 
   @@unique([id, projectId])
@@ -371,6 +375,8 @@ enum LegacyPrismaObservationType {
   SPAN
   EVENT
   GENERATION
+
+  @@map("ObservationType")
 }
 
 enum LegacyPrismaObservationLevel {
@@ -378,12 +384,15 @@ enum LegacyPrismaObservationLevel {
   DEFAULT
   WARNING
   ERROR
+
+  @@map("ObservationLevel")
 }
 
 model LegacyPrismaScore {
   id            String                  @id @default(cuid())
   timestamp     DateTime                @default(now())
   projectId     String                  @map("project_id")
+  project       Project                 @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        LegacyPrismaScoreSource
@@ -398,7 +407,6 @@ model LegacyPrismaScore {
   updatedAt     DateTime                @default(now()) @updatedAt @map("updated_at")
   dataType      ScoreDataType           @default(NUMERIC) @map("data_type")
   scoreConfig   ScoreConfig?            @relation(fields: [configId], references: [id], onDelete: SetNull)
-  project       Project                 @relation(fields: [projectId], references: [id])
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -417,6 +425,8 @@ enum LegacyPrismaScoreSource {
   ANNOTATION
   API
   EVAL
+
+  @@map("ScoreSource")
 }
 
 model ScoreConfig {
@@ -634,15 +644,16 @@ model Prompt {
 
   createdBy String @map("created_by")
 
-  prompt        Json
-  name          String
-  version       Int
-  type          String   @default("text")
-  isActive      Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-  config        Json     @default("{}") @db.Json
-  tags          String[] @default([])
-  labels        String[] @default([])
-  commitMessage String?  @map("commit_message")
+  prompt           Json
+  name             String
+  version          Int
+  type             String             @default("text")
+  isActive         Boolean?           @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+  config           Json               @default("{}") @db.Json
+  tags             String[]           @default([])
+  labels           String[]           @default([])
+  commitMessage    String?            @map("commit_message")
+  PromptDependency PromptDependency[]
 
   @@unique([projectId, name, version])
   @@index([projectId, id])
@@ -652,6 +663,26 @@ model Prompt {
   @@map("prompts")
 }
 
+model PromptDependency {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  parentId String @map("parent_id")
+  parent   Prompt @relation(fields: [parentId], references: [id], onDelete: Cascade)
+
+  childName    String  @map("child_name")
+  childLabel   String? @map("child_label")
+  childVersion Int?    @map("child_version")
+
+  @@index([projectId, parentId], map: "prompt_dependencies_project_id_parent_id")
+  @@index([projectId, childName], map: "prompt_dependencies_project_id_child_name")
+  @@map("prompt_dependencies")
+}
+
 // Update ObservationView below when making changes to this model!
 model Model {
   id        String   @id @default(cuid())
@@ -690,23 +721,31 @@ model Price {
   @@map("prices")
 }
 
+enum AuditLogRecordType {
+  USER
+  API_KEY
+}
+
 // No FK constraints to preserve audit logs
 model AuditLog {
-  id              String   @id @default(cuid())
-  createdAt       DateTime @default(now()) @map("created_at")
-  updatedAt       DateTime @default(now()) @updatedAt @map("updated_at")
-  userId          String   @map("user_id")
-  orgId           String   @map("org_id")
-  userOrgRole     String   @map("user_org_role")
-  projectId       String?  @map("project_id")
-  userProjectRole String?  @map("user_project_role")
-  resourceType    String   @map("resource_type")
-  resourceId      String   @map("resource_id")
+  id              String             @id @default(cuid())
+  createdAt       DateTime           @default(now()) @map("created_at")
+  updatedAt       DateTime           @default(now()) @updatedAt @map("updated_at")
+  type            AuditLogRecordType @default(USER)
+  apiKeyId        String?            @map("api_key_id")
+  userId          String?            @map("user_id")
+  orgId           String             @map("org_id")
+  userOrgRole     String?            @map("user_org_role")
+  projectId       String?            @map("project_id")
+  userProjectRole String?            @map("user_project_role")
+  resourceType    String             @map("resource_type")
+  resourceId      String             @map("resource_id")
   action          String
-  before          String? //stringified JSON
+  before          String? // stringified JSON
   after           String? // stringified JSON
 
   @@index([projectId])
+  @@index([apiKeyId])
   @@index([userId])
   @@index([orgId])
   @@index([createdAt])
@@ -822,9 +861,9 @@ model SsoConfig {
   authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
 
   authConfig Json? @map("auth_config")
+
   // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
   // secrets like clientSecret are encrypted on the application level
-
   @@map("sso_configs")
 }
 
@@ -840,6 +879,40 @@ model PosthogIntegration {
   @@map("posthog_integrations")
 }
 
+model BlobStorageIntegration {
+  projectId String  @id @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // Blob Storage Config
+  type            BlobStorageIntegrationType @map("type")
+  bucketName      String                     @map("bucket_name")
+  prefix          String                     @map("prefix")
+  accessKeyId     String                     @map("access_key_id")
+  secretAccessKey String                     @map("secret_access_key")
+  region          String                     @map("region")
+  endpoint        String?                    @map("endpoint")
+  forcePathStyle  Boolean                    @map("force_path_style")
+
+  // Integration Config
+  nextSyncAt      DateTime? @map("next_sync_at")
+  lastSyncAt      DateTime? @map("last_sync_at")
+  enabled         Boolean
+  exportFrequency String    @map("export_frequency")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@map("blob_storage_integrations")
+}
+
+enum BlobStorageIntegrationType {
+  S3
+  S3_COMPATIBLE
+  AZURE_BLOB_STORAGE
+
+  @@map("BlobStorageIntegrationType")
+}
+
 model BatchExport {
   id        String   @id @default(cuid())
   createdAt DateTime @default(now()) @map("created_at")
@@ -917,17 +990,6 @@ model ObservationMedia {
   @@map("observation_media")
 }
 
-model QueueBackUp {
-  id        String  @id @default(cuid())
-  projectId String? @map("project_id")
-  queueName String  @map("queue_name")
-  content   Json
-
-  createdAt DateTime @default(now()) @map("created_at")
-
-  @@map("queue_backups")
-}
-
 model BillingMeterBackup {
   // unique
   stripeCustomerId String   @map("stripe_customer_id")
@@ -950,3 +1012,35 @@ model BillingMeterBackup {
   @@index([stripeCustomerId, meterId, startTime, endTime])
   @@map("billing_meter_backups")
 }
+
+model LlmSchema {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  schema      Json   @map("schema") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_schemas")
+}
+
+model LlmTool {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  parameters  Json   @map("parameters") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_tools")
+}

packages/shared/prisma/seed.ts

@@ -72,6 +72,9 @@ async function main() {
     where: { id: seedOrgId },
     update: {
       name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
     },
     create: {
       id: seedOrgId,
@@ -371,6 +374,46 @@ async function main() {
     });
 
     await createDatasets(project1, project2, observations);
+
+    await prisma.llmSchema.createMany({
+      data: [
+        {
+          projectId: project1.id,
+          name: "get_weather",
+          description: "Fetches weather in Celsius for a given location",
+          schema: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+              unit: {
+                type: "string",
+                enum: ["celsius", "fahrenheit"],
+              },
+            },
+            required: ["location", "unit"],
+          },
+        },
+        {
+          projectId: project1.id,
+          name: "calculator",
+          description: "Performs basic arithmetic calculations",
+          schema: {
+            type: "object",
+            properties: {
+              expression: {
+                type: "string",
+                description:
+                  "The mathematical expression to evaluate, e.g. '2 + 2'",
+              },
+            },
+            required: ["expression"],
+          },
+        },
+      ],
+    });
   }
 }
 

packages/shared/scripts/load-seed.ts

@@ -27,6 +27,9 @@ const prepareProjectsAndApiKeys = async (
       create: {
         id: orgId,
         name: `Organization for ${projectId}`,
+        cloudConfig: {
+          plan: "Team",
+        },
       },
     });
 

packages/shared/scripts/prepareClickhouse.ts

@@ -145,7 +145,7 @@ export const prepareClickhouse = async (
 
     const scoresQuery = `
     INSERT INTO scores
-    SELECT toString(floor(randUniform(0, 100))) AS id,
+    SELECT toString(number) AS id,
       toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
       '${projectId}' AS project_id,
       'default' AS environment,

packages/shared/src/env.ts

@@ -10,7 +10,7 @@ const EnvSchema = z.object({
   REDIS_PORT: z.coerce
     .number({
       description:
-        ".env files convert numbers to strings, therefoore we have to enforce them to be numbers",
+        ".env files convert numbers to strings, therefore we have to enforce them to be numbers",
     })
     .positive()
     .max(65536, `options.port should be >= 0 and < 65536`)
@@ -50,6 +50,7 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_S3_CONCURRENT_WRITES: z.coerce.number().positive().default(50),
   LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
     required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
   }),
@@ -62,6 +63,8 @@ const EnvSchema = z.object({
     .enum(["true", "false"])
     .default("false"),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
   LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
@@ -70,6 +73,10 @@ const EnvSchema = z.object({
   LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
+
+  LANGFUSE_CUSTOM_SSO_EMAIL_CLAIM: z.string().default("email"),
+  LANGFUSE_CUSTOM_SSO_NAME_CLAIM: z.string().default("name"),
+  LANGFUSE_CUSTOM_SSO_SUB_CLAIM: z.string().default("sub"),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/batchAction/types.ts

@@ -10,6 +10,7 @@ export enum BatchActionType {
 }
 
 const ActionIdSchema = z.enum([
+  "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
 ]);

packages/shared/src/features/batchExport/types.ts

@@ -18,9 +18,10 @@ export enum BatchExportFileFormat {
 }
 
 export enum BatchExportTableName {
+  Scores = "scores",
   Sessions = "sessions",
   Traces = "traces",
-  Generations = "generations",
+  Observations = "observations",
   DatasetRunItems = "dataset_run_items",
 }
 

packages/shared/src/features/entitlements/plans.ts

@@ -1,8 +1,19 @@
+// used on organization.cloudConfig.plan
+export const cloudConfigPlans = [
+  "Hobby",
+  "Core",
+  "Pro",
+  "Team",
+  "Enterprise",
+] as const;
+
 export const planLabels = {
   oss: "OSS",
   "cloud:hobby": "Hobby",
+  "cloud:core": "Core",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
+  "cloud:enterprise": "Enterprise",
   "self-hosted:pro": "Pro (self-hosted)",
   "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
@@ -12,8 +23,9 @@ export type Plan = keyof typeof planLabels;
 export const plans = Object.keys(planLabels) as Plan[];
 
 // These functions are kept here to ensure consistency when updating plan names in the future.
-export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
-export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
+export const isCloudPlan = (plan?: Plan) => plan?.startsWith("cloud");
+export const isSelfHostedPlan = (plan?: Plan) =>
+  plan?.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/features/prompts/parsePromptDependencyTags.ts

@@ -0,0 +1,61 @@
+import { z } from "zod";
+
+export const PromptDependencyRegex = /@@@langfusePrompt:(.*?)@@@/g;
+
+export const ParsedPromptDependencySchema = z.union([
+  z.object({
+    name: z.string(),
+    type: z.literal("version"),
+    version: z.coerce.number(),
+  }),
+  z.object({ name: z.string(), type: z.literal("label"), label: z.string() }),
+]);
+
+export type ParsedPromptDependencyTag = z.infer<
+  typeof ParsedPromptDependencySchema
+>;
+
+export function parsePromptDependencyTags(
+  content: string | object,
+): ParsedPromptDependencyTag[] {
+  const matchedTags = JSON.stringify(content).match(PromptDependencyRegex);
+
+  const validTags: ParsedPromptDependencyTag[] = [];
+
+  for (const match of new Set(matchedTags ?? [])) {
+    const innerContent = match.replace(/^@@@langfusePrompt:|@@@$/g, "");
+    const parts = innerContent.split("|");
+    const params: Record<string, string> = {};
+
+    // Check if the first parameter is name
+    const firstPart = parts[0];
+    if (!firstPart || !firstPart.startsWith("name=")) {
+      continue; // Skip this tag if name is not the first parameter. This makes it easier to replace the tag with the resolved prompt.
+    }
+
+    // There can be only 2 parts
+    if (parts.length !== 2) {
+      continue;
+    }
+
+    parts.forEach((part) => {
+      const [key, value] = part.split("=");
+      params[key] = value;
+    });
+
+    if (params.name) {
+      const parsed = ParsedPromptDependencySchema.safeParse({
+        name: params.name,
+        ...(params.version
+          ? { version: params.version, type: "version" }
+          : { label: params.label, type: "label" }),
+      });
+
+      if (parsed.success) {
+        validTags.push(parsed.data);
+      }
+    }
+  }
+
+  return validTags;
+}

packages/shared/src/index.ts

@@ -38,6 +38,9 @@ export * from "./features/comments/types";
 // experiments
 export * from "./features/experiments/utils";
 
+// prompts
+export * from "./features/prompts/parsePromptDependencyTags";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -1,8 +1,9 @@
 import { z } from "zod";
 import { CloudConfigRateLimit } from "./rate-limits";
+import { cloudConfigPlans } from "../features/entitlements/plans";
 
 export const CloudConfigSchema = z.object({
-  plan: z.enum(["Hobby", "Pro", "Team", "Enterprise"]).optional(),
+  plan: z.enum(cloudConfigPlans).optional(),
   monthlyObservationLimit: z.number().int().positive().optional(),
   // used for table and dashboard queries
   defaultLookBackDays: z.number().int().positive().optional(),

packages/shared/src/server/auth/customSsoProvider.ts

@@ -1,4 +1,9 @@
 import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import { env } from "../../env";
+
+const CUSTOM_EMAIL_CLAIM = env.LANGFUSE_CUSTOM_SSO_EMAIL_CLAIM;
+const CUSTOM_NAME_CLAIM = env.LANGFUSE_CUSTOM_SSO_NAME_CLAIM;
+const CUSTOM_SUB_CLAIM = env.LANGFUSE_CUSTOM_SSO_SUB_CLAIM;
 
 interface CustomSSOUser extends Record<string, any> {
   email: string;
@@ -7,6 +12,7 @@ interface CustomSSOUser extends Record<string, any> {
   verified: boolean;
 }
 
+
 export function CustomSSOProvider<P extends CustomSSOUser>(
   options: OAuthUserConfig<P>
 ): OAuthConfig<P> {
@@ -19,9 +25,9 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     checks: ["pkce", "state"],
     profile(profile) {
       return {
-        id: profile.sub,
-        name: profile.name,
-        email: profile.email,
+        id: profile[CUSTOM_SUB_CLAIM],
+        name: profile[CUSTOM_NAME_CLAIM],
+        email: profile[CUSTOM_EMAIL_CLAIM],
         image: null,
       };
     },

packages/shared/src/server/auth/types.ts

@@ -43,4 +43,5 @@ export type ApiAccessScope = {
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
+  apiKeyId: string;
 };

packages/shared/src/server/clickhouse/client.ts

@@ -26,9 +26,11 @@ export const clickhouseClient = (
     database: env.CLICKHOUSE_DB,
     http_headers: headers,
     clickhouse_settings: {
+      ...(params.opts?.clickhouse_settings ?? {}),
       log_comment: JSON.stringify(params.tags ?? {}),
       async_insert: 1,
       wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
+      input_format_json_throw_on_bad_escape_sequence: 0,
     },
   });
 };

packages/shared/src/server/clickhouse/schema.ts

@@ -2,6 +2,11 @@ export const ClickhouseTableNames = {
   traces: "traces",
   observations: "observations",
   scores: "scores",
+
+  // Virtual tables for dashboards
+  // TODO: Check if we can do this more elegantly
+  scores_numeric: "scores_numeric",
+  scores_categorical: "scores_categorical",
 } as const;
 
 export type ClickhouseTableName = keyof typeof ClickhouseTableNames;

packages/shared/src/server/index.ts

@@ -17,7 +17,6 @@ export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
-export * from "./ingestion/modelMatch";
 export * from "./ingestion/processEventBatch";
 export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
@@ -28,12 +27,15 @@ export * from "./redis/cloudUsageMeteringQueue";
 export * from "./redis/getQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
+export * from "./redis/scoreDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
 export * from "./redis/ingestionQueue";
 export * from "./redis/postHogIntegrationQueue";
 export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/blobStorageIntegrationQueue";
+export * from "./redis/blobStorageIntegrationProcessingQueue";
 export * from "./redis/dataRetentionQueue";
 export * from "./redis/dataRetentionProcessingQueue";
 export * from "./redis/coreDataS3ExportQueue";
@@ -49,6 +51,7 @@ export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
+export * from "./services/datasets-ui-table-service";
 
 // test utils
 export * from "./test-utils";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -1,64 +0,0 @@
-import { Model, Prisma } from "@prisma/client";
-import { prisma } from "../../db";
-import { Observation } from "../repositories";
-
-export async function findModel(p: {
-  event: {
-    projectId: string;
-    model?: string;
-    unit?: string;
-    startTime?: Date;
-  };
-  existingDbObservation?: Omit<Observation, "input" | "output">;
-}): Promise<Model | null> {
-  const { event, existingDbObservation } = p;
-  // either get the model from the existing observation
-  // or match pattern on the user provided model name
-  const modelCondition = event.model
-    ? Prisma.sql`AND ${event.model} ~ match_pattern`
-    : existingDbObservation?.internalModel
-      ? Prisma.sql`AND model_name = ${existingDbObservation.internalModel}`
-      : undefined;
-  if (!modelCondition) return null;
-
-  // unit based on the current event or the existing observation, both can be undefined
-  const mergedUnit = event.unit ?? existingDbObservation?.unit;
-
-  const unitCondition = mergedUnit
-    ? Prisma.sql`AND unit = ${mergedUnit}`
-    : Prisma.empty;
-
-  const sql = Prisma.sql`
-    SELECT
-      id,
-      created_at AS "createdAt",
-      updated_at AS "updatedAt",
-      project_id AS "projectId",
-      model_name AS "modelName",
-      match_pattern AS "matchPattern",
-      start_date AS "startDate",
-      input_price AS "inputPrice",
-      output_price AS "outputPrice",
-      total_price AS "totalPrice",
-      unit, 
-      tokenizer_id AS "tokenizerId",
-      tokenizer_config AS "tokenizerConfig"
-    FROM
-      models
-    WHERE (project_id = ${event.projectId}
-      OR project_id IS NULL)
-    ${modelCondition}
-    ${unitCondition}
-    AND (start_date IS NULL OR start_date <= ${
-      event.startTime ? new Date(event.startTime) : new Date()
-    }::timestamp with time zone at time zone 'UTC')
-    ORDER BY
-      project_id ASC,
-      start_date DESC NULLS LAST
-    LIMIT 1
-  `;
-
-  const foundModels = await prisma.$queryRaw<Array<Model>>(sql);
-
-  return foundModels[0] ?? null;
-}

packages/shared/src/server/ingestion/types.ts

@@ -5,6 +5,14 @@ import { NonEmptyString, jsonSchema } from "../../utils/zod";
 import { ModelUsageUnit } from "../../constants";
 import { type ScoreSourceType } from "../repositories";
 
+export const idSchema = z
+  .string()
+  .min(1)
+  .max(800) // AWS S3 allows for 1024 bytes for object keys and we need enough room to construct the entire key/path. https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+  .refine((id) => !id.includes("\r"), {
+    message: "ID cannot contain carriage return characters",
+  });
+
 const ObservationLevel = z.enum(["DEBUG", "DEFAULT", "WARNING", "ERROR"]);
 
 export const Usage = z.object({
@@ -58,21 +66,25 @@ export const usage = MixedUsage.nullish()
   // ensure output is always of new usage model
   .pipe(Usage.nullish());
 
-const RawUsageOrCostDetails = z.record(
+const CostDetails = z
+  .record(z.string(), z.number().nonnegative().nullish())
+  .nullish();
+
+const RawUsageDetails = z.record(
   z.string(),
-  z.number().nonnegative().nullish(),
+  z.number().int().nonnegative().nullish(),
 );
 
-const OpenAIUsageSchema = z
+const OpenAICompletionUsageSchema = z
   .object({
-    prompt_tokens: z.number().nonnegative(),
-    completion_tokens: z.number().nonnegative(),
-    total_tokens: z.number().nonnegative(),
+    prompt_tokens: z.number().int().nonnegative(),
+    completion_tokens: z.number().int().nonnegative(),
+    total_tokens: z.number().int().nonnegative(),
     prompt_tokens_details: z
-      .record(z.string(), z.number().nonnegative())
+      .record(z.string(), z.number().int().nonnegative().nullish())
       .nullish(),
     completion_tokens_details: z
-      .record(z.string(), z.number().nonnegative())
+      .record(z.string(), z.number().int().nonnegative().nullish())
       .nullish(),
   })
   .strict()
@@ -86,7 +98,7 @@ const OpenAIUsageSchema = z
       prompt_tokens_details,
       completion_tokens_details,
     } = v;
-    const result: z.infer<typeof RawUsageOrCostDetails> & {
+    const result: z.infer<typeof RawUsageDetails> & {
       input: number;
       output: number;
       total: number;
@@ -98,24 +110,88 @@ const OpenAIUsageSchema = z
 
     if (prompt_tokens_details) {
       for (const [key, value] of Object.entries(prompt_tokens_details)) {
-        result[`input_${key}`] = value;
-        result.input = Math.max(result.input - (value ?? 0), 0);
+        if (value !== null && value !== undefined) {
+          result[`input_${key}`] = value;
+          result.input = Math.max(result.input - (value ?? 0), 0);
+        }
       }
     }
 
     if (completion_tokens_details) {
       for (const [key, value] of Object.entries(completion_tokens_details)) {
-        result[`output_${key}`] = value;
-        result.output = Math.max(result.output - (value ?? 0), 0);
+        if (value !== null && value !== undefined) {
+          result[`output_${key}`] = value;
+          result.output = Math.max(result.output - (value ?? 0), 0);
+        }
       }
     }
 
     return result;
   })
-  .pipe(RawUsageOrCostDetails);
+  .pipe(RawUsageDetails);
 
-export const UsageOrCostDetails = z
-  .union([OpenAIUsageSchema, RawUsageOrCostDetails])
+// The new OpenAI Response API uses a new Usage schema that departs from the Completion API Usage schema
+const OpenAIResponseUsageSchema = z
+  .object({
+    input_tokens: z.number().int().nonnegative(),
+    output_tokens: z.number().int().nonnegative(),
+    total_tokens: z.number().int().nonnegative(),
+    input_tokens_details: z
+      .record(z.string(), z.number().int().nonnegative().nullish())
+      .nullish(),
+    output_tokens_details: z
+      .record(z.string(), z.number().int().nonnegative().nullish())
+      .nullish(),
+  })
+  .strict()
+  .transform((v) => {
+    if (!v) return;
+
+    const {
+      input_tokens,
+      output_tokens,
+      total_tokens,
+      input_tokens_details,
+      output_tokens_details,
+    } = v;
+    const result: z.infer<typeof RawUsageDetails> & {
+      input: number;
+      output: number;
+      total: number;
+    } = {
+      input: input_tokens,
+      output: output_tokens,
+      total: total_tokens,
+    };
+
+    if (input_tokens_details) {
+      for (const [key, value] of Object.entries(input_tokens_details)) {
+        if (value !== null && value !== undefined) {
+          result[`input_${key}`] = value;
+          result.input = Math.max(result.input - (value ?? 0), 0);
+        }
+      }
+    }
+
+    if (output_tokens_details) {
+      for (const [key, value] of Object.entries(output_tokens_details)) {
+        if (value !== null && value !== undefined) {
+          result[`output_${key}`] = value;
+          result.output = Math.max(result.output - (value ?? 0), 0);
+        }
+      }
+    }
+
+    return result;
+  })
+  .pipe(RawUsageDetails);
+
+export const UsageDetails = z
+  .union([
+    OpenAICompletionUsageSchema,
+    OpenAIResponseUsageSchema,
+    RawUsageDetails,
+  ])
   .nullish();
 
 export const EnvironmentName = z
@@ -131,7 +207,7 @@ export const EnvironmentName = z
 // With this setup parsing should be more lightweight and doesn't block other requests.
 // As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 export const TraceBody = z.object({
-  id: z.string().nullish(),
+  id: idSchema.nullish(),
   timestamp: stringDateTime,
   name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
@@ -148,7 +224,7 @@ export const TraceBody = z.object({
 });
 
 export const OptionalObservationBody = z.object({
-  traceId: z.string().nullish(),
+  traceId: idSchema.nullish(),
   environment: EnvironmentName,
   name: z.string().nullish(),
   startTime: stringDateTime,
@@ -162,11 +238,11 @@ export const OptionalObservationBody = z.object({
 });
 
 export const CreateEventEvent = OptionalObservationBody.extend({
-  id: NonEmptyString,
+  id: idSchema,
 });
 
 export const UpdateEventEvent = OptionalObservationBody.extend({
-  id: NonEmptyString,
+  id: idSchema,
 });
 
 export const CreateSpanBody = CreateEventEvent.extend({
@@ -195,8 +271,8 @@ export const CreateGenerationBody = CreateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
-  usageDetails: UsageOrCostDetails,
-  costDetails: UsageOrCostDetails,
+  usageDetails: UsageDetails,
+  costDetails: CostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -225,8 +301,8 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
-  usageDetails: UsageOrCostDetails,
-  costDetails: UsageOrCostDetails,
+  usageDetails: UsageDetails,
+  costDetails: CostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -238,7 +314,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
 });
 
 const BaseScoreBody = z.object({
-  id: z.string().nullish(),
+  id: idSchema.nullish(),
   name: NonEmptyString,
   traceId: z.string(),
   environment: EnvironmentName,
@@ -288,8 +364,8 @@ export const ScoreBody = z.discriminatedUnion("dataType", [
 
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
-  id: z.string().nullish(),
-  traceId: z.string().nullish(),
+  id: idSchema.nullish(),
+  traceId: idSchema.nullish(),
   name: z.string().nullish(),
   startTime: stringDateTime,
   endTime: stringDateTime,
@@ -303,8 +379,8 @@ export const LegacySpanPostSchema = z.object({
 });
 
 export const LegacySpanPatchSchema = z.object({
-  spanId: z.string(),
-  traceId: z.string().nullish(),
+  spanId: idSchema,
+  traceId: idSchema.nullish(),
   name: z.string().nullish(),
   startTime: stringDateTime,
   endTime: stringDateTime,
@@ -317,8 +393,8 @@ export const LegacySpanPatchSchema = z.object({
 });
 
 export const LegacyGenerationsCreateSchema = z.object({
-  id: z.string().nullish(),
-  traceId: z.string().nullish(),
+  id: idSchema.nullish(),
+  traceId: idSchema.nullish(),
   name: z.string().nullish(),
   startTime: stringDateTime,
   endTime: stringDateTime,
@@ -341,8 +417,8 @@ export const LegacyGenerationsCreateSchema = z.object({
 });
 
 export const LegacyGenerationPatchSchema = z.object({
-  generationId: z.string(),
-  traceId: z.string().nullish(),
+  generationId: idSchema,
+  traceId: idSchema.nullish(),
   name: z.string().nullish(),
   startTime: stringDateTime,
   endTime: stringDateTime,
@@ -364,8 +440,8 @@ export const LegacyGenerationPatchSchema = z.object({
 });
 
 export const LegacyObservationBody = z.object({
-  id: z.string().nullish(),
-  traceId: z.string().nullish(),
+  id: idSchema.nullish(),
+  traceId: idSchema.nullish(),
   type: z.enum(["GENERATION", "SPAN", "EVENT"]),
   name: z.string().nullish(),
   startTime: stringDateTime,
@@ -381,8 +457,8 @@ export const LegacyObservationBody = z.object({
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
   usage: usage,
-  usageDetails: UsageOrCostDetails,
-  costDetails: UsageOrCostDetails,
+  usageDetails: UsageDetails,
+  costDetails: CostDetails,
   metadata: jsonSchema.nullish(),
   parentObservationId: z.string().nullish(),
   level: ObservationLevel.nullish(),
@@ -410,7 +486,7 @@ export const eventTypes = {
 } as const;
 
 const base = z.object({
-  id: z.string(),
+  id: idSchema,
   timestamp: z.string().datetime({ offset: true }),
   metadata: jsonSchema.nullish(),
 });
@@ -473,13 +549,6 @@ export const ingestionEvent = z.discriminatedUnion("type", [
 ]);
 export type IngestionEventType = z.infer<typeof ingestionEvent>;
 
-export const ingestionBatchEvent = z.array(ingestionEvent);
-
-export const ingestionApiSchema = z.object({
-  batch: ingestionBatchEvent,
-  metadata: jsonSchema.nullish(),
-});
-
 export type ObservationEvent =
   | z.infer<typeof legacyObservationCreateEvent>
   | z.infer<typeof legacyObservationUpdateEvent>

packages/shared/src/server/instrumentation/index.ts

@@ -160,35 +160,49 @@ export const addUserToSpan = (
 export const getTracer = (name: string) => opentelemetry.trace.getTracer(name);
 
 const cloudWatchClient = new CloudWatchClient();
-const cloudWatchLastSubmitted: Record<string, number> = {};
-const sendCloudWatchMetric = (key: string, value: number | undefined) => {
-  const currentTime = Date.now();
-  const interval = 30 * 1000;
+let lastFlushTime = 0;
+let metricCache: Record<string, number> = {};
 
-  // Check if the function has been executed in the last 30s for this key
-  if (
-    !cloudWatchLastSubmitted[key] ||
-    currentTime - cloudWatchLastSubmitted[key] >= interval
-  ) {
-    cloudWatchLastSubmitted[key] = currentTime;
-    cloudWatchClient
-      .send(
-        new PutMetricDataCommand({
-          Namespace: "Langfuse",
-          MetricData: [
-            {
-              MetricName: key,
-              Value: value ?? 0,
-            },
-          ],
-        }),
-      )
-      .catch((error) => {
-        logger.warn("Failed to send metric to CloudWatch", error);
-      });
+// Caches metrics and flushes them on schedule
+const sendCloudWatchMetric = (key: string, value: number, replace: boolean) => {
+  // Store the latest value for each metric key. If replace is false (e.g. for increments) we add the value to the existing value.
+  metricCache[key] = replace ? value : (metricCache[key] || 0) + value;
+
+  const currentTime = Date.now();
+  const flushInterval = 30 * 1000; // 30 seconds
+
+  // Check if it's time to flush the metrics
+  if (currentTime - lastFlushTime >= flushInterval) {
+    flushMetricsToCloudWatch();
   }
 };
 
+// Flush all cached metrics in a single API call
+const flushMetricsToCloudWatch = () => {
+  if (Object.keys(metricCache).length === 0) return;
+
+  lastFlushTime = Date.now();
+
+  const metricData = Object.entries(metricCache).map(([key, value]) => ({
+    MetricName: key,
+    Value: value,
+  }));
+
+  // Clear the cache after preparing the metrics
+  metricCache = {};
+
+  cloudWatchClient
+    .send(
+      new PutMetricDataCommand({
+        Namespace: "Langfuse",
+        MetricData: metricData,
+      }),
+    )
+    .catch((error) => {
+      logger.warn("Failed to send metrics to CloudWatch", error);
+    });
+};
+
 export const recordGauge = (
   stat: string,
   value?: number | undefined,
@@ -199,7 +213,7 @@ export const recordGauge = (
     | undefined,
 ) => {
   if (env.ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING === "true") {
-    sendCloudWatchMetric(stat, value);
+    sendCloudWatchMetric(stat, value ?? 0, true);
   }
   dd.dogstatsd.gauge(stat, value, tags);
 };
@@ -210,7 +224,7 @@ export const recordIncrement = (
   tags?: { [tag: string]: string | number } | undefined,
 ) => {
   if (env.ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING === "true") {
-    sendCloudWatchMetric(stat, value);
+    sendCloudWatchMetric(stat, value ?? 1, false);
   }
   dd.dogstatsd.increment(stat, value, tags);
 };
@@ -220,9 +234,6 @@ export const recordHistogram = (
   value?: number | undefined,
   tags?: { [tag: string]: string | number } | undefined,
 ) => {
-  if (env.ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING === "true") {
-    sendCloudWatchMetric(stat, value);
-  }
   dd.dogstatsd.histogram(stat, value, tags);
 };
 

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,4 +1,4 @@
-import type { ZodSchema } from "zod";
+import { type ZodSchema } from "zod";
 
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatVertexAI } from "@langchain/google-vertexai";
@@ -9,6 +9,7 @@ import {
   BaseMessage,
   HumanMessage,
   SystemMessage,
+  ToolMessage,
 } from "@langchain/core/messages";
 import {
   BytesOutputParser,
@@ -25,8 +26,13 @@ import { logger } from "../logger";
 import {
   ChatMessage,
   ChatMessageRole,
+  ChatMessageType,
   LLMAdapter,
+  LLMJSONSchema,
+  LLMToolDefinition,
   ModelParams,
+  ToolCallResponse,
+  ToolCallResponseSchema,
   TraceParams,
 } from "./types";
 import { CallbackHandler } from "langfuse-langchain";
@@ -37,7 +43,7 @@ type ProcessTracedEvents = () => Promise<void>;
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
-  structuredOutputSchema?: ZodSchema;
+  structuredOutputSchema?: ZodSchema | LLMJSONSchema;
   callbacks?: BaseCallbackHandler[];
   baseURL?: string;
   apiKey: string;
@@ -50,6 +56,7 @@ type LLMCompletionParams = {
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
   streaming: boolean;
+  tools?: LLMToolDefinition[];
 };
 
 export async function fetchLLMCompletion(
@@ -65,7 +72,10 @@ export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
   },
-): Promise<{ completion: string; processTracedEvents: ProcessTracedEvents }>;
+): Promise<{
+  completion: string;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
@@ -73,19 +83,34 @@ export async function fetchLLMCompletion(
     structuredOutputSchema: ZodSchema;
   },
 ): Promise<{
-  completion: unknown;
+  completion: Record<string, unknown>;
+  processTracedEvents: ProcessTracedEvents;
+}>;
+
+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    tools: LLMToolDefinition[];
+    streaming: false;
+  },
+): Promise<{
+  completion: ToolCallResponse;
   processTracedEvents: ProcessTracedEvents;
 }>;
 
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams,
 ): Promise<{
-  completion: string | IterableReadableStream<Uint8Array> | unknown;
+  completion:
+    | string
+    | IterableReadableStream<Uint8Array>
+    | Record<string, unknown>
+    | ToolCallResponse;
   processTracedEvents: ProcessTracedEvents;
 }> {
   // the apiKey must never be printed to the console
   const {
     messages,
+    tools,
     modelParams,
     streaming,
     callbacks,
@@ -140,11 +165,25 @@ export async function fetchLLMCompletion(
       )
         return new SystemMessage(message.content);
 
-      return new AIMessage(message.content);
+      if (message.type === ChatMessageType.ToolResult)
+        return new ToolMessage({
+          content: message.content,
+          tool_call_id: message.toolCallId,
+        });
+
+      return new AIMessage({
+        content: message.content,
+        tool_calls:
+          message.type === ChatMessageType.AssistantToolCall
+            ? (message.toolCalls as any)
+            : undefined,
+      });
     });
   }
 
-  finalMessages = finalMessages.filter((m) => m.content.length > 0);
+  finalMessages = finalMessages.filter(
+    (m) => m.content.length > 0 || "tool_calls" in m,
+  );
 
   let chatModel:
     | ChatOpenAI
@@ -184,13 +223,16 @@ export async function fetchLLMCompletion(
       azureOpenAIApiKey: apiKey,
       azureOpenAIBasePath: baseURL,
       azureOpenAIApiDeploymentName: modelParams.model,
-      azureOpenAIApiVersion: "2024-02-01",
+      azureOpenAIApiVersion: "2025-02-01-preview",
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
       timeout: 1000 * 60 * 2, // 2 minutes timeout
+      configuration: {
+        defaultHeaders: extraHeaders,
+      },
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
@@ -234,6 +276,22 @@ export async function fetchLLMCompletion(
       maxRetries,
       apiKey,
     });
+  } else if (modelParams.adapter === LLMAdapter.Atla) {
+    // Atla models do not support:
+    // - temperature
+    // - max_tokens
+    // - top_p
+    chatModel = new ChatOpenAI({
+      openAIApiKey: apiKey,
+      modelName: modelParams.model,
+      callbacks: finalCallbacks,
+      maxRetries,
+      configuration: {
+        baseURL: baseURL,
+        defaultHeaders: extraHeaders,
+      },
+      timeout: 1000 * 60, // 1 minute timeout
+    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;
@@ -301,6 +359,25 @@ export async function fetchLLMCompletion(
       };
     }
 
+    if (tools && tools.length > 0) {
+      const langchainTools = tools.map((tool) => ({
+        type: "function",
+        function: tool,
+      }));
+
+      const result = await chatModel
+        .bindTools(langchainTools)
+        .invoke(finalMessages, runConfig);
+
+      const parsed = ToolCallResponseSchema.safeParse(result);
+      if (!parsed.success) throw Error("Failed to parse LLM tool call result");
+
+      return {
+        completion: parsed.data,
+        processTracedEvents,
+      };
+    }
+
     if (streaming) {
       return {
         completion: await chatModel
@@ -320,6 +397,7 @@ export async function fetchLLMCompletion(
     if (throwOnError) {
       throw error;
     }
-    return { completion: null, processTracedEvents };
+
+    return { completion: "", processTracedEvents };
   }
 }

packages/shared/src/server/llm/types.ts

@@ -7,48 +7,211 @@ import { AuthHeaderValidVerificationResult } from "../auth/types";
 /* eslint-disable no-unused-vars */
 // disable lint as this is exported and used in web/worker
 
-export type PromptVariable = { name: string; value: string; isUsed: boolean };
+export const LLMJSONSchema = z.record(z.string(), z.unknown());
+export type LLMJSONSchema = z.infer<typeof LLMJSONSchema>;
 
-export type ChatMessage = {
-  role: ChatMessageRole | string; // Users may ingest any string as role via API/SDK
-  content: string;
-};
+export const JSONSchemaFormSchema = z
+  .string()
+  .transform((value, ctx) => {
+    try {
+      const parsed = JSON.parse(value);
+      return parsed;
+    } catch {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "Parameters must be valid JSON",
+      });
+      return z.NEVER;
+    }
+  })
+  .pipe(
+    z
+      .object({
+        type: z.literal("object"),
+        properties: z.record(z.any()),
+        required: z.array(z.string()).optional(),
+        additionalProperties: z.boolean().optional(),
+      })
+      .transform((data) => JSON.stringify(data, null, 2)),
+  );
 
+export const LLMToolDefinitionSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  parameters: LLMJSONSchema,
+});
+export type LLMToolDefinition = z.infer<typeof LLMToolDefinitionSchema>;
+
+const AnthropicMessageContentWithToolUse = z.union([
+  z.object({
+    type: z.literal("text"),
+    text: z.string(),
+  }),
+  z.object({
+    type: z.literal("tool_use"),
+    id: z.string(),
+    name: z.string(),
+    input: z.unknown(),
+  }),
+]);
+
+export const LLMToolCallSchema = z.object({
+  name: z.string(),
+  id: z.string(),
+  args: z.record(z.string(), z.unknown()),
+});
+export type LLMToolCall = z.infer<typeof LLMToolCallSchema>;
+
+export const OpenAIToolCallSchema = z.object({
+  id: z.string(),
+  function: z.object({
+    name: z.string(),
+    arguments: z.union([
+      z.record(z.string(), z.unknown()),
+      z
+        .string()
+        .transform((v) => JSON.parse(v))
+        .pipe(z.record(z.string(), z.unknown())),
+    ]),
+  }),
+  type: z.literal("function"),
+});
+export type OpenAIToolCallSchema = z.infer<typeof OpenAIToolCallSchema>;
+
+export const OpenAIToolSchema = z.object({
+  type: z.literal("function"),
+  function: LLMToolDefinitionSchema,
+});
+export type OpenAIToolSchema = z.infer<typeof OpenAIToolSchema>;
+
+export const OpenAIResponseFormatSchema = z.object({
+  type: z.literal("json_schema"),
+  json_schema: z.object({
+    name: z.string(),
+    schema: LLMJSONSchema,
+  }),
+});
+
+export const ToolCallResponseSchema = z.object({
+  content: z.union([z.string(), z.array(AnthropicMessageContentWithToolUse)]),
+  tool_calls: z.array(LLMToolCallSchema),
+});
+export type ToolCallResponse = z.infer<typeof ToolCallResponseSchema>;
+export enum ChatMessageRole {
+  System = "system",
+  Developer = "developer",
+  User = "user",
+  Assistant = "assistant",
+  Tool = "tool",
+}
+
+export enum ChatMessageType {
+  System = "system",
+  Developer = "developer",
+  User = "user",
+  AssistantText = "assistant-text",
+  AssistantToolCall = "assistant-tool-call",
+  ToolResult = "tool-result",
+  PublicAPICreated = "public-api-created",
+}
+
+export const SystemMessageSchema = z.object({
+  type: z.literal(ChatMessageType.System),
+  role: z.literal(ChatMessageRole.System),
+  content: z.string(),
+});
+export type SystemMessage = z.infer<typeof SystemMessageSchema>;
+
+export const DeveloperMessageSchema = z.object({
+  type: z.literal(ChatMessageType.Developer),
+  role: z.literal(ChatMessageRole.Developer),
+  content: z.string(),
+});
+export type DeveloperMessage = z.infer<typeof DeveloperMessageSchema>;
+
+export const UserMessageSchema = z.object({
+  type: z.literal(ChatMessageType.User),
+  role: z.literal(ChatMessageRole.User),
+  content: z.string(),
+});
+export type UserMessage = z.infer<typeof UserMessageSchema>;
+
+export const AssistantTextMessageSchema = z.object({
+  type: z.literal(ChatMessageType.AssistantText),
+  role: z.literal(ChatMessageRole.Assistant),
+  content: z.string(),
+});
+export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
+
+export const AssistantToolCallMessageSchema = z.object({
+  type: z.literal(ChatMessageType.AssistantToolCall),
+  role: z.literal(ChatMessageRole.Assistant),
+  content: z.string(),
+  toolCalls: z.array(LLMToolCallSchema),
+});
+export type AssistantToolCallMessage = z.infer<
+  typeof AssistantToolCallMessageSchema
+>;
+
+export const ToolResultMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ToolResult),
+  role: z.literal(ChatMessageRole.Tool),
+  content: z.string(),
+  toolCallId: z.string(),
+});
+export type ToolResultMessage = z.infer<typeof ToolResultMessageSchema>;
+
+export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
+export const ChatMessageSchema = z.union([
+  SystemMessageSchema,
+  DeveloperMessageSchema,
+  UserMessageSchema,
+  AssistantTextMessageSchema,
+  AssistantToolCallMessageSchema,
+  ToolResultMessageSchema,
+  z
+    .object({
+      role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+      content: z.string(),
+    })
+    .transform((msg) => {
+      return {
+        ...msg,
+        type: ChatMessageType.PublicAPICreated as const,
+      };
+    }),
+]);
+
+export type ChatMessage = z.infer<typeof ChatMessageSchema>;
 export type ChatMessageWithId = ChatMessage & { id: string };
 
+export const PromptChatMessageSchema = z.object({
+  role: z.string(),
+  content: z.string(),
+});
+export const PromptChatMessageListSchema = z.array(PromptChatMessageSchema);
+
+export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
+  Atla = "atla",
   Azure = "azure",
   Bedrock = "bedrock",
   VertexAI = "google-vertex-ai",
   GoogleAIStudio = "google-ai-studio",
 }
 
-export enum ChatMessageRole {
-  System = "system",
-  User = "user",
-  Assistant = "assistant",
-  Developer = "developer",
-}
-
 export const SYSTEM_ROLES: string[] = [
   ChatMessageRole.System,
   ChatMessageRole.Developer,
 ];
 
-export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
-
-const ChatMessageSchema = z.object({
-  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
-  content: z.string(),
-});
-
-export const ChatMessageListSchema = z.array(ChatMessageSchema);
 export const TextPromptSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
-  ChatMessageListSchema,
+  PromptChatMessageListSchema,
   TextPromptSchema,
 ]);
 export type PromptContent = z.infer<typeof PromptContentSchema>;
@@ -152,6 +315,8 @@ export const googleAIStudioModels = [
   "gemini-1.5-flash-8b",
 ] as const;
 
+export const atlaModels = ["atla-selene", "atla-selene-20250214"] as const;
+
 export type AnthropicModel = (typeof anthropicModels)[number];
 export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
@@ -161,6 +326,7 @@ export const supportedModels = {
   [LLMAdapter.GoogleAIStudio]: googleAIStudioModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
+  [LLMAdapter.Atla]: atlaModels,
 } as const;
 
 export type LLMFunctionCall = {

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -391,6 +391,10 @@ export class FilterList {
     return this.filters.some(predicate);
   }
 
+  forEach(callback: (filter: Filter) => void) {
+    this.filters.forEach(callback);
+  }
+
   length() {
     return this.filters.length;
   }

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -3,7 +3,7 @@ import { singleFilter } from "../../../interfaces/filters";
 import { FilterCondition } from "../../../types";
 import { isValidTableName } from "../../clickhouse/schemaUtils";
 import { logger } from "../../logger";
-import { UiColumnMapping } from "../../../tableDefinitions";
+import { UiColumnMappings } from "../../../tableDefinitions";
 import {
   StringFilter,
   DateTimeFilter,
@@ -29,7 +29,7 @@ export class QueryBuilderError extends Error {
 // User input for values (e.g. project_id = <value>) are sent to Clickhouse as parameters to prevent SQL injection
 export const createFilterFromFilterState = (
   filter: FilterCondition[],
-  columnMapping: UiColumnMapping[],
+  columnMapping: UiColumnMappings,
 ) => {
   return filter.map((frontEndFilter) => {
     // checks if the column exists in the clickhouse schema
@@ -120,7 +120,7 @@ export const createFilterFromFilterState = (
 
 const matchAndVerifyTracesUiColumn = (
   filter: z.infer<typeof singleFilter>,
-  uiTableDefinitions: UiColumnMapping[],
+  uiTableDefinitions: UiColumnMappings,
 ) => {
   // tries to match the column name to the clickhouse table name
   logger.debug(`Filter to match: ${JSON.stringify(filter)}`);

packages/shared/src/server/queries/clickhouse-sql/orderby-factory.ts

@@ -1,13 +1,13 @@
 import z from "zod";
 import { OrderByState } from "../../../interfaces/orderBy";
-import { UiColumnMapping } from "../../../tableDefinitions";
+import { UiColumnMappings } from "../../../tableDefinitions";
 import { logger } from "../../logger";
 
 type OrderByStateNotNull = Exclude<OrderByState, null>;
 
 export function orderByToClickhouseSql(
   orderBy: OrderByState | OrderByState[] = [],
-  tableColumns: UiColumnMapping[],
+  tableColumns: UiColumnMappings,
 ): string {
   if (
     !orderBy ||

packages/shared/src/server/queries/index.ts

@@ -18,3 +18,4 @@ export {
   type ClickhouseOperator,
 } from "./clickhouse-sql/clickhouse-filter";
 export { orderByToClickhouseSql } from "./clickhouse-sql/orderby-factory";
+export { createFilterFromFilterState } from "./clickhouse-sql/factory";

packages/shared/src/server/queues.ts

@@ -34,6 +34,10 @@ export const TracesQueueEventSchema = z.object({
   projectId: z.string(),
   traceIds: z.array(z.string()),
 });
+export const ScoresQueueEventSchema = z.object({
+  projectId: z.string(),
+  scoreIds: z.array(z.string()),
+});
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -52,6 +56,9 @@ export const EvalExecutionEvent = z.object({
 export const PostHogIntegrationProcessingEventSchema = z.object({
   projectId: z.string(),
 });
+export const BlobStorageIntegrationProcessingEventSchema = z.object({
+  projectId: z.string(),
+});
 export const ExperimentCreateEventSchema = z.object({
   projectId: z.string(),
   datasetId: z.string(),
@@ -65,6 +72,15 @@ export const DataRetentionProcessingEventSchema = z.object({
 export const BatchActionProcessingEventSchema = z.discriminatedUnion(
   "actionId",
   [
+    z.object({
+      actionId: z.literal("score-delete"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.nativeEnum(BatchExportTableName),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.nativeEnum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("trace-delete"),
       projectId: z.string(),
@@ -114,6 +130,7 @@ export type CreateEvalQueueEventType = z.infer<
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
+export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -132,6 +149,9 @@ export type DataRetentionProcessingEventType = z.infer<
 export type BatchActionProcessingEventType = z.infer<
   typeof BatchActionProcessingEventSchema
 >;
+export type BlobStorageIntegrationProcessingEventType = z.infer<
+  typeof BlobStorageIntegrationProcessingEventSchema
+>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
@@ -146,12 +166,15 @@ export enum QueueName {
   ExperimentCreate = "experiment-create-queue",
   PostHogIntegrationQueue = "posthog-integration-queue",
   PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
+  BlobStorageIntegrationQueue = "blobstorage-integration-queue",
+  BlobStorageIntegrationProcessingQueue = "blobstorage-integration-processing-queue",
   CoreDataS3ExportQueue = "core-data-s3-export-queue",
   MeteringDataPostgresExportQueue = "metering-data-postgres-export-queue",
   DataRetentionQueue = "data-retention-queue",
   DataRetentionProcessingQueue = "data-retention-processing-queue",
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
+  ScoreDelete = "score-delete",
 }
 
 export enum QueueJobs {
@@ -167,12 +190,15 @@ export enum QueueJobs {
   ExperimentCreateJob = "experiment-create-job",
   PostHogIntegrationJob = "posthog-integration-job",
   PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
+  BlobStorageIntegrationJob = "blobstorage-integration-job",
+  BlobStorageIntegrationProcessingJob = "blobstorage-integration-processing-job",
   CoreDataS3ExportJob = "core-data-s3-export-job",
   MeteringDataPostgresExportJob = "metering-data-postgres-export-job",
   DataRetentionJob = "data-retention-job",
   DataRetentionProcessingJob = "data-retention-processing-job",
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
+  ScoreDelete = "score-delete",
 }
 
 export type TQueueJobTypes = {
@@ -188,6 +214,12 @@ export type TQueueJobTypes = {
     payload: TracesQueueEventType | TraceQueueEventType;
     name: QueueJobs.TraceDelete;
   };
+  [QueueName.ScoreDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: ScoresQueueEventType;
+    name: QueueJobs.ScoreDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -254,4 +286,10 @@ export type TQueueJobTypes = {
     payload: CreateEvalQueueEventType;
     name: QueueJobs.CreateEvalJob;
   };
+  [QueueName.BlobStorageIntegrationProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: BlobStorageIntegrationProcessingEventType;
+    name: QueueJobs.BlobStorageIntegrationProcessingJob;
+  };
 };

packages/shared/src/server/redis/blobStorageIntegrationProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class BlobStorageIntegrationProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (BlobStorageIntegrationProcessingQueue.instance) {
+      return BlobStorageIntegrationProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    BlobStorageIntegrationProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.BlobStorageIntegrationProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100_000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    BlobStorageIntegrationProcessingQueue.instance?.on("error", (err) => {
+      logger.error("BlobStorageIntegrationProcessingQueue error", err);
+    });
+
+    return BlobStorageIntegrationProcessingQueue.instance;
+  }
+} 

packages/shared/src/server/redis/blobStorageIntegrationQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class BlobStorageIntegrationQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (BlobStorageIntegrationQueue.instance) {
+      return BlobStorageIntegrationQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    BlobStorageIntegrationQueue.instance = newRedis
+      ? new Queue(QueueName.BlobStorageIntegrationQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    BlobStorageIntegrationQueue.instance?.on("error", (err) => {
+      logger.error("BlobStorageIntegrationQueue error", err);
+    });
+
+    if (BlobStorageIntegrationQueue.instance) {
+      logger.debug("Scheduling jobs for BlobStorageIntegrationQueue");
+      BlobStorageIntegrationQueue.instance
+        .add(
+          QueueJobs.BlobStorageIntegrationJob,
+          {},
+          {
+            repeat: { pattern: "20 * * * *" }, // every hour at 20 minutes past
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding BlobStorageIntegrationJob schedule", err);
+        });
+    }
+
+    return BlobStorageIntegrationQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -11,12 +11,15 @@ import { TraceDeleteQueue } from "./traceDelete";
 import { ProjectDeleteQueue } from "./projectDelete";
 import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
 import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
+import { BlobStorageIntegrationQueue } from "./blobStorageIntegrationQueue";
+import { BlobStorageIntegrationProcessingQueue } from "./blobStorageIntegrationProcessingQueue";
 import { CoreDataS3ExportQueue } from "./coreDataS3ExportQueue";
 import { MeteringDataPostgresExportQueue } from "./meteringDataPostgresExportQueue";
 import { DataRetentionQueue } from "./dataRetentionQueue";
 import { DataRetentionProcessingQueue } from "./dataRetentionProcessingQueue";
 import { BatchActionQueue } from "./batchActionQueue";
 import { CreateEvalQueue } from "./createEvalQueue";
+import { ScoreDeleteQueue } from "./scoreDelete";
 
 export function getQueue(queueName: QueueName): Queue | null {
   switch (queueName) {
@@ -42,6 +45,10 @@ export function getQueue(queueName: QueueName): Queue | null {
       return PostHogIntegrationQueue.getInstance();
     case QueueName.PostHogIntegrationProcessingQueue:
       return PostHogIntegrationProcessingQueue.getInstance();
+    case QueueName.BlobStorageIntegrationQueue:
+      return BlobStorageIntegrationQueue.getInstance();
+    case QueueName.BlobStorageIntegrationProcessingQueue:
+      return BlobStorageIntegrationProcessingQueue.getInstance();
     case QueueName.IngestionSecondaryQueue:
       return SecondaryIngestionQueue.getInstance();
     case QueueName.CoreDataS3ExportQueue:
@@ -56,6 +63,8 @@ export function getQueue(queueName: QueueName): Queue | null {
       return BatchActionQueue.getInstance();
     case QueueName.CreateEvalQueue:
       return CreateEvalQueue.getInstance();
+    case QueueName.ScoreDelete:
+      return ScoreDeleteQueue.getInstance();
     default:
       const exhaustiveCheckDefault: never = queueName;
       throw new Error(`Queue ${queueName} not found`);

packages/shared/src/server/redis/scoreDelete.ts

@@ -0,0 +1,38 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class ScoreDeleteQueue {
+  private static instance: Queue<TQueueJobTypes[QueueName.ScoreDelete]> | null = null;
+
+  public static getInstance(): Queue<TQueueJobTypes[QueueName.ScoreDelete]> | null {
+    if (ScoreDeleteQueue.instance) return ScoreDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ScoreDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ScoreDelete]>(QueueName.ScoreDelete, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100_000,
+            attempts: 2,
+            backoff: {
+              type: "exponential",
+              delay: 30_000,
+            },
+          },
+        })
+      : null;
+
+    ScoreDeleteQueue.instance?.on("error", (err) => {
+      logger.error("ScoreDeleteQueue error", err);
+    });
+
+    return ScoreDeleteQueue.instance;
+  }
+} 

packages/shared/src/server/repositories/dashboards.ts

@@ -4,11 +4,8 @@ import {
 } from "./clickhouse";
 import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
 import { FilterState } from "../../types";
-import {
-  DateTimeFilter,
-  FilterList,
-} from "../queries/clickhouse-sql/clickhouse-filter";
-import { dashboardColumnDefinitions } from "../../tableDefinitions/mapDashboards";
+import { DateTimeFilter, FilterList } from "../queries";
+import { dashboardColumnDefinitions } from "../../tableDefinitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
@@ -18,12 +15,37 @@ import {
 
 export type DateTrunc = "year" | "month" | "week" | "day" | "hour" | "minute";
 
+const extractEnvironmentFilterFromFilters = (
+  filter: FilterState,
+): { envFilter: FilterState; remainingFilters: FilterState } => {
+  return {
+    envFilter: filter.filter((f) => f.column === "environment"),
+    remainingFilters: filter.filter((f) => f.column !== "environment"),
+  };
+};
+
+const convertEnvFilterToClickhouseFilter = (filter: FilterState) => {
+  return createFilterFromFilterState(filter, [
+    {
+      clickhouseSelect: "environment",
+      clickhouseTableName: "traces",
+      uiTableId: "environment",
+      uiTableName: "Environment",
+    },
+  ]);
+};
+
 export const getTotalTraces = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   ).apply();
 
   const query = `
@@ -31,13 +53,16 @@ export const getTotalTraces = async (
       count(id) as count 
     FROM traces t FINAL 
     WHERE project_id = {projectId: String}
-    AND ${chFilter.query}`;
+    AND ${chFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
+  `;
 
   const result = await queryClickhouse<{ count: number }>({
     query,
     params: {
       projectId,
       ...chFilter.params,
+      ...environmentFilter.params,
     },
     tags: {
       feature: "dashboard",
@@ -58,8 +83,13 @@ export const getObservationsCostGroupedByName = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -74,6 +104,7 @@ export const getObservationsCostGroupedByName = async (
     FROM observations o FINAL ${hasTraceFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     ${appliedFilter.query ? `AND ${appliedFilter.query}` : ""}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     GROUP BY provided_model_name
     ORDER BY sumMap(cost_details)['total'] DESC
     LIMIT 50
@@ -88,6 +119,7 @@ export const getObservationsCostGroupedByName = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
     },
     tags: {
       feature: "dashboard",
@@ -104,8 +136,13 @@ export const getScoreAggregate = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const timeFilter = chFilter.find(
@@ -129,6 +166,7 @@ export const getScoreAggregate = async (
      ${hasTraceFilter ? "JOIN traces t FINAL ON t.id = s.trace_id AND t.project_id = s.project_id" : ""}
     WHERE s.project_id = {projectId: String}
     AND ${chFilterApplied.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${timeFilter && hasTraceFilter ? `AND t.timestamp >= {tracesTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
     GROUP BY s.name, s.source, s.data_type
     ORDER BY count(*) DESC
@@ -145,6 +183,7 @@ export const getScoreAggregate = async (
     params: {
       projectId,
       ...chFilterApplied.params,
+      ...environmentFilter.params,
       ...(timeFilter
         ? { tracesTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
         : {}),
@@ -164,8 +203,13 @@ export const groupTracesByTime = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   ).apply();
 
   const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
@@ -180,6 +224,7 @@ export const groupTracesByTime = async (
     FROM traces t FINAL
     WHERE project_id = {projectId: String}
     AND ${chFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     GROUP BY timestamp
     ${orderByQuery}
     `;
@@ -191,6 +236,7 @@ export const groupTracesByTime = async (
     params: {
       projectId,
       ...chFilter.params,
+      ...environmentFilter.params,
       ...orderByParams,
     },
     tags: {
@@ -207,12 +253,17 @@ export const groupTracesByTime = async (
   }));
 };
 
-export const getObservationUsageByTime = async (
+export const getTotalObservationUsageByTimeByModel = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -235,13 +286,14 @@ export const getObservationUsageByTime = async (
   const query = `
     SELECT 
       ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
-      sumMap(usage_details) as units,
-      sumMap(cost_details) as cost, 
+      sumMap(usage_details)['total'] as units,
+      sumMap(cost_details)['total'] as cost, 
       provided_model_name
     FROM observations o FINAL
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY start_time, provided_model_name
     ${orderByQuery}
@@ -249,14 +301,15 @@ export const getObservationUsageByTime = async (
 
   const result = await queryClickhouse<{
     start_time: string;
-    units: Record<string, number>;
-    cost: Record<string, number>;
+    units: string;
+    cost: string;
     provided_model_name: string;
   }>({
     query,
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
       ...orderByParams,
       ...(timeFilter
         ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
@@ -271,29 +324,220 @@ export const getObservationUsageByTime = async (
   });
 
   return result.map((row) => ({
-    start_time: parseClickhouseUTCDateTimeFormat(row.start_time),
-    units: Object.fromEntries(
-      Object.entries(row.units ?? {}).map(([key, value]) => [
-        key,
-        Number(value),
-      ]),
-    ),
-    cost: Object.fromEntries(
-      Object.entries(row.cost ?? {}).map(([key, value]) => [
-        key,
-        Number(value),
-      ]),
-    ),
-    provided_model_name: row.provided_model_name,
+    startTime: parseClickhouseUTCDateTimeFormat(row.start_time),
+    units: Number(row.units),
+    cost: Number(row.cost),
+    model: row.provided_model_name,
   }));
 };
 
+export const getObservationCostByTypeByTime = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
+  const chFilter = new FilterList(
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  const timeFilter = tracesFilter
+    ? (chFilter.find(
+        (f) =>
+          f.clickhouseTable === "observations" &&
+          f.field.includes("start_time") &&
+          (f.operator === ">=" || f.operator === ">"),
+      ) as DateTimeFilter | undefined)
+    : undefined;
+
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time",
+  );
+
+  const query = `
+    SELECT 
+        start_time, 
+        groupArray((cost_key, cost_sum)) AS costs
+    FROM (
+        SELECT 
+            ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
+            cost_key, 
+            SUM(cost) AS cost_sum
+        FROM 
+            observations o FINAL
+        ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+        ARRAY JOIN
+            mapKeys(cost_details) AS cost_key, 
+            mapValues(cost_details) AS cost
+        WHERE project_id = {projectId: String}
+        AND ${appliedFilter.query}
+        ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
+        ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        GROUP BY 
+            start_time, 
+            cost_key
+    ) 
+    GROUP BY 
+        start_time 
+    ${orderByQuery}
+  `;
+
+  const result = await queryClickhouse<{
+    start_time: string;
+    costs: Array<[string, number | null]>;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...environmentFilter.params,
+      ...orderByParams,
+      ...(timeFilter
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
+    },
+    tags: {
+      feature: "dashboard",
+      type: "observationCostByTypeByTime",
+      kind: "analytic",
+      projectId,
+    },
+  });
+
+  const types = result.flatMap((row) => {
+    return row.costs.map((cost) => cost[0]);
+  });
+
+  const uniqueTypes = [...new Set(types)];
+
+  return result.flatMap((row) => {
+    const intervalStart = parseClickhouseUTCDateTimeFormat(row.start_time);
+    return uniqueTypes.map((type) => ({
+      intervalStart: intervalStart,
+      key: type,
+      sum: row.costs.find((cost) => cost[0] === type)?.[1]
+        ? Number(row.costs.find((cost) => cost[0] === type)?.[1])
+        : 0,
+    }));
+  });
+};
+
+export const getObservationUsageByTypeByTime = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
+  const chFilter = new FilterList(
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  const timeFilter = tracesFilter
+    ? (chFilter.find(
+        (f) =>
+          f.clickhouseTable === "observations" &&
+          f.field.includes("start_time") &&
+          (f.operator === ">=" || f.operator === ">"),
+      ) as DateTimeFilter | undefined)
+    : undefined;
+
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time",
+  );
+
+  const query = `
+    SELECT 
+        start_time, 
+        groupArray((usage_key, usage_sum)) AS usages
+    FROM (
+        SELECT 
+            ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")} ,
+            usage_key, 
+            SUM(usage) AS usage_sum
+        FROM 
+            observations o FINAL
+        ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+        ARRAY JOIN
+            mapKeys(usage_details) AS usage_key, 
+            mapValues(usage_details) AS usage
+        WHERE project_id = {projectId: String}
+        AND ${appliedFilter.query}
+        ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
+        ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        GROUP BY 
+            start_time, 
+            usage_key
+    ) 
+    GROUP BY 
+        start_time 
+    ${orderByQuery}
+  `;
+
+  const result = await queryClickhouse<{
+    start_time: string;
+    usages: Array<[string, number | null]>;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...environmentFilter.params,
+      ...orderByParams,
+      ...(timeFilter
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
+    },
+    tags: {
+      feature: "dashboard",
+      type: "observationUsageByTime",
+      kind: "analytic",
+      projectId,
+    },
+  });
+
+  const types = result.flatMap((row) => {
+    return row.usages.map((usage) => usage[0]);
+  });
+
+  const uniqueTypes = [...new Set(types)];
+
+  return result.flatMap((row) => {
+    const intervalStart = parseClickhouseUTCDateTimeFormat(row.start_time);
+    return uniqueTypes.map((type) => ({
+      intervalStart: intervalStart,
+      key: type,
+      sum: row.usages.find((usage) => usage[0] === type)?.[1]
+        ? Number(row.usages.find((usage) => usage[0] === type)?.[1])
+        : 0,
+    }));
+  });
+};
+
 export const getDistinctModels = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -315,6 +559,7 @@ export const getDistinctModels = async (
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY provided_model_name
     ORDER BY count(*) DESC
@@ -326,6 +571,7 @@ export const getDistinctModels = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
       ...(timeFilter
         ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
         : {}),
@@ -345,8 +591,13 @@ export const getScoresAggregateOverTime = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -369,6 +620,7 @@ export const getScoresAggregateOverTime = async (
   FROM scores FINAL
   ${traceFilter ? "JOIN traces t ON scores.trace_id = t.id AND scores.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
+  ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
   AND ${appliedFilter.query}
   AND data_type IN ('NUMERIC', 'BOOLEAN')
   GROUP BY 
@@ -390,6 +642,7 @@ export const getScoresAggregateOverTime = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
       ...orderByParams,
     },
     tags: {
@@ -413,8 +666,13 @@ export const getModelUsageByUser = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -437,6 +695,7 @@ export const getModelUsageByUser = async (
     WHERE project_id = {projectId: String}
     AND t.user_id IS NOT NULL
     AND ${appliedFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY user_id
     ORDER BY sum_cost_details DESC
@@ -451,6 +710,7 @@ export const getModelUsageByUser = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
       ...(timeFilter
         ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
         : {}),
@@ -474,8 +734,13 @@ export const getObservationLatencies = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -489,13 +754,18 @@ export const getObservationLatencies = async (
     ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     GROUP BY name
     ORDER BY quantiles[2] DESC
     `;
 
   const result = await queryClickhouse<{ quantiles: string[]; name: string }>({
     query,
-    params: { projectId, ...appliedFilter.params },
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...environmentFilter.params,
+    },
     tags: {
       feature: "dashboard",
       type: "observationLatencies",
@@ -517,8 +787,13 @@ export const getTracesLatencies = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -542,6 +817,7 @@ export const getTracesLatencies = async (
       ON o.trace_id = t.id AND o.project_id = t.project_id
       WHERE project_id = {projectId: String}
       AND ${appliedFilter.query}
+      ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
       ${timestampFilter ? `AND o.start_time > {dateTimeFilterObservations: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
       GROUP BY o.project_id, o.trace_id, t.name
     )
@@ -559,6 +835,7 @@ export const getTracesLatencies = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...environmentFilter.params,
       ...(timestampFilter
         ? { dateTimeFilterObservations: timestampFilter.value }
         : {}),
@@ -584,8 +861,13 @@ export const getModelLatenciesOverTime = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
 
   const appliedFilter = chFilter.apply();
@@ -606,6 +888,7 @@ export const getModelLatenciesOverTime = async (
   FROM observations o
   ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
+  ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
   AND ${appliedFilter.query}
   GROUP BY provided_model_name, start_time_bucket
   ${orderByQuery};
@@ -617,7 +900,12 @@ export const getModelLatenciesOverTime = async (
     quantiles: string[];
   }>({
     query,
-    params: { projectId, ...appliedFilter.params, ...orderByParams },
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...environmentFilter.params,
+      ...orderByParams,
+    },
     tags: {
       feature: "dashboard",
       type: "modelLatenciesOverTime",
@@ -641,8 +929,13 @@ export const getNumericScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
   const chFilterRes = chFilter.apply();
 
@@ -661,6 +954,7 @@ export const getNumericScoreTimeSeries = async (
     FROM scores s final
     ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
     WHERE s.project_id = {projectId: String}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     GROUP BY score_name, score_timestamp
     ${orderByQuery}
@@ -675,6 +969,7 @@ export const getNumericScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...environmentFilter.params,
       ...orderByParams,
     },
     tags: {
@@ -696,8 +991,13 @@ export const getCategoricalScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
   const chFilterRes = chFilter.apply();
 
@@ -719,6 +1019,7 @@ export const getCategoricalScoreTimeSeries = async (
     FROM scores s final
     ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
     WHERE s.project_id = {projectId: String}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     GROUP BY score_name, score_data_type, score_source, score_value ${bucketSizeInSeconds ? ", score_timestamp" : ""}
       ${orderByQuery}
@@ -736,6 +1037,7 @@ export const getCategoricalScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...environmentFilter.params,
       ...orderByParams,
     },
     tags: {
@@ -762,8 +1064,13 @@ export const getObservationsStatusTimeSeries = async (
   projectId: string,
   filter: FilterState,
 ) => {
+  const { envFilter, remainingFilters } =
+    extractEnvironmentFilterFromFilters(filter);
+  const environmentFilter = new FilterList(
+    convertEnvFilterToClickhouseFilter(envFilter),
+  ).apply();
   const chFilter = new FilterList(
-    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
   );
   const chFilterRes = chFilter.apply();
 
@@ -782,6 +1089,7 @@ export const getObservationsStatusTimeSeries = async (
     FROM observations o
     ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
+    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
     AND o.level IS NOT NULL
     AND ${chFilterRes?.query}
     GROUP BY level ${bucketSizeInSeconds ? ", start_time_bucket" : ""}
@@ -797,6 +1105,7 @@ export const getObservationsStatusTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...environmentFilter.params,
       ...orderByParams,
     },
     tags: {

packages/shared/src/server/repositories/environments.ts

@@ -0,0 +1,36 @@
+import { queryClickhouse } from "./clickhouse";
+
+export type EnvironmentFilterProps = {
+  projectId: string;
+};
+
+export const getEnvironmentsForProject = async (
+  props: EnvironmentFilterProps,
+): Promise<{ environment: string }[]> => {
+  const { projectId } = props;
+
+  const query = `
+    SELECT environments
+    FROM project_environments FINAL
+    WHERE project_id = {projectId: String}
+  `;
+
+  const results = await queryClickhouse<{
+    environments: string[];
+  }>({
+    query,
+    params: { projectId },
+    tags: {
+      feature: "tracing",
+      type: "environment",
+      kind: "byId",
+      projectId,
+    },
+  });
+
+  const environments = results.length > 0 ? results[0].environments : [];
+  environments.push("default");
+  return Array.from(new Set(environments)).map((environment) => ({
+    environment,
+  }));
+};

packages/shared/src/server/repositories/eventLog.ts

@@ -81,6 +81,37 @@ export const getEventLogByProjectIdBeforeDate = (
   });
 };
 
+export const getEventLogByProjectIdAndEntityIds = (
+  projectId: string,
+  entityType: "observation" | "trace" | "score",
+  entityIds: string[],
+): AsyncGenerator<EventLogRecordReadType> => {
+  const query = `
+    select *
+    from event_log
+    where project_id = {projectId: String}
+      and entity_type = {entityType: String}
+      and entity_id in ({entityIds: Array(String)})
+  `;
+
+  return queryClickhouseStream<EventLogRecordReadType>({
+    query,
+    params: {
+      projectId,
+      entityType,
+      entityIds,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+    tags: {
+      feature: "eventLog",
+      kind: "list",
+      projectId,
+    },
+  });
+};
+
 export const getEventLogByProjectIdAndTraceIds = (
   projectId: string,
   traceIds: string[],

packages/shared/src/server/repositories/index.ts

@@ -10,3 +10,4 @@ export * from "./clickhouse";
 export * from "./constants";
 export * from "./trace-sessions";
 export * from "./eventLog";
+export * from "./environments";

packages/shared/src/server/repositories/observations.ts

@@ -510,6 +510,7 @@ const getObservationsTableInternal = async <T>(
         o.provided_cost_details as "provided_cost_details",
         o.cost_details as "cost_details",
         o.level as level,
+        o.environment as "environment",
         o.status_message as "status_message",
         o.version as version,
         o.parent_observation_id as "parent_observation_id",
@@ -1323,6 +1324,59 @@ export const getTraceIdsForObservations = async (
   }));
 };
 
+export const getObservationsForBlobStorageExport = function (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) {
+  const query = `
+    SELECT
+      id,
+      trace_id,
+      project_id,
+      environment,
+      type,
+      parent_observation_id,
+      start_time,
+      end_time,
+      name,
+      metadata,
+      level,
+      status_message,
+      version,
+      input,
+      output,
+      provided_model_name,
+      model_parameters,
+      usage_details,
+      cost_details,
+      completion_start_time,
+      prompt_name,
+      prompt_version
+    FROM observations FINAL
+    WHERE project_id = {projectId: String}
+    AND start_time >= {minTimestamp: DateTime64(3)}
+    AND start_time <= {maxTimestamp: DateTime64(3)}
+  `;
+
+  const records = queryClickhouseStream<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+    tags: {
+      feature: "blobstorage",
+      type: "observation",
+      kind: "analytic",
+      projectId,
+    },
+  });
+
+  return records;
+};
+
 export const getGenerationsForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
@@ -1374,6 +1428,13 @@ export const getGenerationsForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");

packages/shared/src/server/repositories/scores.ts

@@ -346,9 +346,7 @@ export const getScoresUiCount = async (props: {
   offset?: number;
 }) => {
   const rows = await getScoresUiGeneric<{ count: string }>({
-    select: `
-    count(*) as count
-    `,
+    select: "count",
     tags: { kind: "count" },
     ...props,
   });
@@ -394,32 +392,7 @@ export const getScoresUiTable = async (props: {
     created_at: string;
     updated_at: string;
   }>({
-    select: `
-        s.id,
-        s.project_id,
-        s.environment,
-        s.name,
-        s.value,
-        s.string_value,
-        s.timestamp,
-        s.source,
-        s.data_type,
-        s.comment,
-        s.trace_id,
-        s.observation_id,
-        s.author_user_id,
-        t.user_id,
-        t.name,
-        t.tags,
-        s.created_at,
-        s.updated_at,
-        s.source,
-        s.config_id,
-        s.queue_id,
-        t.user_id,
-        t.name as trace_name,
-        t.tags as trace_tags
-    `,
+    select: "rows",
     tags: { kind: "analytic" },
     ...props,
   });
@@ -449,7 +422,7 @@ export const getScoresUiTable = async (props: {
 };
 
 export const getScoresUiGeneric = async <T>(props: {
-  select: string;
+  select: "count" | "rows";
   projectId: string;
   filter: FilterState;
   orderBy: OrderByState;
@@ -457,26 +430,56 @@ export const getScoresUiGeneric = async <T>(props: {
   offset?: number;
   tags?: Record<string, string>;
 }): Promise<T[]> => {
-  const { select, projectId, filter, orderBy, limit, offset } = props;
+  const { projectId, filter, orderBy, limit, offset } = props;
+
+  const select =
+    props.select === "count"
+      ? "count(*) as count"
+      : `
+        s.id,
+        s.project_id,
+        s.environment,
+        s.name,
+        s.value,
+        s.string_value,
+        s.timestamp,
+        s.source,
+        s.data_type,
+        s.comment,
+        s.trace_id,
+        s.observation_id,
+        s.author_user_id,
+        t.user_id,
+        t.name,
+        t.tags,
+        s.created_at,
+        s.updated_at,
+        s.source,
+        s.config_id,
+        s.queue_id,
+        t.user_id,
+        t.name as trace_name,
+        t.tags as trace_tags
+      `;
 
   const { scoresFilter } = getProjectIdDefaultFilter(projectId, {
     tracesPrefix: "t",
   });
-
   scoresFilter.push(
     ...createFilterFromFilterState(filter, scoresTableUiColumnDefinitions),
   );
-
   const scoresFilterRes = scoresFilter.apply();
 
-  // TODO: Can we realistically apply a traces time filter here? Is an order by event_ts a risk?
+  // Only join traces for rows or if there is a trace filter on counts
+  const performTracesJoin =
+    props.select === "rows" ||
+    scoresFilter.some((f) => f.clickhouseTable === "traces");
+
   const query = `
       SELECT 
           ${select}
       FROM scores s final
-      LEFT JOIN traces t
-      ON s.trace_id = t.id 
-      AND t.project_id = s.project_id
+      ${performTracesJoin ? "LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
@@ -551,17 +554,17 @@ export const getScoreNames = async (
   }));
 };
 
-export const deleteScore = async (projectId: string, scoreId: string) => {
+export const deleteScores = async (projectId: string, scoreIds: string[]) => {
   const query = `
     DELETE FROM scores
     WHERE project_id = {projectId: String}
-    AND id = {scoreId: String};
+    AND id in ({scoreIds: Array(String)});
   `;
   await commandClickhouse({
     query: query,
     params: {
       projectId,
-      scoreId,
+      scoreIds,
     },
     tags: {
       feature: "tracing",
@@ -771,6 +774,37 @@ export const getScoreCountsByProjectInCreationInterval = async ({
   }));
 };
 
+export const getScoreCountOfProjectsSinceCreationDate = async ({
+  projectIds,
+  start,
+}: {
+  projectIds: string[];
+  start: Date;
+}) => {
+  const query = `
+    SELECT 
+      count(*) as count
+    FROM scores
+    WHERE project_id IN ({projectIds: Array(String)})
+    AND created_at >= {start: DateTime64(3)}
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectIds,
+      start: convertDateToClickhouseDateTime(start),
+    },
+    tags: {
+      feature: "tracing",
+      type: "score",
+      kind: "analytic",
+    },
+  });
+
+  return Number(rows[0]?.count ?? 0);
+};
+
 export const getDistinctScoreNames = async (
   projectId: string,
   cutoffCreatedAt: Date,
@@ -812,6 +846,49 @@ export const getDistinctScoreNames = async (
   return rows.map((row) => row.name);
 };
 
+export const getScoresForBlobStorageExport = function (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) {
+  const query = `
+    SELECT
+      id,
+      timestamp,
+      project_id,
+      environment,
+      trace_id,
+      observation_id,
+      name,
+      value,
+      source,
+      comment,
+      data_type,
+      string_value
+    FROM scores FINAL
+    WHERE project_id = {projectId: String}
+    AND timestamp >= {minTimestamp: DateTime64(3)}
+    AND timestamp <= {maxTimestamp: DateTime64(3)}
+  `;
+
+  const records = queryClickhouseStream<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+    tags: {
+      feature: "blobstorage",
+      type: "score",
+      kind: "analytic",
+      projectId,
+    },
+  });
+
+  return records;
+};
+
 export const getScoresForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
@@ -853,6 +930,13 @@ export const getScoresForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");

packages/shared/src/server/repositories/trace-sessions.ts

@@ -1,59 +1,68 @@
 import z from "zod";
 import { prisma } from "../../db";
-import { singleFilter, stringOptionsFilter } from "../../interfaces/filters";
+import { singleFilter } from "../../interfaces/filters";
 
 export const getPublicSessionsFilter = async (
   projectId: string,
   filter: z.infer<typeof singleFilter>[],
 ) => {
+  // Theoretically we should also filter the sessions by environment here. As this would return a huge list that's probably not feasible.
+  // I.e. we only perform the environment check on the ClickHouse queries.
+
   const sessionsBookmarkedFilter = filter?.find((f) => f.column === "⭐️");
 
-  // we are only fetching bookmarked sessions.
-  // They need to be manipulated in the UI and should not be as many.
-  const filteredSessions = sessionsBookmarkedFilter
-    ? await prisma.traceSession.findMany({
-        where: {
-          projectId: projectId,
-          bookmarked: true,
-        },
-        select: {
-          id: true,
-          createdAt: true,
-          bookmarked: true,
-          public: true,
-        },
-      })
-    : [];
+  let additionalBookmarkFilter: z.infer<typeof singleFilter>[] = [];
+  if (sessionsBookmarkedFilter) {
+    // We are only fetching bookmarked sessions.
+    // They need to be manipulated in the UI and should not be as many.
+    const filteredSessions = await prisma.traceSession.findMany({
+      where: {
+        projectId: projectId,
+        bookmarked: true,
+      },
+      select: {
+        id: true,
+        createdAt: true,
+        bookmarked: true,
+        public: true,
+      },
+    });
 
-  const additionalBookmarkFilter: z.infer<typeof stringOptionsFilter>[] =
-    sessionsBookmarkedFilter &&
-    typeof sessionsBookmarkedFilter.value === "boolean" &&
-    ((sessionsBookmarkedFilter.value === true &&
-      sessionsBookmarkedFilter.operator === "=") ||
+    // Check which operator we want to use for the bookmark filter.
+    let operator: "any of" | "none of" | undefined = undefined;
+
+    // If the value is true and we check for equality we include it or if the value is false and we check for inequality we include it.
+    if (
+      (sessionsBookmarkedFilter.value === true &&
+        sessionsBookmarkedFilter.operator === "=") ||
       (sessionsBookmarkedFilter.value === false &&
-        sessionsBookmarkedFilter.operator === "<>"))
-      ? [
-          {
-            column: "id",
-            type: "stringOptions" as const,
-            operator: "any of" as const,
-            value: filteredSessions.map((s) => s.id),
-          },
-        ]
-      : sessionsBookmarkedFilter &&
-          typeof sessionsBookmarkedFilter.value === "boolean" &&
-          (sessionsBookmarkedFilter.value === false ||
-            (sessionsBookmarkedFilter.value === true &&
-              sessionsBookmarkedFilter.operator === "<>"))
+        sessionsBookmarkedFilter.operator === "<>")
+    ) {
+      operator = "any of";
+    }
+
+    // Now we check the opposite case where the value should be none of the given values.
+    if (
+      sessionsBookmarkedFilter.value === false ||
+      (sessionsBookmarkedFilter.value === true &&
+        sessionsBookmarkedFilter.operator === "<>")
+    ) {
+      operator = "none of";
+    }
+
+    // Decide whether we want to propagate the bookmark filter down to the sessions queries
+    additionalBookmarkFilter =
+      operator && typeof sessionsBookmarkedFilter.value === "boolean"
         ? [
             {
               column: "id",
               type: "stringOptions" as const,
-              operator: "none of" as const,
+              operator,
               value: filteredSessions.map((s) => s.id),
             },
           ]
         : [];
+  }
 
   return filter
     ? [...filter.filter((f) => f.column !== "⭐️"), ...additionalBookmarkFilter]

packages/shared/src/server/repositories/traces.ts

@@ -17,7 +17,7 @@ import {
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
-import { UiColumnMapping } from "../../tableDefinitions";
+import { UiColumnMappings } from "../../tableDefinitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -254,6 +254,37 @@ export const getTraceCountsByProjectInCreationInterval = async ({
   }));
 };
 
+export const getTraceCountOfProjectsSinceCreationDate = async ({
+  projectIds,
+  start,
+}: {
+  projectIds: string[];
+  start: Date;
+}) => {
+  const query = `
+    SELECT 
+      count(*) as count
+    FROM traces
+    WHERE project_id IN ({projectIds: Array(String)})
+    AND created_at >= {start: DateTime64(3)}
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectIds,
+      start: convertDateToClickhouseDateTime(start),
+    },
+    tags: {
+      feature: "tracing",
+      type: "trace",
+      kind: "analytic",
+    },
+  });
+
+  return Number(rows[0]?.count ?? 0);
+};
+
 export const getTraceById = async (
   traceId: string,
   projectId: string,
@@ -293,7 +324,7 @@ export const getTraceById = async (
 
 export const getTracesGroupedByName = async (
   projectId: string,
-  tableDefinitions: UiColumnMapping[] = tracesTableUiColumnDefinitions,
+  tableDefinitions: UiColumnMappings = tracesTableUiColumnDefinitions,
   timestampFilter?: FilterState,
 ) => {
   const chFilter = timestampFilter
@@ -345,7 +376,7 @@ export const getTracesGroupedByUsers = async (
   searchQuery?: string,
   limit?: number,
   offset?: number,
-  columns?: UiColumnMapping[],
+  columns?: UiColumnMappings,
 ) => {
   const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
     tracesPrefix: "t",
@@ -404,7 +435,7 @@ export const getTracesGroupedByUsers = async (
 export type GroupedTracesQueryProp = {
   projectId: string;
   filter: FilterState;
-  columns?: UiColumnMapping[];
+  columns?: UiColumnMappings;
 };
 
 export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
@@ -653,6 +684,7 @@ export const getUserMetrics = async (
       WITH stats as (
         SELECT
             t.user_id as user_id,
+            anyLast(t.environment) as environment,
             count(distinct o.id) as obs_count,
             sumMap(usage_details) as sum_usage_details,
             sum(total_cost) as sum_total_cost,
@@ -695,6 +727,7 @@ export const getUserMetrics = async (
                     t.user_id,
                     t.project_id,
                     t.timestamp,
+                    t.environment,
                     ROW_NUMBER() OVER (
                         PARTITION BY id
                         ORDER BY
@@ -721,6 +754,7 @@ export const getUserMetrics = async (
         obs_count,
         trace_count,
         user_id,
+        environment,
         sum_total_cost,
         max_timestamp,
         min_timestamp
@@ -731,6 +765,7 @@ export const getUserMetrics = async (
 
   const rows = await queryClickhouse<{
     user_id: string;
+    environment: string;
     max_timestamp: string;
     min_timestamp: string;
     input_usage: string;
@@ -763,6 +798,7 @@ export const getUserMetrics = async (
 
   return rows.map((row) => ({
     userId: row.user_id,
+    environment: row.environment,
     maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
     minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
     inputUsage: Number(row.input_usage),
@@ -774,6 +810,52 @@ export const getUserMetrics = async (
   }));
 };
 
+export const getTracesForBlobStorageExport = function (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) {
+  const query = `
+    SELECT
+      id,
+      timestamp,
+      name,
+      environment,
+      project_id,
+      metadata,
+      user_id,
+      session_id,
+      release,
+      version,
+      public,
+      bookmarked,
+      tags,
+      input,
+      output
+    FROM traces FINAL
+    WHERE project_id = {projectId: String}
+    AND timestamp >= {minTimestamp: DateTime64(3)}
+    AND timestamp <= {maxTimestamp: DateTime64(3)}
+  `;
+
+  const records = queryClickhouseStream<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+    tags: {
+      feature: "blobstorage",
+      type: "trace",
+      kind: "analytic",
+      projectId,
+    },
+  });
+
+  return records;
+};
+
 export const getTracesForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
@@ -825,6 +907,13 @@ export const getTracesForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");

packages/shared/src/server/services/PromptService.ts

@@ -1,250 +0,0 @@
-import { Prompt, PrismaClient } from "@prisma/client";
-import { Redis } from "ioredis";
-import { env } from "../../env";
-import { logger } from "../logger";
-
-export class PromptService {
-  private cacheEnabled: boolean;
-  private ttlSeconds: number;
-
-  constructor(
-    private prisma: PrismaClient,
-    private redis: Redis | null,
-    private metricIncrementer?: // used for otel metrics
-    (name: string, value?: number) => void,
-    cacheEnabled?: boolean, // used for testing
-  ) {
-    this.cacheEnabled =
-      Boolean(redis) &&
-      (cacheEnabled || env.LANGFUSE_CACHE_PROMPT_ENABLED === "true");
-
-    this.ttlSeconds = env.LANGFUSE_CACHE_PROMPT_TTL_SECONDS;
-  }
-
-  public async getPrompt(params: PromptParams): Promise<Prompt | null> {
-    if (await this.shouldUseCache(params)) {
-      const cachedPrompt = await this.getCachedPrompt(params);
-
-      this.incrementMetric(
-        cachedPrompt ? Metrics.PromptCacheHit : Metrics.PromptCacheMiss,
-      );
-
-      if (cachedPrompt) {
-        this.logDebug("Returning cached prompt for params", params);
-
-        return cachedPrompt;
-      }
-    }
-
-    const dbPrompt = await this.getDbPrompt(params);
-
-    if ((await this.shouldUseCache(params)) && dbPrompt) {
-      await this.cachePrompt({ ...params, prompt: dbPrompt });
-
-      this.logInfo("Successfully cached prompt for params", params);
-    }
-
-    this.logInfo("Returning DB prompt for params", params);
-
-    return dbPrompt;
-  }
-
-  private async getDbPrompt(params: PromptParams): Promise<Prompt | null> {
-    const { projectId, promptName, version, label } = params;
-
-    if (version) {
-      return await this.prisma.prompt.findFirst({
-        where: {
-          projectId,
-          name: promptName,
-          version,
-        },
-      });
-    }
-
-    if (label) {
-      return await this.prisma.prompt.findFirst({
-        where: {
-          projectId,
-          name: promptName,
-          labels: {
-            has: label,
-          },
-        },
-      });
-    }
-
-    this.logError("Invalid prompt params", params);
-
-    return null;
-  }
-
-  private async shouldUseCache(params: PromptParams): Promise<boolean> {
-    if (!this.cacheEnabled) return false;
-
-    const isLocked = await this.isCacheLocked(params);
-
-    if (isLocked) {
-      this.logInfo("Cache is locked for params", params);
-    }
-
-    return !isLocked;
-  }
-
-  private async getCachedPrompt(params: PromptParams): Promise<Prompt | null> {
-    try {
-      const key = this.getCacheKey(params);
-      const value = await this.redis?.getex(key, "EX", this.ttlSeconds);
-
-      if (value) return JSON.parse(value) as Prompt;
-    } catch (e) {
-      this.logError("Error getting cached prompt", e);
-    }
-
-    return null;
-  }
-
-  private async cachePrompt(params: PromptParams & { prompt: Prompt }) {
-    try {
-      const keyIndexKey = this.getKeyIndexKey(params);
-      const key = this.getCacheKey(params);
-      const value = JSON.stringify(params.prompt);
-
-      await this.redis?.sadd(keyIndexKey, key);
-      await this.redis?.set(key, value, "EX", this.ttlSeconds);
-    } catch (e) {
-      this.logError("Error caching prompt", e);
-    }
-  }
-
-  public async lockCache(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): Promise<void> {
-    if (!this.cacheEnabled) return;
-
-    const lockKey = this.getLockKey(params);
-
-    try {
-      await this.redis?.setex(lockKey, 30, "locked");
-    } catch (e) {
-      this.logError("Error locking cache key prefix", lockKey, e);
-
-      throw e;
-    }
-  }
-
-  public async unlockCache(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): Promise<void> {
-    if (!this.cacheEnabled) return;
-
-    const lockKey = this.getLockKey(params);
-
-    try {
-      await this.redis?.del(lockKey);
-    } catch (e) {
-      this.logError("Error unlocking cache key prefix", lockKey, e);
-
-      // Don't re-throw error as lock TTL is short and it's not critical
-    }
-  }
-
-  private async isCacheLocked(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): Promise<boolean> {
-    const lockKey = this.getLockKey(params);
-
-    try {
-      return Boolean(await this.redis?.exists(lockKey));
-    } catch (e) {
-      this.logError("Error checking if cache is locked", lockKey, e);
-
-      return false;
-    }
-  }
-
-  private getLockKey(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): string {
-    // Important to *pre*fix LOCK as otherwise it would be deleted by deleteKeysByPrefix
-    return `LOCK:${this.getCacheKeyPrefix(params)}`;
-  }
-
-  public async invalidateCache(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): Promise<void> {
-    if (!this.cacheEnabled) return;
-
-    const cacheKeyPrefix = this.getCacheKeyPrefix(params);
-
-    try {
-      const startTime = Date.now();
-      this.logInfo("Invalidating cache for prefix", cacheKeyPrefix);
-
-      const keyIndexKey = this.getKeyIndexKey(params);
-      const keys = await this.redis?.smembers(keyIndexKey);
-
-      // Delete all keys for the prefix and the key index
-      await this.redis?.del([...(keys ?? []), keyIndexKey]);
-
-      this.logInfo(
-        `Cache invalidated for prefix ${cacheKeyPrefix} in ${Date.now() - startTime}ms`,
-      );
-    } catch (e) {
-      this.logError("Error deleting keys for prefix", cacheKeyPrefix, e);
-
-      throw e;
-    }
-  }
-
-  private getCacheKey(params: PromptParams): string {
-    const prefix = this.getCacheKeyPrefix(params);
-
-    return `${prefix}:${params.version ?? params.label}`;
-  }
-
-  private getCacheKeyPrefix(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): string {
-    return `prompt:${params.projectId}:${params.promptName}`;
-  }
-
-  private getKeyIndexKey(
-    params: Pick<PromptParams, "projectId" | "promptName">,
-  ): string {
-    return `prompt_key_index:${params.projectId}:${params.promptName}`;
-  }
-
-  private logError(message: string, ...args: any[]) {
-    logger.error(`[PromptService] ${message}`, ...args);
-  }
-
-  private logInfo(message: string, ...args: any[]) {
-    logger.info(`[PromptService] ${message}`, ...args);
-  }
-
-  private logDebug(message: string, ...args: any[]) {
-    logger.debug(`[PromptService] ${message}`, ...args);
-  }
-
-  private incrementMetric(name: Metrics, value: number = 1) {
-    try {
-      this.metricIncrementer?.(name, value);
-    } catch (e) {
-      this.logError("Error incrementing metric", name, e);
-    }
-  }
-}
-
-type PromptParams = {
-  projectId: string;
-  promptName: string;
-} & (
-  | { version: number; label: undefined }
-  | { version: null | undefined; label: string }
-);
-
-enum Metrics {
-  PromptCacheHit = "prompt_cache_hit",
-  PromptCacheMiss = "prompt_cache_miss",
-}

packages/shared/src/server/services/PromptService/index.ts

@@ -0,0 +1,424 @@
+import { Prompt, PrismaClient } from "@prisma/client";
+import { Redis } from "ioredis";
+import { env } from "../../../env";
+import { logger } from "../../logger";
+import { escapeRegex } from "./utils";
+import {
+  PromptGraph,
+  PromptParams,
+  PartialPrompt,
+  ResolvedPromptGraph,
+  PromptServiceMetrics,
+  PromptResult,
+} from "./types";
+
+import { ParsedPromptDependencyTag } from "../../../features/prompts/parsePromptDependencyTags";
+
+export const MAX_PROMPT_NESTING_DEPTH = 5;
+
+export class PromptService {
+  private cacheEnabled: boolean;
+  private ttlSeconds: number;
+
+  constructor(
+    private prisma: PrismaClient,
+    private redis: Redis | null,
+    private metricIncrementer?: // used for otel metrics
+    (name: string, value?: number) => void,
+    cacheEnabled?: boolean, // used for testing
+  ) {
+    this.cacheEnabled =
+      Boolean(redis) &&
+      (cacheEnabled || env.LANGFUSE_CACHE_PROMPT_ENABLED === "true");
+
+    this.ttlSeconds = env.LANGFUSE_CACHE_PROMPT_TTL_SECONDS;
+  }
+
+  public async getPrompt(params: PromptParams): Promise<PromptResult | null> {
+    if (await this.shouldUseCache(params)) {
+      const cachedPrompt = await this.getCachedPrompt(params);
+
+      this.incrementMetric(
+        cachedPrompt
+          ? PromptServiceMetrics.PromptCacheHit
+          : PromptServiceMetrics.PromptCacheMiss,
+      );
+
+      if (cachedPrompt) {
+        this.logDebug("Returning cached prompt for params", params);
+
+        return cachedPrompt;
+      }
+    }
+
+    const dbPrompt = await this.getDbPrompt(params);
+
+    if ((await this.shouldUseCache(params)) && dbPrompt) {
+      await this.cachePrompt({ ...params, prompt: dbPrompt });
+
+      this.logInfo("Successfully cached prompt for params", params);
+    }
+
+    this.logInfo("Returning DB prompt for params", params);
+
+    return dbPrompt;
+  }
+
+  private async getDbPrompt(
+    params: PromptParams,
+  ): Promise<PromptResult | null> {
+    const { projectId, promptName, version, label } = params;
+
+    if (version) {
+      const prompt = await this.prisma.prompt.findFirst({
+        where: {
+          projectId,
+          name: promptName,
+          version,
+        },
+      });
+
+      return this.resolvePrompt(prompt);
+    }
+
+    if (label) {
+      const prompt = await this.prisma.prompt.findFirst({
+        where: {
+          projectId,
+          name: promptName,
+          labels: {
+            has: label,
+          },
+        },
+      });
+
+      return this.resolvePrompt(prompt);
+    }
+
+    this.logError("Invalid prompt params", params);
+
+    return null;
+  }
+
+  private async resolvePrompt(
+    prompt: Prompt | null,
+  ): Promise<PromptResult | null> {
+    if (!prompt) return prompt;
+
+    const promptGraph = await this.buildAndResolvePromptGraph({
+      projectId: prompt.projectId,
+      parentPrompt: prompt,
+    });
+
+    return {
+      ...prompt,
+      prompt: promptGraph.resolvedPrompt,
+      resolutionGraph: promptGraph.graph,
+    };
+  }
+
+  private async shouldUseCache(params: PromptParams): Promise<boolean> {
+    if (!this.cacheEnabled) return false;
+
+    const isLocked = await this.isCacheLocked(params);
+
+    if (isLocked) {
+      this.logInfo("Cache is locked for params", params);
+    }
+
+    return !isLocked;
+  }
+
+  private async getCachedPrompt(
+    params: PromptParams,
+  ): Promise<PromptResult | null> {
+    try {
+      const key = this.getCacheKey(params);
+      const value = await this.redis?.getex(key, "EX", this.ttlSeconds);
+
+      if (value) return JSON.parse(value) as PromptResult;
+    } catch (e) {
+      this.logError("Error getting cached prompt", e);
+    }
+
+    return null;
+  }
+
+  private async cachePrompt(params: PromptParams & { prompt: PromptResult }) {
+    try {
+      const keyIndexKey = this.getKeyIndexKey(params);
+      const key = this.getCacheKey(params);
+      const value = JSON.stringify(params.prompt);
+
+      await this.redis?.sadd(keyIndexKey, key);
+      await this.redis?.set(key, value, "EX", this.ttlSeconds);
+    } catch (e) {
+      this.logError("Error caching prompt", e);
+    }
+  }
+
+  public async lockCache(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): Promise<void> {
+    if (!this.cacheEnabled) return;
+
+    const lockKey = this.getLockKey(params);
+
+    try {
+      await this.redis?.setex(lockKey, 30, "locked");
+    } catch (e) {
+      this.logError("Error locking cache key prefix", lockKey, e);
+
+      throw e;
+    }
+  }
+
+  public async unlockCache(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): Promise<void> {
+    if (!this.cacheEnabled) return;
+
+    const lockKey = this.getLockKey(params);
+
+    try {
+      await this.redis?.del(lockKey);
+    } catch (e) {
+      this.logError("Error unlocking cache key prefix", lockKey, e);
+
+      // Don't re-throw error as lock TTL is short and it's not critical
+    }
+  }
+
+  private async isCacheLocked(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): Promise<boolean> {
+    const lockKey = this.getLockKey(params);
+
+    try {
+      return Boolean(await this.redis?.exists(lockKey));
+    } catch (e) {
+      this.logError("Error checking if cache is locked", lockKey, e);
+
+      return false;
+    }
+  }
+
+  private getLockKey(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): string {
+    // Important to *pre*fix LOCK as otherwise it would be deleted by deleteKeysByPrefix
+    return `LOCK:prompt:${params.projectId}`;
+  }
+
+  public async invalidateCache(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): Promise<void> {
+    if (!this.cacheEnabled) return;
+
+    const keyIndexKey = this.getKeyIndexKey(params);
+    const keys = await this.redis?.smembers(keyIndexKey);
+
+    /*
+     * Previously, the cache key index was based on both projectId and promptName.
+     * Now with prompt composability, we only use projectId for the key index.
+     * When invalidating the cache, we delete all keys for a projectId.
+     * For backwards compatibility, we also clear any existing entries in the old
+     * key index format (projectId + promptName) to ensure consistent caching.
+     */
+    const legacyKeyIndexKey = `${keyIndexKey}:${params.promptName}`;
+    const legacyKeys = await this.redis?.smembers(legacyKeyIndexKey);
+
+    // Delete all keys for the prefix and the key index
+    await this.redis?.del([
+      ...(keys ?? []),
+      keyIndexKey,
+      ...(legacyKeys ?? []),
+      legacyKeyIndexKey,
+    ]);
+  }
+
+  private getCacheKey(params: PromptParams): string {
+    const prefix = this.getCacheKeyPrefix(params);
+
+    return `${prefix}:${params.version ?? params.label}`;
+  }
+
+  private getCacheKeyPrefix(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): string {
+    return `prompt:${params.projectId}:${params.promptName}`;
+  }
+
+  private getKeyIndexKey(
+    params: Pick<PromptParams, "projectId" | "promptName">,
+  ): string {
+    return `prompt_key_index:${params.projectId}`;
+  }
+
+  public async buildAndResolvePromptGraph(params: {
+    projectId: string;
+    parentPrompt: PartialPrompt;
+    dependencies?: ParsedPromptDependencyTag[];
+  }): Promise<ResolvedPromptGraph> {
+    try {
+      const { projectId, parentPrompt, dependencies } = params;
+
+      const graph: PromptGraph = {
+        root: {
+          name: parentPrompt.name,
+          version: parentPrompt.version,
+          id: parentPrompt.id,
+        },
+        dependencies: {},
+      };
+      const seen = new Set<string>();
+
+      const resolve = async (
+        currentPrompt: PartialPrompt,
+        deps: ParsedPromptDependencyTag[] | undefined,
+        level: number,
+      ) => {
+        // Nesting depth check
+        if (level >= MAX_PROMPT_NESTING_DEPTH) {
+          throw Error(
+            `Maximum nesting depth exceeded (${MAX_PROMPT_NESTING_DEPTH})`,
+          );
+        }
+
+        // Circular dependency check
+        if (
+          seen.has(currentPrompt.id) ||
+          (currentPrompt.name === parentPrompt.name &&
+            currentPrompt.id !== parentPrompt.id) // ensure that the parent prompt cannot reference a prompt of the same name but different version
+        ) {
+          throw Error(
+            `Circular dependency detected involving prompt '${currentPrompt.name}' version ${currentPrompt.version}`,
+          );
+        }
+
+        seen.add(currentPrompt.id);
+
+        // deps can be either passed (if a prompt is created and content was scanned) or retrieved from db
+        let promptDependencies = deps;
+        if (!deps) {
+          promptDependencies = (
+            await this.prisma.promptDependency.findMany({
+              where: {
+                projectId,
+                parentId: currentPrompt.id,
+              },
+              select: {
+                childName: true,
+                childLabel: true,
+                childVersion: true,
+              },
+            })
+          ).map(
+            (dep) =>
+              ({
+                name: dep.childName,
+                ...(dep.childVersion
+                  ? { type: "version", version: dep.childVersion }
+                  : { type: "label", label: dep.childLabel }),
+              }) as ParsedPromptDependencyTag,
+          );
+        }
+
+        if (promptDependencies && promptDependencies.length) {
+          // Instantiate resolved prompt, use stringfied version for regex operations
+          // Do this inside if clause to skip stringify/parse overhead for prompts without dependencies
+          let resolvedPrompt = JSON.stringify(currentPrompt.prompt);
+
+          for (const dep of promptDependencies) {
+            const depPrompt = await this.prisma.prompt.findFirst({
+              where: {
+                projectId,
+                name: dep.name,
+                ...(dep.type === "version"
+                  ? { version: dep.version }
+                  : { labels: { has: dep.label } }),
+              },
+            });
+
+            const logName = `${dep.name} - ${dep.type} ${dep.type === "version" ? dep.version : dep.label}`;
+
+            if (!depPrompt)
+              throw Error(`Prompt dependency not found: ${logName}`);
+            if (depPrompt.type !== "text")
+              throw Error(`Prompt dependency is not a text prompt: ${logName}`);
+
+            // side-effect: populate adjacency list to return later as well
+            graph.dependencies[currentPrompt.id] ??= []; // initializes an empty list if it does not exist yet
+            graph.dependencies[currentPrompt.id].push({
+              id: depPrompt.id,
+              name: depPrompt.name,
+              version: depPrompt.version,
+            });
+
+            // resolve the prompt content recursively
+            const resolvedDepPrompt = await resolve(
+              depPrompt,
+              undefined,
+              level + 1,
+            );
+
+            const versionPattern = `@@@langfusePrompt:name=${escapeRegex(depPrompt.name)}\\|version=${escapeRegex(depPrompt.version)}@@@`;
+            const labelPatterns = depPrompt.labels.map(
+              (label) =>
+                `@@@langfusePrompt:name=${escapeRegex(depPrompt.name)}\\|label=${escapeRegex(label)}@@@`,
+            );
+            const combinedPattern = [versionPattern, ...labelPatterns].join(
+              "|",
+            );
+            const regex = new RegExp(combinedPattern, "g");
+
+            const replaceValue = JSON.stringify(resolvedDepPrompt)
+              .slice(1, -1) // this is necessary to avoid parsing errors as resolved value is unstringified
+              .replace(/\$/g, "$$$$"); // Escape dollar signs in replacement string. $ has special meaning in replace(), so we need to escape it with $$. Since we're in a string that will be used as a replacement, we need to double escape it (hence $$$$)
+
+            resolvedPrompt = resolvedPrompt.replace(regex, replaceValue);
+          }
+
+          seen.delete(currentPrompt.id);
+
+          return JSON.parse(resolvedPrompt);
+        } else {
+          seen.delete(currentPrompt.id);
+
+          return currentPrompt.prompt;
+        }
+      };
+
+      const resolvedPrompt = await resolve(parentPrompt, dependencies, 0);
+
+      return {
+        graph: Object.keys(graph.dependencies).length > 0 ? graph : null,
+        resolvedPrompt,
+      };
+    } catch (err) {
+      console.error(err);
+
+      throw err;
+    }
+  }
+
+  private logError(message: string, ...args: any[]) {
+    logger.error(`[PromptService] ${message}`, ...args);
+  }
+
+  private logInfo(message: string, ...args: any[]) {
+    logger.info(`[PromptService] ${message}`, ...args);
+  }
+
+  private logDebug(message: string, ...args: any[]) {
+    logger.debug(`[PromptService] ${message}`, ...args);
+  }
+
+  private incrementMetric(name: PromptServiceMetrics, value: number = 1) {
+    try {
+      this.metricIncrementer?.(name, value);
+    } catch (e) {
+      this.logError("Error incrementing metric", name, e);
+    }
+  }
+}

packages/shared/src/server/services/PromptService/types.ts

@@ -0,0 +1,35 @@
+import { type Prompt } from "../../../db";
+
+export type PromptResult = Prompt & {
+  resolutionGraph: PromptGraph | null;
+};
+
+export type PromptParams = {
+  projectId: string;
+  promptName: string;
+} & (
+  | { version: number; label: undefined }
+  | { version: null | undefined; label: string }
+);
+
+export enum PromptServiceMetrics {
+  PromptCacheHit = "prompt_cache_hit",
+  PromptCacheMiss = "prompt_cache_miss",
+}
+
+export type PartialPrompt = Pick<
+  Prompt,
+  "id" | "prompt" | "name" | "version" | "labels"
+>;
+
+export type PromptReference = Pick<Prompt, "id" | "version" | "name">;
+
+export type PromptGraph = {
+  root: PromptReference;
+  dependencies: Record<string, PromptReference[]>;
+};
+
+export type ResolvedPromptGraph = {
+  graph: PromptGraph | null;
+  resolvedPrompt: Prompt["prompt"];
+};

packages/shared/src/server/services/PromptService/utils.ts

@@ -0,0 +1,3 @@
+export function escapeRegex(str: string | number) {
+  return String(str).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

packages/shared/src/server/services/StorageService.ts

@@ -14,6 +14,7 @@ import {
   ContainerClient,
   StorageSharedKeyCredential,
 } from "@azure/storage-blob";
+import { Storage, Bucket, GetSignedUrlConfig } from "@google-cloud/storage";
 import { logger } from "../logger";
 import { env } from "../../env";
 
@@ -51,17 +52,47 @@ export interface StorageService {
 }
 
 export class StorageServiceFactory {
+  /**
+   * Get an instance of the StorageService
+   * @param params.accessKeyId - Access key ID
+   * @param params.secretAccessKey - Secret access key
+   * @param params.bucketName - Bucket name to store files
+   * @param params.endpoint - Endpoint - Endpoint to an S3 compatible API (or Azure Blob Storage)
+   * @param params.externalEndpoint - External endpoint to replace the internal endpoint in the signed URL.
+   * @param params.region - Region in which the bucket resides
+   * @param params.forcePathStyle - Add bucket name into the path instead of the domain name. Mainly used for MinIO.
+   * @param params.useAzureBlob - Use Azure Blob Storage instead of S3
+   * @param params.useGoogleCloudStorage - Use Google Cloud Storage instead of S3
+   * @param params.googleCloudCredentials - Google Cloud Storage credentials JSON string or path to credentials file
+   */
   public static getInstance(params: {
     accessKeyId: string | undefined;
     secretAccessKey: string | undefined;
     bucketName: string;
     endpoint: string | undefined;
+    externalEndpoint?: string | undefined;
     region: string | undefined;
     forcePathStyle: boolean;
+    useAzureBlob?: boolean;
+    useGoogleCloudStorage?: boolean;
+    googleCloudCredentials?: string;
   }): StorageService {
-    if (env.LANGFUSE_USE_AZURE_BLOB === "true") {
+    if (params.useAzureBlob || env.LANGFUSE_USE_AZURE_BLOB === "true") {
       return new AzureBlobStorageService(params);
     }
+    if (
+      params.useGoogleCloudStorage ||
+      env.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE === "true"
+    ) {
+      // Use provided credentials or fall back to environment variable
+      const googleParams = {
+        ...params,
+        googleCloudCredentials:
+          params.googleCloudCredentials ||
+          env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS,
+      };
+      return new GoogleCloudStorageService(googleParams);
+    }
     return new S3StorageService(params);
   }
 }
@@ -69,22 +100,25 @@ export class StorageServiceFactory {
 class AzureBlobStorageService implements StorageService {
   private client: ContainerClient;
   private container: string;
+  private externalEndpoint: string | undefined;
 
   constructor(params: {
     accessKeyId: string | undefined;
     secretAccessKey: string | undefined;
     bucketName: string;
     endpoint: string | undefined;
+    externalEndpoint?: string | undefined;
     region: string | undefined;
     forcePathStyle: boolean;
   }) {
-    const { accessKeyId, secretAccessKey, endpoint } = params;
+    const { accessKeyId, secretAccessKey, endpoint, externalEndpoint } = params;
     if (!accessKeyId || !secretAccessKey || !endpoint) {
       throw new Error(
         `Endpoint, account and account key must be configured to use Azure Blob Storage`,
       );
     }
 
+    this.externalEndpoint = externalEndpoint;
     const sharedKeyCredential = new StorageSharedKeyCredential(
       accessKeyId,
       secretAccessKey,
@@ -119,10 +153,13 @@ class AzureBlobStorageService implements StorageService {
       if (typeof data === "string") {
         await blockBlobClient.upload(data, data.length);
       } else if (data instanceof Readable) {
-        let offset = 0;
         const blockIds = [];
         for await (const chunk of data) {
-          const blockId = Buffer.from(`block-${offset}`).toString("base64");
+          // Azure requires block IDs to be base64 strings of the same length
+          // Use a fixed format with padded index to ensure consistent length
+          const blockIdStr: string = `block-${blockIds.length.toString().padStart(10, "0")}`;
+          const blockId = Buffer.from(blockIdStr).toString("base64");
+
           const bufferChunk = Buffer.isBuffer(chunk)
             ? chunk
             : Buffer.from(chunk);
@@ -133,11 +170,10 @@ class AzureBlobStorageService implements StorageService {
             bufferChunk.length,
           );
           blockIds.push(blockId);
-
-          offset += bufferChunk.length;
         }
-
-        await blockBlobClient.commitBlockList(blockIds);
+        if (blockIds.length > 0) {
+          await blockBlobClient.commitBlockList(blockIds);
+        }
       } else {
         throw new Error("Unsupported data type. Must be Readable or string.");
       }
@@ -258,13 +294,20 @@ class AzureBlobStorageService implements StorageService {
       await this.createContainerIfNotExists();
 
       const blockBlobClient = this.client.getBlockBlobClient(fileName);
-      return blockBlobClient.generateSasUrl({
+      let url = await blockBlobClient.generateSasUrl({
         permissions: BlobSASPermissions.parse("r"),
         expiresOn: new Date(Date.now() + ttlSeconds * 1000),
         contentDisposition: asAttachment
           ? `attachment; filename="${fileName}"`
           : undefined,
       });
+
+      // Replace internal endpoint with external endpoint if configured
+      if (this.externalEndpoint && url.includes(this.client.url)) {
+        url = url.replace(this.client.url, this.externalEndpoint);
+      }
+
+      return url;
     } catch (err) {
       logger.error(
         `Failed to generate presigned URL for Azure Blob Storage ${fileName}`,
@@ -286,11 +329,18 @@ class AzureBlobStorageService implements StorageService {
       await this.createContainerIfNotExists();
 
       const blockBlobClient = this.client.getBlockBlobClient(path);
-      return blockBlobClient.generateSasUrl({
+      let url = await blockBlobClient.generateSasUrl({
         permissions: BlobSASPermissions.parse("w"),
         expiresOn: new Date(Date.now() + ttlSeconds * 1000),
         contentType: contentType,
       });
+
+      // Replace internal endpoint with external endpoint if configured
+      if (this.externalEndpoint && url.includes(this.client.url)) {
+        url = url.replace(this.client.url, this.externalEndpoint);
+      }
+
+      return url;
     } catch (err) {
       logger.error(
         `Failed to generate presigned upload URL for Azure Blob Storage ${path}`,
@@ -306,12 +356,15 @@ class AzureBlobStorageService implements StorageService {
 class S3StorageService implements StorageService {
   private client: S3Client;
   private bucketName: string;
+  private endpoint: string | undefined;
+  private externalEndpoint: string | undefined;
 
   constructor(params: {
     accessKeyId: string | undefined;
     secretAccessKey: string | undefined;
     bucketName: string;
     endpoint: string | undefined;
+    externalEndpoint?: string | undefined;
     region: string | undefined;
     forcePathStyle: boolean;
   }) {
@@ -330,8 +383,15 @@ class S3StorageService implements StorageService {
       endpoint: params.endpoint,
       region: params.region,
       forcePathStyle: params.forcePathStyle,
+      requestHandler: {
+        httpsAgent: {
+          maxSockets: env.LANGFUSE_S3_CONCURRENT_WRITES,
+        },
+      },
     });
     this.bucketName = params.bucketName;
+    this.endpoint = params.endpoint;
+    this.externalEndpoint = params.externalEndpoint;
   }
 
   public async uploadFile({
@@ -420,7 +480,7 @@ class S3StorageService implements StorageService {
     asAttachment: boolean = true,
   ): Promise<string> {
     try {
-      return await getSignedUrl(
+      let url = await getSignedUrl(
         this.client,
         new GetObjectCommand({
           Bucket: this.bucketName,
@@ -431,6 +491,17 @@ class S3StorageService implements StorageService {
         }),
         { expiresIn: ttlSeconds },
       );
+
+      // Replace internal endpoint with external endpoint if configured
+      if (
+        this.externalEndpoint &&
+        this.endpoint &&
+        url.includes(this.endpoint)
+      ) {
+        url = url.replace(this.endpoint, this.externalEndpoint);
+      }
+
+      return url;
     } catch (err) {
       logger.error(`Failed to generate presigned URL for ${fileName}`, err);
       throw Error("Failed to generate signed URL");
@@ -477,7 +548,7 @@ class S3StorageService implements StorageService {
   }): Promise<string> {
     const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
 
-    return await getSignedUrl(
+    let url = await getSignedUrl(
       this.client,
       new PutObjectCommand({
         Bucket: this.bucketName,
@@ -492,5 +563,228 @@ class S3StorageService implements StorageService {
         unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
       },
     );
+
+    // Replace internal endpoint with external endpoint if configured
+    if (this.externalEndpoint && this.endpoint && url.includes(this.endpoint)) {
+      url = url.replace(this.endpoint, this.externalEndpoint);
+    }
+
+    return url;
+  }
+}
+
+class GoogleCloudStorageService implements StorageService {
+  private storage: Storage;
+  private bucket: Bucket;
+
+  constructor(params: { bucketName: string; googleCloudCredentials?: string }) {
+    // Initialize Google Cloud Storage client
+    if (params.googleCloudCredentials) {
+      try {
+        // Check if the credentials are a JSON string or a path to a file
+        if (params.googleCloudCredentials.trim().startsWith("{")) {
+          // It's a JSON string
+          this.storage = new Storage({
+            credentials: JSON.parse(params.googleCloudCredentials),
+          });
+        } else {
+          // It's a path to a credentials file
+          this.storage = new Storage({
+            keyFilename: params.googleCloudCredentials,
+          });
+        }
+      } catch (err) {
+        logger.error("Failed to parse Google Cloud Storage credentials", err);
+        throw new Error("Failed to initialize Google Cloud Storage");
+      }
+    } else {
+      // Use default authentication (environment variables or instance metadata)
+      this.storage = new Storage();
+    }
+
+    this.bucket = this.storage.bucket(params.bucketName);
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      const file = this.bucket.file(fileName);
+      const options = {
+        contentType: fileType,
+        resumable: false,
+      };
+
+      if (typeof data === "string") {
+        await file.save(data, options);
+        const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+        return { signedUrl };
+      } else if (data instanceof Readable) {
+        return new Promise((resolve, reject) => {
+          const writeStream = file.createWriteStream(options);
+
+          data
+            .pipe(writeStream)
+            .on("error", (err) => {
+              reject(err);
+            })
+            .on("finish", async () => {
+              try {
+                const signedUrl = await this.getSignedUrl(
+                  fileName,
+                  expiresInSeconds,
+                );
+                resolve({ signedUrl });
+              } catch (err) {
+                reject(err);
+              }
+            });
+        });
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Google Cloud Storage ${fileName}`,
+        err,
+      );
+      throw new Error("Failed to upload to Google Cloud Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    try {
+      const file = this.bucket.file(path);
+      const content = JSON.stringify(body);
+
+      await file.save(content, {
+        contentType: "application/json",
+        resumable: false,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to upload JSON to Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to upload JSON to Google Cloud Storage");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      const file = this.bucket.file(path);
+      const [content] = await file.download();
+
+      return content.toString();
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Google Cloud Storage");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    try {
+      const [files] = await this.bucket.getFiles({ prefix });
+
+      return files.map((file) => ({
+        file: file.name,
+        createdAt: new Date(file.metadata.timeCreated ?? new Date()),
+      }));
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Google Cloud Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Google Cloud Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = false,
+  ): Promise<string> {
+    try {
+      const file = this.bucket.file(fileName);
+
+      const options: GetSignedUrlConfig = {
+        version: "v4",
+        action: "read",
+        expires: Date.now() + ttlSeconds * 1000,
+      };
+
+      if (asAttachment) {
+        options.responseDisposition = `attachment; filename="${fileName}"`;
+      }
+
+      const [url] = await file.getSignedUrl(options);
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate signed URL for Google Cloud Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate signed URL for Google Cloud Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+
+    try {
+      const file = this.bucket.file(path);
+
+      const options: GetSignedUrlConfig = {
+        version: "v4",
+        action: "write",
+        expires: Date.now() + ttlSeconds * 1000,
+        contentType,
+        extensionHeaders: {
+          "Content-Length": params.contentLength.toString(),
+        },
+      };
+
+      const [url] = await file.getSignedUrl(options);
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate signed upload URL for Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate signed upload URL for Google Cloud Storage",
+      );
+    }
+  }
+
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      await Promise.all(
+        paths.map(async (path) => {
+          const file = this.bucket.file(path);
+          await file.delete({ ignoreNotFound: true });
+        }),
+      );
+    } catch (err) {
+      logger.error(`Failed to delete files from Google Cloud Storage`, err);
+      throw Error("Failed to delete files from Google Cloud Storage");
+    }
   }
 }

packages/shared/src/server/services/datasets-ui-table-service.ts

@@ -0,0 +1,61 @@
+import { evalDatasetFormFilterCols } from "../../tableDefinitions/tracesTable";
+import { FilterState } from "../../types";
+import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
+import { Prisma, prisma } from "../../db";
+
+type FetchDatasetItemsTableProps = {
+  select: "count";
+  projectId: string;
+  filter: FilterState;
+};
+
+const getDatasetRunItemsTableGeneric = async <T>(
+  props: FetchDatasetItemsTableProps,
+) => {
+  const { select, projectId, filter } = props;
+
+  let sqlSelect: Prisma.Sql;
+  switch (select) {
+    case "count":
+      sqlSelect = Prisma.sql`count(*) as count`;
+      break;
+    default:
+      // eslint-disable-next-line no-case-declarations, no-unused-vars
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
+  const datasetItemsFilter = tableColumnsToSqlFilterAndPrefix(
+    filter,
+    evalDatasetFormFilterCols,
+    "dataset_items",
+  );
+
+  const query = Prisma.sql`
+    SELECT 
+      ${sqlSelect}
+      FROM dataset_run_items as dri
+        JOIN dataset_items as di ON di.id = dri.dataset_item_id AND di.project_id = ${projectId}
+        WHERE dri.project_id = ${projectId}
+        ${datasetItemsFilter}
+  `;
+
+  const res = await prisma.$queryRaw<T>(query);
+
+  return res;
+};
+
+export const getDatasetRunItemsTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+}) => {
+  const res = await getDatasetRunItemsTableGeneric<Array<{ count: bigint }>>({
+    select: "count",
+    projectId: props.projectId,
+    filter: props.filter,
+  });
+
+  const totalCount = res.length > 0 ? Number(res[0].count) : 0;
+
+  return { totalCount };
+};

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -20,6 +20,7 @@ export type SessionDataReturnType = {
   user_ids: string[];
   trace_count: number;
   trace_tags: string[];
+  trace_environment?: string;
 };
 
 export type SessionWithMetricsReturnType = SessionDataReturnType & {
@@ -129,7 +130,8 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
           trace_ids, 
           user_ids, 
           trace_count, 
-          trace_tags`;
+          trace_tags,
+          trace_environment`;
       break;
     case "metrics":
       sqlSelect = `
@@ -140,6 +142,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         user_ids, 
         trace_count, 
         trace_tags,
+        trace_environment,
         total_observations,
         duration,
         session_usage_details,
@@ -151,9 +154,10 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         session_output_usage,
         session_total_usage`;
       break;
-    default:
+    default: {
       const exhaustiveCheckDefault: never = select;
-      throw new Error(`Unknown select type: ${select}`);
+      throw new Error(`Unknown select type: ${exhaustiveCheckDefault}`);
+    }
   }
 
   const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
@@ -162,7 +166,9 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
 
   tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
 
-  const tracesFilterRes = tracesFilter.apply();
+  const tracesFilterRes = tracesFilter
+    .filter((f) => f.field !== "environment")
+    .apply();
 
   const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
     (f) =>
@@ -183,7 +189,10 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
   }
 
   const additionalSingleTraceFilter = tracesFilter.find(
-    (f) => f.field === "bookmarked" || f.field === "session_id",
+    (f) =>
+      f.field === "bookmarked" ||
+      f.field === "session_id" ||
+      f.field === "environment",
   );
 
   if (additionalSingleTraceFilter) {
@@ -264,7 +273,8 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
             groupArray(t.id) AS trace_ids,
             groupUniqArray(t.user_id) AS user_ids,
             count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags
+            groupUniqArrayArray(t.tags) as trace_tags,
+            anyLast(t.environment) as trace_environment
             -- Aggregate observations data at session level
             ${
               selectMetrics

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -35,6 +35,7 @@ export type TracesTableReturnType = Pick<
   | "version"
   | "user_id"
   | "session_id"
+  | "environment"
   | "tags"
   | "public"
 >;
@@ -49,6 +50,7 @@ export type TracesAllUiReturnType = {
   version: string | null;
   public: boolean;
   bookmarked: boolean;
+  environment: string | null;
   sessionId: string | null;
   tags: string[];
 };
@@ -87,6 +89,7 @@ export const convertToUiTableRows = (
     release: row.release ?? null,
     version: row.version ?? null,
     userId: row.user_id ?? null,
+    environment: row.environment ?? null,
     sessionId: row.session_id ?? null,
     public: row.public,
   };
@@ -259,6 +262,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
         t.release as release,
         t.version as version,
         t.user_id as user_id,
+        t.environment as environment,
         t.session_id as session_id,
         t.public as public`;
       break;

packages/shared/src/server/test-utils/org-factory.ts

@@ -1,6 +1,7 @@
 import { v4 } from "uuid";
 import { prisma } from "../../db";
 import { hashSecretKey, getDisplaySecretKey } from "../auth/apiKeys";
+import { CloudConfigSchema } from "../../interfaces/cloudConfigSchema";
 
 export function createBasicAuthHeader(
   username: string,
@@ -23,9 +24,12 @@ export const createOrgProjectAndApiKey = async (
     data: {
       id: v4(),
       name: v4(),
+      cloudConfig: CloudConfigSchema.parse({
+        plan: "Team",
+      }),
     },
   });
-  await prisma.project.create({
+  const project = await prisma.project.create({
     data: {
       id: projectId,
       name: v4(),
@@ -46,5 +50,5 @@ export const createOrgProjectAndApiKey = async (
     },
   });
 
-  return { projectId, publicKey, secretKey, auth };
+  return { projectId, orgId: org.id, publicKey, secretKey, auth, org, project };
 };

packages/shared/src/tableDefinitions/mapDashboards.ts

@@ -1,6 +1,8 @@
-import { UiColumnMapping } from "./types";
+import { UiColumnMappings } from "./types";
 
-export const dashboardColumnDefinitions: UiColumnMapping[] = [
+// Make sure to update web/src/features/query/dashboardUiTableToViewMapping.ts if you make changes
+
+export const dashboardColumnDefinitions: UiColumnMappings = [
   {
     uiTableName: "Trace Name",
     uiTableId: "traceName",
@@ -91,4 +93,10 @@ export const dashboardColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "model",
     uiTableName: "Model",
   },
+  {
+    clickhouseTableName: "traces",
+    clickhouseSelect: "environment",
+    uiTableId: "environment",
+    uiTableName: "Environment",
+  },
 ];

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -1,9 +1,9 @@
 // This structure is maintained to relate the frontend table definitions with the clickhouse table definitions.
 // The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.
 
-import { UiColumnMapping } from "./types";
+import { UiColumnMappings } from "./types";
 
-export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [
+export const observationsTableTraceUiColumnDefinitions: UiColumnMappings = [
   {
     uiTableName: "Trace Tags",
     uiTableId: "traceTags",
@@ -22,10 +22,22 @@ export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "traces",
     clickhouseSelect: 't."name"',
   },
+  {
+    uiTableName: "Trace Environment",
+    uiTableId: "traceEnvironment",
+    clickhouseTableName: "traces",
+    clickhouseSelect: 't."environment"',
+  },
 ];
 
-export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
+export const observationsTableUiColumnDefinitions: UiColumnMappings = [
   ...observationsTableTraceUiColumnDefinitions,
+  {
+    uiTableName: "Environment",
+    uiTableId: "environment",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."environment"',
+  },
   {
     uiTableName: "type",
     uiTableId: "type",

packages/shared/src/tableDefinitions/mapScoresTable.ts

@@ -1,6 +1,6 @@
-import { UiColumnMapping } from "./types";
+import { UiColumnMappings } from "./types";
 
-export const scoresTableUiColumnDefinitions: UiColumnMapping[] = [
+export const scoresTableUiColumnDefinitions: UiColumnMappings = [
   {
     uiTableName: "ID",
     uiTableId: "id",
@@ -13,6 +13,12 @@ export const scoresTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "scores",
     clickhouseSelect: "timestamp",
   },
+  {
+    uiTableName: "Environment",
+    uiTableId: "environment",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "environment",
+  },
   {
     uiTableName: "Trace ID",
     uiTableId: "traceId",

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -1,6 +1,6 @@
-import { UiColumnMapping } from ".";
+import { UiColumnMappings } from ".";
 
-export const sessionCols: UiColumnMapping[] = [
+export const sessionCols: UiColumnMappings = [
   // we do not access the traces scores in ClickHouse. We default back to the trace timestamps.
 
   {
@@ -21,6 +21,12 @@ export const sessionCols: UiColumnMapping[] = [
     clickhouseTableName: "traces",
     clickhouseSelect: "user_ids",
   },
+  {
+    uiTableName: "Environment",
+    uiTableId: "environment",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "environment",
+  },
   {
     uiTableName: "Session Duration",
     uiTableId: "sessionDuration",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -1,6 +1,6 @@
-import { UiColumnMapping } from "./types";
+import { UiColumnMappings } from "./types";
 
-export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
+export const tracesTableUiColumnDefinitions: UiColumnMappings = [
   {
     uiTableName: "⭐️",
     uiTableId: "bookmarked",
@@ -62,6 +62,12 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "traces",
     clickhouseSelect: "release",
   },
+  {
+    uiTableName: "Environment",
+    uiTableId: "environment",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "environment",
+  },
   {
     uiTableName: "Tags",
     uiTableId: "tags",

packages/shared/src/tableDefinitions/types.ts

@@ -1,11 +1,13 @@
-export type UiColumnMapping = {
+export type UiColumnMappings = readonly UiColumnMapping[];
+
+export type UiColumnMapping = Readonly<{
   uiTableName: string;
   uiTableId: string;
   clickhouseTableName: string;
   clickhouseSelect: string;
   clickhouseTypeOverwrite?: string;
   queryPrefix?: string;
-};
+}>;
 
 export type OptionsDefinition = {
   value: string;

packages/shared/src/utils/json.ts

@@ -1,4 +1,5 @@
 import { JsonNested } from "./zod";
+import { parse, isSafeNumber, isNumber } from "lossless-json";
 
 /**
  * Deeply parses a JSON string or object for nested stringified JSON
@@ -41,7 +42,18 @@ export const parseJsonPrioritised = (
   json: string,
 ): JsonNested | string | undefined => {
   try {
-    return JSON.parse(json);
+    return parse(json, null, (value) => {
+      if (isNumber(value)) {
+        if (isSafeNumber(value)) {
+          // Safe numbers (integers and decimals) can be converted to Number
+          return Number(value.valueOf());
+        } else {
+          // For large integers beyond safe limits, preserve string representation
+          return value.toString();
+        }
+      }
+      return value;
+    }) as JsonNested;
   } catch (error) {
     return json;
   }

turbo.json

@@ -20,7 +20,7 @@
     "dev": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate"]
+      "dependsOn": ["db:generate", "@langfuse/shared#build"]
     },
     "db:generate": {
       "cache": false,

web/Dockerfile

@@ -97,6 +97,8 @@ WORKDIR /app
 
 ARG NEXT_PUBLIC_BUILD_ID
 ENV BUILD_ID=$NEXT_PUBLIC_BUILD_ID
+ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ENV NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
 
 ENV NODE_ENV production
 # Uncomment the following line in case you want to disable telemetry during runtime.
@@ -116,6 +118,12 @@ RUN adduser --system --uid ${UID} nextjs
 
 RUN npm install -g --no-package-lock --no-save prisma@6.3.0
 
+# Install dd-trace only if NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is configured
+ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+RUN if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \
+        npm install --no-package-lock --no-save dd-trace@5.36.0; \
+    fi
+
 RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
     wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
     mv migrate /usr/bin/migrate
@@ -141,8 +149,11 @@ USER nextjs
 # Default port to 3000
 ENV PORT 3000
 
-
 ENTRYPOINT ["dumb-init", "--", "./web/entrypoint.sh"]
 
-# startup command
-CMD ["node", "./web/server.js", "--keepAliveTimeout", "110000"]
+# startup command - use dd-trace if NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is configured
+CMD if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \
+      node --import dd-trace/initialize.mjs ./web/server.js --keepAliveTimeout 110000; \
+    else \
+      node ./web/server.js --keepAliveTimeout 110000; \
+    fi

web/next.config.mjs

@@ -79,6 +79,16 @@ const nextConfig = {
 
   async headers() {
     return [
+      {
+        // Add noindex for all pages except root and /auth*
+        source: "/:path((?!auth|^$).*)*",
+        headers: [
+          {
+            key: "X-Robots-Tag",
+            value: "noindex",
+          },
+        ],
+      },
       {
         source: "/:path*",
         headers: [
@@ -170,6 +180,9 @@ const nextConfig = {
       layers: true,
     };
 
+    // Exclude Datadog packages from webpack bundling to avoid issues
+    config.externals.push("@datadog/pprof", "dd-trace");
+
     return config;
   },
 };
@@ -201,7 +214,7 @@ export default withSentryConfig(nextConfig, {
   // This can increase your server load as well as your hosting bill.
   // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client-
   // side errors will fail.
-  // tunnelRoute: "/api/monitoring-tunnel",
+  tunnelRoute: "/api/monitoring-tunnel",
 
   // Hides source maps from generated client bundles
   hideSourceMaps: true,

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.35.1",
+  "version": "3.46.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -14,12 +14,12 @@
     "prettier": "prettier --write ./src *.{ts,js}",
     "clean": "rm -rf node_modules",
     "start": "dotenv -e ../.env -- sh -c 'NEXT_MANUAL_SIG_HANDLE=true next start'",
-    "test": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects async-server",
-    "test-sync": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects sync-server",
-    "test-client": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
-    "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
+    "test": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects async-server",
+    "test-sync": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects sync-server",
+    "test-client": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
+    "test:watch": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server"
+    "test:e2e:server": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
@@ -122,7 +122,7 @@
     "langchain": "^0.3.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.462.0",
-    "next": "^14.2.21",
+    "next": "^14.2.25",
     "next-auth": "^4.24.11",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
@@ -155,7 +155,7 @@
     "unified": "^11.0.5",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
-    "vaul": "^0.9.1",
+    "vaul": "^1.1.2",
     "vis-network": "^9.1.9",
     "zod": "^3.23.8"
   },
@@ -181,6 +181,7 @@
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",
     "autoprefixer": "^10.4.19",
+    "cross-env": "^7.0.3",
     "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
     "eslint-config-next": "^14.2.15",

web/public/generated/api/openapi.yml

@@ -24,6 +24,391 @@ info:
     - Postman collection:
     https://cloud.langfuse.com/generated/postman/collection.json
 paths:
+  /api/public/annotation-queues:
+    get:
+      description: Get all annotation queues
+      operationId: annotationQueues_listQueues
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedAnnotationQueues'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+  /api/public/annotation-queues/{queueId}:
+    get:
+      description: Get an annotation queue by ID
+      operationId: annotationQueues_getQueue
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnnotationQueue'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+  /api/public/annotation-queues/{queueId}/items:
+    get:
+      description: Get items for a specific annotation queue
+      operationId: annotationQueues_listQueueItems
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+        - name: status
+          in: query
+          description: Filter by status
+          required: false
+          schema:
+            $ref: '#/components/schemas/AnnotationQueueStatus'
+            nullable: true
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedAnnotationQueueItems'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+    post:
+      description: Add an item to an annotation queue
+      operationId: annotationQueues_createQueueItem
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnnotationQueueItem'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateAnnotationQueueItemRequest'
+  /api/public/annotation-queues/{queueId}/items/{itemId}:
+    get:
+      description: Get a specific item from an annotation queue
+      operationId: annotationQueues_getQueueItem
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+        - name: itemId
+          in: path
+          description: The unique identifier of the annotation queue item
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnnotationQueueItem'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+    patch:
+      description: Update an annotation queue item
+      operationId: annotationQueues_updateQueueItem
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+        - name: itemId
+          in: path
+          description: The unique identifier of the annotation queue item
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnnotationQueueItem'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UpdateAnnotationQueueItemRequest'
+    delete:
+      description: Remove an item from an annotation queue
+      operationId: annotationQueues_deleteQueueItem
+      tags:
+        - AnnotationQueues
+      parameters:
+        - name: queueId
+          in: path
+          description: The unique identifier of the annotation queue
+          required: true
+          schema:
+            type: string
+        - name: itemId
+          in: path
+          description: The unique identifier of the annotation queue item
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteAnnotationQueueItemResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
   /api/public/comments:
     post:
       description: >-
@@ -2536,6 +2921,52 @@ paths:
               schema: {}
       security:
         - BasicAuth: []
+    delete:
+      description: Delete a specific trace
+      operationId: trace_delete
+      tags:
+        - Trace
+      parameters:
+        - name: traceId
+          in: path
+          description: The unique langfuse identifier of the trace to delete
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteTraceResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
   /api/public/traces:
     get:
       description: Get list of traces
@@ -2675,8 +3106,192 @@ paths:
               schema: {}
       security:
         - BasicAuth: []
+    delete:
+      description: Delete multiple traces
+      operationId: trace_deleteMultiple
+      tags:
+        - Trace
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteTraceResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                traceIds:
+                  type: array
+                  items:
+                    type: string
+                  description: List of trace IDs to delete
+              required:
+                - traceIds
 components:
   schemas:
+    AnnotationQueueStatus:
+      title: AnnotationQueueStatus
+      type: string
+      enum:
+        - PENDING
+        - COMPLETED
+    AnnotationQueueObjectType:
+      title: AnnotationQueueObjectType
+      type: string
+      enum:
+        - TRACE
+        - OBSERVATION
+    AnnotationQueue:
+      title: AnnotationQueue
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        description:
+          type: string
+          nullable: true
+        scoreConfigIds:
+          type: array
+          items:
+            type: string
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+      required:
+        - id
+        - name
+        - scoreConfigIds
+        - createdAt
+        - updatedAt
+    AnnotationQueueItem:
+      title: AnnotationQueueItem
+      type: object
+      properties:
+        id:
+          type: string
+        queueId:
+          type: string
+        objectId:
+          type: string
+        objectType:
+          $ref: '#/components/schemas/AnnotationQueueObjectType'
+        status:
+          $ref: '#/components/schemas/AnnotationQueueStatus'
+        completedAt:
+          type: string
+          format: date-time
+          nullable: true
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+      required:
+        - id
+        - queueId
+        - objectId
+        - objectType
+        - status
+        - createdAt
+        - updatedAt
+    PaginatedAnnotationQueues:
+      title: PaginatedAnnotationQueues
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/AnnotationQueue'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    PaginatedAnnotationQueueItems:
+      title: PaginatedAnnotationQueueItems
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/AnnotationQueueItem'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    CreateAnnotationQueueItemRequest:
+      title: CreateAnnotationQueueItemRequest
+      type: object
+      properties:
+        objectId:
+          type: string
+        objectType:
+          $ref: '#/components/schemas/AnnotationQueueObjectType'
+        status:
+          $ref: '#/components/schemas/AnnotationQueueStatus'
+          nullable: true
+          description: Defaults to PENDING for new queue items
+      required:
+        - objectId
+        - objectType
+    UpdateAnnotationQueueItemRequest:
+      title: UpdateAnnotationQueueItemRequest
+      type: object
+      properties:
+        status:
+          $ref: '#/components/schemas/AnnotationQueueStatus'
+          nullable: true
+    DeleteAnnotationQueueItemResponse:
+      title: DeleteAnnotationQueueItemResponse
+      type: object
+      properties:
+        success:
+          type: boolean
+        message:
+          type: string
+      required:
+        - success
+        - message
     CreateCommentRequest:
       title: CreateCommentRequest
       type: object
@@ -4365,9 +4980,10 @@ components:
       required:
         - successes
         - errors
-    OpenAIUsageSchema:
-      title: OpenAIUsageSchema
+    OpenAICompletionUsageSchema:
+      title: OpenAICompletionUsageSchema
       type: object
+      description: OpenAI Usage schema from (Chat-)Completion APIs
       properties:
         prompt_tokens:
           type: integer
@@ -4379,23 +4995,53 @@ components:
           type: object
           additionalProperties:
             type: integer
+            nullable: true
           nullable: true
         completion_tokens_details:
           type: object
           additionalProperties:
             type: integer
+            nullable: true
           nullable: true
       required:
         - prompt_tokens
         - completion_tokens
         - total_tokens
+    OpenAIResponseUsageSchema:
+      title: OpenAIResponseUsageSchema
+      type: object
+      description: OpenAI Usage schema from Response API
+      properties:
+        input_tokens:
+          type: integer
+        output_tokens:
+          type: integer
+        total_tokens:
+          type: integer
+        input_tokens_details:
+          type: object
+          additionalProperties:
+            type: integer
+            nullable: true
+          nullable: true
+        output_tokens_details:
+          type: object
+          additionalProperties:
+            type: integer
+            nullable: true
+          nullable: true
+      required:
+        - input_tokens
+        - output_tokens
+        - total_tokens
     UsageDetails:
       title: UsageDetails
       oneOf:
         - type: object
           additionalProperties:
             type: integer
-        - $ref: '#/components/schemas/OpenAIUsageSchema'
+        - $ref: '#/components/schemas/OpenAICompletionUsageSchema'
+        - $ref: '#/components/schemas/OpenAIResponseUsageSchema'
     GetMediaResponse:
       title: GetMediaResponse
       type: object
@@ -4899,6 +5545,13 @@ components:
           type: string
           nullable: true
           description: Commit message for this prompt version.
+        resolutionGraph:
+          type: object
+          additionalProperties: true
+          nullable: true
+          description: >-
+            The dependency resolution graph for the current prompt. Null if
+            prompt has no dependencies.
       required:
         - name
         - version
@@ -5174,6 +5827,14 @@ components:
       required:
         - data
         - meta
+    DeleteTraceResponse:
+      title: DeleteTraceResponse
+      type: object
+      properties:
+        message:
+          type: string
+      required:
+        - message
     Sort:
       title: Sort
       type: object

web/public/generated/postman/collection.json

@@ -37,6 +37,296 @@
     ]
   },
   "item": [
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Annotation Queues",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "List Queues",
+          "request": {
+            "description": "Get all annotation queues",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Queue",
+          "request": {
+            "description": "Get an annotation queue by ID",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List Queue Items",
+          "request": {
+            "description": "Get items for a specific annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/items?status=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "items"
+              ],
+              "query": [
+                {
+                  "key": "status",
+                  "value": "",
+                  "description": "Filter by status"
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Queue Item",
+          "request": {
+            "description": "Get a specific item from an annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/items/:itemId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "items",
+                ":itemId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                },
+                {
+                  "key": "itemId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue item"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create Queue Item",
+          "request": {
+            "description": "Add an item to an annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/items",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "items"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                }
+              ]
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"objectId\": \"example\",\n    \"objectType\": \"TRACE\",\n    \"status\": \"PENDING\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Update Queue Item",
+          "request": {
+            "description": "Update an annotation queue item",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/items/:itemId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "items",
+                ":itemId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                },
+                {
+                  "key": "itemId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue item"
+                }
+              ]
+            },
+            "header": [],
+            "method": "PATCH",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"status\": \"PENDING\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete Queue Item",
+          "request": {
+            "description": "Remove an item from an annotation queue",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/annotation-queues/:queueId/items/:itemId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "annotation-queues",
+                ":queueId",
+                "items",
+                ":itemId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "queueId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue"
+                },
+                {
+                  "key": "itemId",
+                  "value": "",
+                  "description": "The unique identifier of the annotation queue item"
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -1913,6 +2203,38 @@
           },
           "response": []
         },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a specific trace",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/traces/:traceId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "traces",
+                ":traceId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "traceId",
+                  "value": "",
+                  "description": "The unique langfuse identifier of the trace to delete"
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "List",
@@ -1998,6 +2320,39 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete Multiple",
+          "request": {
+            "description": "Delete multiple traces",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/traces",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "traces"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"traceIds\": [\n        \"example\"\n    ]\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
         }
       ]
     }

web/sentry.client.config.ts

@@ -7,10 +7,13 @@ Sentry.init({
   // Replay may only be enabled for the client-side
   integrations: [
     Sentry.replayIntegration(),
+    Sentry.browserTracingIntegration(),
+    Sentry.httpClientIntegration(),
     // Sentry.debugIntegration(),
-    // Sentry.captureConsoleIntegration({
-    //   levels: ["error"],
-    // }),
+    Sentry.captureConsoleIntegration({
+      levels: ["error"],
+    }),
+    Sentry.browserProfilingIntegration(),
   ],
 
   // Set tracesSampleRate to 1.0 to capture 100%
@@ -28,6 +31,12 @@ Sentry.init({
   replaysOnErrorSampleRate: 1.0,
   debug: false,
 
+  // Set profilesSampleRate to 1.0 to profile every transaction.
+  // Since profilesSampleRate is relative to tracesSampleRate,
+  // the final profiling rate can be computed as tracesSampleRate * profilesSampleRate
+  // For example, a tracesSampleRate of 0.5 and profilesSampleRate of 0.5 would
+  // result in 25% of transactions being profiled (0.5*0.5=0.25)
+  profilesSampleRate: 0.5,
   // ...
 
   // Note: if you want to override the automatic release value, do not set a

web/src/__tests__/annotation-queues-api.servertest.ts

@@ -0,0 +1,656 @@
+/** @jest-environment node */
+
+import { prisma } from "@langfuse/shared/src/db";
+import {
+  makeZodVerifiedAPICall,
+  makeAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  GetAnnotationQueuesResponse,
+  GetAnnotationQueueByIdResponse,
+  GetAnnotationQueueItemsResponse,
+  GetAnnotationQueueItemByIdResponse,
+  CreateAnnotationQueueItemResponse,
+  UpdateAnnotationQueueItemResponse,
+  DeleteAnnotationQueueItemResponse,
+} from "@/src/features/public-api/types/annotation-queues";
+import {
+  AnnotationQueueObjectType,
+  AnnotationQueueStatus,
+} from "@langfuse/shared";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import { v4 as uuidv4 } from "uuid";
+
+describe("Annotation Queues API Endpoints", () => {
+  let auth: string;
+  let projectId: string;
+  let queueId: string;
+  let queueItemId: string;
+  const TOTAL_TEST_QUEUES = 15; // Create enough queues to test pagination
+  const TOTAL_TEST_QUEUE_ITEMS = 20; // Create enough queue items to test pagination
+
+  beforeAll(async () => {
+    // Create organization, project, and API key for testing
+    const { auth: newAuth, projectId: newProjectId } =
+      await createOrgProjectAndApiKey();
+    auth = newAuth;
+    projectId = newProjectId;
+
+    // Create multiple test annotation queues
+    const queuePromises = [];
+    for (let i = 0; i < TOTAL_TEST_QUEUES; i++) {
+      queuePromises.push(
+        prisma.annotationQueue.create({
+          data: {
+            name: `Test Queue ${i + 1}`,
+            description: `Test Queue Description ${i + 1}`,
+            scoreConfigIds: [],
+            projectId,
+          },
+        }),
+      );
+    }
+
+    const queues = await Promise.all(queuePromises);
+    queueId = queues[0].id; // Use the first queue for specific tests
+
+    // Create multiple test annotation queue items
+    const queueItemPromises = [];
+    for (let i = 0; i < TOTAL_TEST_QUEUE_ITEMS; i++) {
+      // Distribute items across the first 3 queues to test filtering
+      const targetQueueId = queues[i % 3].id;
+
+      // Alternate between PENDING and COMPLETED status
+      const status =
+        i % 2 === 0
+          ? AnnotationQueueStatus.PENDING
+          : AnnotationQueueStatus.COMPLETED;
+
+      // Set completedAt for COMPLETED items
+      const completedAt =
+        status === AnnotationQueueStatus.COMPLETED ? new Date() : null;
+
+      queueItemPromises.push(
+        prisma.annotationQueueItem.create({
+          data: {
+            queueId: targetQueueId,
+            objectId: uuidv4(),
+            objectType: AnnotationQueueObjectType.TRACE,
+            status,
+            completedAt,
+            projectId,
+          },
+        }),
+      );
+    }
+
+    const queueItems = await Promise.all(queueItemPromises);
+    queueItemId = queueItems[0].id; // Use the first queue item for specific tests
+  });
+
+  afterAll(async () => {
+    await pruneDatabase();
+  });
+
+  describe("GET /annotation-queues", () => {
+    it("should get all annotation queues", async () => {
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueuesResponse,
+        "GET",
+        "/api/public/annotation-queues",
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.data.length).toBeGreaterThan(0);
+      expect(response.body.data[0]).toHaveProperty("id");
+      expect(response.body.data[0]).toHaveProperty("name");
+      expect(response.body.meta).toHaveProperty("totalItems");
+      expect(response.body.meta.totalItems).toBe(TOTAL_TEST_QUEUES);
+    });
+
+    it("should support pagination with correct limits", async () => {
+      const limit = 5;
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueuesResponse,
+        "GET",
+        `/api/public/annotation-queues?page=1&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.meta.page).toBe(1);
+      expect(response.body.meta.limit).toBe(limit);
+      expect(response.body.data.length).toBe(limit);
+      expect(response.body.meta.totalItems).toBe(TOTAL_TEST_QUEUES);
+      expect(response.body.meta.totalPages).toBe(
+        Math.ceil(TOTAL_TEST_QUEUES / limit),
+      );
+    });
+
+    it("should return different results for different pages", async () => {
+      const limit = 5;
+
+      // Get first page
+      const firstPageResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueuesResponse,
+        "GET",
+        `/api/public/annotation-queues?page=1&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      // Get second page
+      const secondPageResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueuesResponse,
+        "GET",
+        `/api/public/annotation-queues?page=2&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      expect(firstPageResponse.status).toBe(200);
+      expect(secondPageResponse.status).toBe(200);
+
+      // Check that we got different items on each page
+      const firstPageIds = firstPageResponse.body.data.map((queue) => queue.id);
+      const secondPageIds = secondPageResponse.body.data.map(
+        (queue) => queue.id,
+      );
+
+      // No IDs should be in both pages
+      const intersection = firstPageIds.filter((id) =>
+        secondPageIds.includes(id),
+      );
+      expect(intersection.length).toBe(0);
+
+      // Both pages should have the expected number of items
+      expect(firstPageResponse.body.data.length).toBe(limit);
+      expect(secondPageResponse.body.data.length).toBe(limit);
+    });
+  });
+
+  describe("GET /annotation-queues/:queueId", () => {
+    it("should get a specific annotation queue", async () => {
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueueByIdResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.id).toBe(queueId);
+      expect(response.body.name).toBe("Test Queue 1");
+      expect(response.body.description).toBe("Test Queue Description 1");
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "GET",
+        `/api/public/annotation-queues/${nonExistentId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("GET /annotation-queues/:queueId/items", () => {
+    it("should get all items for a specific queue", async () => {
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.data.length).toBeGreaterThan(0);
+      expect(response.body.data[0]).toHaveProperty("id");
+      expect(response.body.data[0]).toHaveProperty("queueId");
+      expect(response.body.meta).toHaveProperty("totalItems");
+      expect(response.body.data.every((item) => item.queueId === queueId)).toBe(
+        true,
+      );
+    });
+
+    it("should support pagination with correct limits for queue items", async () => {
+      const limit = 7;
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items?page=1&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.meta.page).toBe(1);
+      expect(response.body.meta.limit).toBe(limit);
+      expect(response.body.data.length).toBeGreaterThan(0);
+      expect(response.body.data.every((item) => item.queueId === queueId)).toBe(
+        true,
+      );
+    });
+
+    it("should return different results for different pages of queue items", async () => {
+      // First, create enough items to ensure we have multiple pages
+      const itemsToCreate = 15;
+      const createPromises = [];
+
+      for (let i = 0; i < itemsToCreate; i++) {
+        createPromises.push(
+          makeZodVerifiedAPICall(
+            CreateAnnotationQueueItemResponse,
+            "POST",
+            `/api/public/annotation-queues/${queueId}/items`,
+            {
+              objectId: uuidv4(),
+              objectType: AnnotationQueueObjectType.TRACE,
+            },
+            auth,
+          ),
+        );
+      }
+
+      await Promise.all(createPromises);
+
+      const limit = 7;
+
+      // Get first page
+      const firstPageResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items?page=1&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      // Get second page
+      const secondPageResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items?page=2&limit=${limit}`,
+        undefined,
+        auth,
+      );
+
+      expect(firstPageResponse.status).toBe(200);
+      expect(secondPageResponse.status).toBe(200);
+
+      // Check that we got different items on each page
+      const firstPageIds = firstPageResponse.body.data.map((item) => item.id);
+      const secondPageIds = secondPageResponse.body.data.map((item) => item.id);
+
+      // No IDs should be in both pages
+      const intersection = firstPageIds.filter((id) =>
+        secondPageIds.includes(id),
+      );
+      expect(intersection.length).toBe(0);
+
+      // Both pages should have items
+      expect(firstPageResponse.body.data.length).toBe(limit);
+      expect(secondPageResponse.body.data.length).toBeGreaterThan(0);
+    });
+
+    it("should filter by status", async () => {
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items?status=PENDING`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.data.length).toBeGreaterThan(0);
+      expect(
+        response.body.data.every(
+          (item) =>
+            item.queueId === queueId &&
+            item.status === AnnotationQueueStatus.PENDING,
+        ),
+      ).toBe(true);
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "GET",
+        `/api/public/annotation-queues/${nonExistentId}/items`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("GET /annotation-queues/:queueId/items/:itemId", () => {
+    it("should get a specific annotation queue item", async () => {
+      // First, get an item that belongs to the queue
+      const itemsResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items?limit=1`,
+        undefined,
+        auth,
+      );
+
+      expect(itemsResponse.status).toBe(200);
+      expect(itemsResponse.body.data.length).toBeGreaterThan(0);
+
+      const itemId = itemsResponse.body.data[0].id;
+
+      const response = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemByIdResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items/${itemId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.id).toBe(itemId);
+      expect(response.body.queueId).toBe(queueId);
+    });
+
+    it("should return 404 for non-existent queue item", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items/${nonExistentId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "GET",
+        `/api/public/annotation-queues/${nonExistentId}/items/${queueItemId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("POST /annotation-queues/:queueId/items", () => {
+    it("should create a new annotation queue item", async () => {
+      const objectId = uuidv4();
+      const response = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId,
+          objectType: AnnotationQueueObjectType.TRACE,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.queueId).toBe(queueId);
+      expect(response.body.objectId).toBe(objectId);
+      expect(response.body.objectType).toBe(AnnotationQueueObjectType.TRACE);
+      expect(response.body.status).toBe(AnnotationQueueStatus.PENDING);
+    });
+
+    it("should create queue items with different object types and statuses", async () => {
+      // Create a queue item with TRACE object type
+      const traceObjectId = uuidv4();
+      const traceResponse = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId: traceObjectId,
+          objectType: AnnotationQueueObjectType.TRACE,
+          status: AnnotationQueueStatus.COMPLETED,
+        },
+        auth,
+      );
+
+      expect(traceResponse.status).toBe(200);
+      expect(traceResponse.body.objectType).toBe(
+        AnnotationQueueObjectType.TRACE,
+      );
+
+      // Create a queue item with OBSERVATION object type
+      const observationObjectId = uuidv4();
+      const observationResponse = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId: observationObjectId,
+          objectType: AnnotationQueueObjectType.OBSERVATION,
+          status: AnnotationQueueStatus.PENDING,
+        },
+        auth,
+      );
+
+      expect(observationResponse.status).toBe(200);
+      expect(observationResponse.body.objectType).toBe(
+        AnnotationQueueObjectType.OBSERVATION,
+      );
+
+      // Verify we can retrieve items with different object types
+      const itemsResponse = await makeZodVerifiedAPICall(
+        GetAnnotationQueueItemsResponse,
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items`,
+        undefined,
+        auth,
+      );
+
+      expect(itemsResponse.status).toBe(200);
+
+      // Find our created items in the response
+      const traceItem = itemsResponse.body.data.find(
+        (item) => item.objectId === traceObjectId,
+      );
+      const observationItem = itemsResponse.body.data.find(
+        (item) => item.objectId === observationObjectId,
+      );
+
+      expect(traceItem).toBeDefined();
+      expect(traceItem?.objectType).toBe(AnnotationQueueObjectType.TRACE);
+      expect(traceItem?.status).toBe(AnnotationQueueStatus.COMPLETED);
+      expect(observationItem).toBeDefined();
+      expect(observationItem?.objectType).toBe(
+        AnnotationQueueObjectType.OBSERVATION,
+      );
+      expect(observationItem?.status).toBe(AnnotationQueueStatus.PENDING);
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "POST",
+        `/api/public/annotation-queues/${nonExistentId}/items`,
+        {
+          objectId: uuidv4(),
+          objectType: AnnotationQueueObjectType.TRACE,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("PATCH /annotation-queues/:queueId/items/:itemId", () => {
+    it("should update an annotation queue item to COMPLETED", async () => {
+      // Create a new item to update
+      const createResponse = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId: uuidv4(),
+          objectType: AnnotationQueueObjectType.TRACE,
+        },
+        auth,
+      );
+
+      const newItemId = createResponse.body.id;
+
+      const response = await makeZodVerifiedAPICall(
+        UpdateAnnotationQueueItemResponse,
+        "PATCH",
+        `/api/public/annotation-queues/${queueId}/items/${newItemId}`,
+        {
+          status: AnnotationQueueStatus.COMPLETED,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.id).toBe(newItemId);
+      expect(response.body.status).toBe(AnnotationQueueStatus.COMPLETED);
+      expect(response.body.completedAt).not.toBeNull();
+    });
+
+    it("should update an annotation queue item from COMPLETED to PENDING", async () => {
+      // Create a new item with COMPLETED status
+      const createResponse = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId: uuidv4(),
+          objectType: AnnotationQueueObjectType.TRACE,
+          status: AnnotationQueueStatus.COMPLETED,
+        },
+        auth,
+      );
+
+      const newItemId = createResponse.body.id;
+
+      // Verify it was created with COMPLETED status
+      expect(createResponse.body.status).toBe(AnnotationQueueStatus.COMPLETED);
+      expect(createResponse.body.completedAt).not.toBeNull();
+
+      // Update it to PENDING
+      const response = await makeZodVerifiedAPICall(
+        UpdateAnnotationQueueItemResponse,
+        "PATCH",
+        `/api/public/annotation-queues/${queueId}/items/${newItemId}`,
+        {
+          status: AnnotationQueueStatus.PENDING,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.id).toBe(newItemId);
+      expect(response.body.status).toBe(AnnotationQueueStatus.PENDING);
+      expect(response.body.completedAt).not.toBeNull(); // is not reset by moving to PENDING
+    });
+
+    it("should return 404 for non-existent queue item", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "PATCH",
+        `/api/public/annotation-queues/${queueId}/items/${nonExistentId}`,
+        {
+          status: AnnotationQueueStatus.COMPLETED,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "PATCH",
+        `/api/public/annotation-queues/${nonExistentId}/items/${queueItemId}`,
+        {
+          status: AnnotationQueueStatus.COMPLETED,
+        },
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+
+  describe("DELETE /annotation-queues/:queueId/items/:itemId", () => {
+    it("should delete an annotation queue item", async () => {
+      // Create a new item to delete
+      const createResponse = await makeZodVerifiedAPICall(
+        CreateAnnotationQueueItemResponse,
+        "POST",
+        `/api/public/annotation-queues/${queueId}/items`,
+        {
+          objectId: uuidv4(),
+          objectType: AnnotationQueueObjectType.TRACE,
+        },
+        auth,
+      );
+
+      const newItemId = createResponse.body.id;
+
+      const response = await makeZodVerifiedAPICall(
+        DeleteAnnotationQueueItemResponse,
+        "DELETE",
+        `/api/public/annotation-queues/${queueId}/items/${newItemId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.success).toBe(true);
+
+      // Verify the item is deleted
+      const getResponse = await makeAPICall(
+        "GET",
+        `/api/public/annotation-queues/${queueId}/items/${newItemId}`,
+        undefined,
+        auth,
+      );
+
+      expect(getResponse.status).toBe(404);
+    });
+
+    it("should return 404 for non-existent queue item", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "DELETE",
+        `/api/public/annotation-queues/${queueId}/items/${nonExistentId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+
+    it("should return 404 for non-existent queue", async () => {
+      const nonExistentId = uuidv4();
+      const response = await makeAPICall(
+        "DELETE",
+        `/api/public/annotation-queues/${nonExistentId}/items/${queueItemId}`,
+        undefined,
+        auth,
+      );
+
+      expect(response.status).toBe(404);
+    });
+  });
+});

web/src/__tests__/async/datasets-ui-table.servertest.ts

@@ -0,0 +1,85 @@
+import {
+  createOrgProjectAndApiKey,
+  getDatasetRunItemsTableCount,
+} from "@langfuse/shared/src/server";
+import { v4 as uuidv4 } from "uuid";
+import { prisma } from "@langfuse/shared/src/db";
+import { type FilterState } from "@langfuse/shared";
+
+const generateFilter = (datasetIds: string[]): FilterState => {
+  return [
+    {
+      column: "Dataset",
+      operator: "any of",
+      type: "stringOptions",
+      value: datasetIds,
+    },
+  ];
+};
+
+describe("trpc.datasets", () => {
+  let projectId: string;
+  let datasetIds: string[];
+
+  beforeAll(async () => {
+    const { projectId: newProjectId } = await createOrgProjectAndApiKey();
+    const datasetItemIds = [uuidv4(), uuidv4()];
+    const datasetRunIds = [uuidv4(), uuidv4()];
+    projectId = newProjectId;
+    datasetIds = [uuidv4(), uuidv4()];
+
+    await prisma.dataset.createMany({
+      data: datasetIds.map((datasetId, index) => ({
+        id: datasetId,
+        projectId: projectId,
+        name: `test-${index}`,
+      })),
+    });
+
+    await prisma.datasetItem.createMany({
+      data: datasetIds.map((datasetId, index) => ({
+        id: datasetItemIds[index],
+        projectId: projectId,
+        datasetId: datasetId,
+      })),
+    });
+
+    await prisma.datasetRuns.createMany({
+      data: datasetRunIds.map((datasetRunId, index) => ({
+        id: datasetRunId,
+        projectId: projectId,
+        datasetId: datasetIds[index],
+        name: `test-${index}`,
+      })),
+    });
+
+    await prisma.datasetRunItems.createMany({
+      data: datasetItemIds.map((datasetItemId, index) => ({
+        id: uuidv4(),
+        projectId: projectId,
+        datasetItemId: datasetItemId,
+        traceId: uuidv4(),
+        datasetRunId: datasetRunIds[index],
+      })),
+    });
+  });
+  describe("GET datasetItems.countAll", () => {
+    it("should GET all dataset run items with no filter", async () => {
+      const { totalCount } = await getDatasetRunItemsTableCount({
+        projectId: projectId,
+        filter: [],
+      });
+
+      expect(totalCount).toBe(2);
+    });
+
+    it("should GET all dataset run items with filter", async () => {
+      const { totalCount } = await getDatasetRunItemsTableCount({
+        projectId: projectId,
+        filter: generateFilter([datasetIds[0]]),
+      });
+
+      expect(totalCount).toBe(1);
+    });
+  });
+});

web/src/__tests__/async/evals-trpc.servertest.ts

@@ -1,21 +1,16 @@
 /** @jest-environment node */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 
-import type { Session } from "next-auth";
-import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import { prisma } from "@langfuse/shared/src/db";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import type { Session } from "next-auth";
 
-describe("evals trpc", () => {
-  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+const __orgIds: string[] = [];
 
-  beforeEach(async () => {
-    await pruneDatabase();
-    await prisma.jobExecution.deleteMany();
-    await prisma.jobConfiguration.deleteMany();
-    await prisma.evalTemplate.deleteMany();
-  });
+async function prepare() {
+  const { project, org } = await createOrgProjectAndApiKey();
 
   const session: Session = {
     expires: "1",
@@ -25,18 +20,18 @@ describe("evals trpc", () => {
       name: "Demo User",
       organizations: [
         {
-          id: "seed-org-id",
-          name: "Test Organization",
+          id: org.id,
+          name: org.name,
           role: "OWNER",
           plan: "cloud:hobby",
           cloudConfig: undefined,
           projects: [
             {
-              id: projectId,
+              id: project.id,
               role: "ADMIN",
               retentionDays: 30,
               deletedAt: null,
-              name: "Test Project",
+              name: project.name,
             },
           ],
         },
@@ -47,17 +42,36 @@ describe("evals trpc", () => {
       },
       admin: true,
     },
-    environment: {} as any,
+    environment: {
+      enableExperimentalFeatures: false,
+      selfHostedInstancePlan: "cloud:hobby",
+    },
   };
 
   const ctx = createInnerTRPCContext({ session });
   const caller = appRouter.createCaller({ ...ctx, prisma });
 
+  __orgIds.push(org.id);
+
+  return { project, org, session, ctx, caller };
+}
+
+describe("evals trpc", () => {
+  afterAll(async () => {
+    await prisma.organization.deleteMany({
+      where: {
+        id: { in: __orgIds },
+      },
+    });
+  });
+
   describe("evals.allConfigs", () => {
     it("should retrieve all evaluator configurations with execution status counts", async () => {
+      const { project, caller } = await prepare();
+
       const evalJobConfig1 = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -73,7 +87,7 @@ describe("evals trpc", () => {
         data: {
           jobConfigurationId: evalJobConfig1.id,
           status: "PENDING",
-          projectId,
+          projectId: project.id,
         },
       });
 
@@ -81,7 +95,7 @@ describe("evals trpc", () => {
         data: {
           jobConfigurationId: evalJobConfig1.id,
           status: "COMPLETED",
-          projectId,
+          projectId: project.id,
         },
       });
 
@@ -89,13 +103,13 @@ describe("evals trpc", () => {
         data: {
           jobConfigurationId: evalJobConfig1.id,
           status: "ERROR",
-          projectId,
+          projectId: project.id,
         },
       });
 
       const evalJobConfig2 = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -108,7 +122,7 @@ describe("evals trpc", () => {
       });
 
       const response = await caller.evals.allConfigs({
-        projectId,
+        projectId: project.id,
         limit: 10,
         page: 0,
       });
@@ -147,9 +161,11 @@ describe("evals trpc", () => {
 
   describe("evals.updateConfig", () => {
     it("should update an evaluator configuration", async () => {
+      const { project, caller } = await prepare();
+
       const evalJobConfig = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -163,7 +179,7 @@ describe("evals trpc", () => {
       });
 
       const response = await caller.evals.updateEvalJob({
-        projectId,
+        projectId: project.id,
         evalConfigId: evalJobConfig.id,
         config: {
           status: "INACTIVE",
@@ -187,9 +203,11 @@ describe("evals trpc", () => {
     });
 
     it("when the evaluator ran on existing traces, time scope cannot be changed to NEW only", async () => {
+      const { project, caller } = await prepare();
+
       const evalJobConfig = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -204,7 +222,7 @@ describe("evals trpc", () => {
 
       expect(
         caller.evals.updateEvalJob({
-          projectId,
+          projectId: project.id,
           evalConfigId: evalJobConfig.id,
           config: {
             timeScope: ["NEW"],
@@ -216,9 +234,11 @@ describe("evals trpc", () => {
     });
 
     it("when the evaluator ran on existing traces, it cannot be deactivated", async () => {
+      const { project, caller } = await prepare();
+
       const evalJobConfig = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -233,7 +253,7 @@ describe("evals trpc", () => {
 
       expect(
         caller.evals.updateEvalJob({
-          projectId,
+          projectId: project.id,
           evalConfigId: evalJobConfig.id,
           config: {
             status: "INACTIVE",
@@ -245,9 +265,11 @@ describe("evals trpc", () => {
     });
 
     it("when the evaluator ran on existing traces, it can be deactivated if it should also run on new traces", async () => {
+      const { project, caller } = await prepare();
+
       const evalJobConfig = await prisma.jobConfiguration.create({
         data: {
-          projectId,
+          projectId: project.id,
           jobType: "EVAL",
           scoreName: "test-score",
           filter: [],
@@ -261,7 +283,7 @@ describe("evals trpc", () => {
       });
 
       const response = await caller.evals.updateEvalJob({
-        projectId,
+        projectId: project.id,
         evalConfigId: evalJobConfig.id,
         config: {
           status: "INACTIVE",
@@ -284,4 +306,319 @@ describe("evals trpc", () => {
       expect(updatedJob?.timeScope).toEqual(["EXISTING", "NEW"]);
     });
   });
+
+  describe("evals.deleteEvalJob", () => {
+    it("should successfully delete an eval job", async () => {
+      const { project, caller } = await prepare();
+
+      // Create a job to delete
+      const evalJobConfig = await prisma.jobConfiguration.create({
+        data: {
+          projectId: project.id,
+          jobType: "EVAL",
+          scoreName: "test-score",
+          filter: [],
+          targetObject: "trace",
+          variableMapping: [],
+          sampling: 1,
+          delay: 0,
+          status: "ACTIVE",
+          timeScope: ["NEW"],
+        },
+      });
+
+      // Create multiple job executions with different statuses
+      await Promise.all([
+        prisma.jobExecution.create({
+          data: {
+            jobConfigurationId: evalJobConfig.id,
+            status: "COMPLETED",
+            projectId: project.id,
+          },
+        }),
+        prisma.jobExecution.create({
+          data: {
+            jobConfigurationId: evalJobConfig.id,
+            status: "PENDING",
+            projectId: project.id,
+          },
+        }),
+        prisma.jobExecution.create({
+          data: {
+            jobConfigurationId: evalJobConfig.id,
+            status: "ERROR",
+            projectId: project.id,
+            error: "Test error",
+          },
+        }),
+      ]);
+
+      // Verify job executions exist before deletion
+      const beforeJobExecutions = await prisma.jobExecution.findMany({
+        where: {
+          jobConfigurationId: evalJobConfig.id,
+        },
+      });
+      expect(beforeJobExecutions).toHaveLength(3);
+
+      // Delete the job
+      await caller.evals.deleteEvalJob({
+        projectId: project.id,
+        evalConfigId: evalJobConfig.id,
+      });
+
+      // Verify job is deleted
+      const deletedJob = await prisma.jobConfiguration.findUnique({
+        where: {
+          id: evalJobConfig.id,
+        },
+      });
+      expect(deletedJob).toBeNull();
+
+      // Verify all job executions are deleted (cascade)
+      const afterJobExecutions = await prisma.jobExecution.findMany({
+        where: {
+          jobConfigurationId: evalJobConfig.id,
+        },
+      });
+      expect(afterJobExecutions).toHaveLength(0);
+    });
+
+    it("should throw error when trying to delete non-existent eval job", async () => {
+      const { project, caller } = await prepare();
+
+      await expect(
+        caller.evals.deleteEvalJob({
+          projectId: project.id,
+          evalConfigId: "non-existent-id",
+        }),
+      ).rejects.toThrow("Job not found");
+    });
+
+    it("should throw error when user lacks evalJob:CUD access scope", async () => {
+      const { project, session } = await prepare();
+
+      // Create a session with limited permissions
+      const limitedSession: Session = {
+        ...session,
+        user: {
+          id: session.user!.id,
+          name: session.user!.name,
+          canCreateOrganizations: session.user!.canCreateOrganizations,
+          admin: false,
+          featureFlags: session.user!.featureFlags,
+          organizations: [
+            {
+              ...session.user!.organizations[0],
+              role: "MEMBER",
+              projects: [
+                {
+                  ...session.user!.organizations[0].projects[0],
+                  role: "VIEWER", // VIEWER role doesn't have evalTemplate:CUD scope
+                },
+              ],
+            },
+          ],
+        },
+        expires: session.expires,
+        environment: session.environment,
+      };
+      const limitedCtx = createInnerTRPCContext({ session: limitedSession });
+      const limitedCaller = appRouter.createCaller({ ...limitedCtx, prisma });
+
+      // Create a job
+      const evalJobConfig = await prisma.jobConfiguration.create({
+        data: {
+          projectId: project.id,
+          jobType: "EVAL",
+          scoreName: "test-score",
+          filter: [],
+          targetObject: "trace",
+          variableMapping: [],
+          sampling: 1,
+          delay: 0,
+          status: "ACTIVE",
+          timeScope: ["NEW"],
+        },
+      });
+
+      // Attempt to delete with limited permissions
+      await expect(
+        limitedCaller.evals.deleteEvalJob({
+          projectId: project.id,
+          evalConfigId: evalJobConfig.id,
+        }),
+      ).rejects.toThrow("User does not have access to this resource or action");
+    });
+  });
+
+  // TODO: moved to LFE-4573
+  // describe("evals.deleteEvalTemplate", () => {
+  //   it("should successfully delete an eval template", async () => {
+  //     const { project, caller } = await prepare();
+
+  //     // Create a template to delete
+  //     const evalTemplate = await prisma.evalTemplate.create({
+  //       data: {
+  //         projectId: project.id,
+  //         name: "test-template",
+  //         version: 1,
+  //         prompt: "test prompt",
+  //         model: "test-model",
+  //         modelParams: {},
+  //         vars: [],
+  //         outputSchema: {
+  //           score: "test-score",
+  //           reasoning: "test-reasoning",
+  //         },
+  //         provider: "test-provider",
+  //       },
+  //     });
+
+  //     // Delete the template
+  //     await caller.evals.deleteEvalTemplate({
+  //       projectId: project.id,
+  //       evalTemplateId: evalTemplate.id,
+  //     });
+
+  //     // Verify template is deleted
+  //     const deletedTemplate = await prisma.evalTemplate.findUnique({
+  //       where: {
+  //         id: evalTemplate.id,
+  //       },
+  //     });
+  //     expect(deletedTemplate).toBeNull();
+  //   });
+
+  //   it("should set evalTemplateId to null for associated eval jobs when template is deleted", async () => {
+  //     const { project, caller } = await prepare();
+
+  //     // Create a template
+  //     const evalTemplate = await prisma.evalTemplate.create({
+  //       data: {
+  //         projectId: project.id,
+  //         name: "test-template",
+  //         version: 1,
+  //         prompt: "test prompt",
+  //         model: "test-model",
+  //         modelParams: {},
+  //         vars: [],
+  //         outputSchema: {
+  //           score: "test-score",
+  //           reasoning: "test-reasoning",
+  //         },
+  //         provider: "test-provider",
+  //       },
+  //     });
+
+  //     // Create an eval job linked to this template
+  //     const evalJob = await prisma.jobConfiguration.create({
+  //       data: {
+  //         projectId: project.id,
+  //         jobType: "EVAL",
+  //         scoreName: "test-score",
+  //         filter: [],
+  //         targetObject: "trace",
+  //         variableMapping: [],
+  //         sampling: 1,
+  //         delay: 0,
+  //         status: "ACTIVE",
+  //         timeScope: ["NEW"],
+  //         evalTemplateId: evalTemplate.id,
+  //       },
+  //     });
+
+  //     // Delete the template
+  //     await caller.evals.deleteEvalTemplate({
+  //       projectId: project.id,
+  //       evalTemplateId: evalTemplate.id,
+  //     });
+
+  //     // Verify template is deleted
+  //     const deletedTemplate = await prisma.evalTemplate.findUnique({
+  //       where: {
+  //         id: evalTemplate.id,
+  //       },
+  //     });
+  //     expect(deletedTemplate).toBeNull();
+
+  //     // Verify eval job still exists but has evalTemplateId set to null
+  //     const updatedJob = await prisma.jobConfiguration.findUnique({
+  //       where: {
+  //         id: evalJob.id,
+  //       },
+  //     });
+  //     expect(updatedJob).not.toBeNull();
+  //     expect(updatedJob?.evalTemplateId).toBeNull();
+  //   });
+
+  //   it("should throw error when trying to delete non-existent eval template", async () => {
+  //     const { project, caller } = await prepare();
+
+  //     await expect(
+  //       caller.evals.deleteEvalTemplate({
+  //         projectId: project.id,
+  //         evalTemplateId: "non-existent-id",
+  //       }),
+  //     ).rejects.toThrow("Template not found");
+  //   });
+
+  //   it("should throw error when user lacks evalTemplate:CUD access scope", async () => {
+  //     const { project, session } = await prepare();
+
+  //     // Create a session with limited permissions
+  //     const limitedSession: Session = {
+  //       ...session,
+  //       user: {
+  //         id: session.user!.id,
+  //         name: session.user!.name,
+  //         canCreateOrganizations: session.user!.canCreateOrganizations,
+  //         admin: false,
+  //         featureFlags: session.user!.featureFlags,
+  //         organizations: [
+  //           {
+  //             ...session.user!.organizations[0],
+  //             role: "MEMBER",
+  //             projects: [
+  //               {
+  //                 ...session.user!.organizations[0].projects[0],
+  //                 role: "VIEWER", // VIEWER role doesn't have evalTemplate:CUD scope
+  //               },
+  //             ],
+  //           },
+  //         ],
+  //       },
+  //       expires: session.expires,
+  //       environment: session.environment,
+  //     };
+  //     const limitedCtx = createInnerTRPCContext({ session: limitedSession });
+  //     const limitedCaller = appRouter.createCaller({ ...limitedCtx, prisma });
+
+  //     // Create a template
+  //     const evalTemplate = await prisma.evalTemplate.create({
+  //       data: {
+  //         projectId: project.id,
+  //         name: "test-template",
+  //         version: 1,
+  //         prompt: "test prompt",
+  //         model: "test-model",
+  //         modelParams: {},
+  //         vars: [],
+  //         outputSchema: {
+  //           score: "test-score",
+  //           reasoning: "test-reasoning",
+  //         },
+  //         provider: "test-provider",
+  //       },
+  //     });
+
+  //     // Attempt to delete with limited permissions
+  //     await expect(
+  //       limitedCaller.evals.deleteEvalTemplate({
+  //         projectId: project.id,
+  //         evalTemplateId: evalTemplate.id,
+  //       }),
+  //     ).rejects.toThrow("User does not have access to this resource or action");
+  //   });
+  // });
 });

web/src/__tests__/async/ingestion-api.servertest.ts

@@ -69,6 +69,21 @@ describe("/api/public/ingestion API Endpoint", () => {
         },
       },
     ],
+    [
+      "emojis in i/o",
+      {
+        id: randomUUID(),
+        type: "trace-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          timestamp: new Date().toISOString(),
+          metadata: { hello: "world" },
+          input: "🪢🚀🌖",
+          output: "👋🌍",
+        },
+      },
+    ],
   ])(
     "should create traces via the ingestion API (%s)",
     async (_name: string, entity: any) => {
@@ -93,6 +108,33 @@ describe("/api/public/ingestion API Endpoint", () => {
     },
   );
 
+  it("should replace bad escape sequences on clickhouse", async () => {
+    const entity = {
+      id: randomUUID(),
+      type: "trace-create",
+      timestamp: new Date().toISOString(),
+      body: {
+        id: randomUUID(),
+        timestamp: new Date().toISOString(),
+        metadata: { hello: "world" },
+        input: "test\\ud8000test",
+        environment: "production",
+      },
+    };
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [entity],
+    });
+
+    expect(response.status).toBe(207);
+    await waitForExpect(async () => {
+      const trace = await getTraceById(entity.body.id, projectId);
+      expect(trace).toBeDefined();
+      expect(trace!.id).toBe(entity.body.id);
+      expect(trace!.projectId).toBe(projectId);
+      expect(trace!.input).toContain("test");
+    });
+  });
+
   it.each([
     [
       "generation",
@@ -242,6 +284,89 @@ describe("/api/public/ingestion API Endpoint", () => {
     },
   );
 
+  it.each([
+    "&",
+    "$",
+    "@",
+    "=",
+    ";",
+    "/",
+    "+",
+    " ",
+    ",",
+    "?",
+    // "\\",
+    "{",
+    "}",
+    "^",
+    "%",
+    "`",
+    "]",
+    '"',
+    ">",
+    "[",
+    "~",
+    "<",
+    "#",
+    "|",
+  ])("should test special S3 characters in IDs (%s)", async (char: string) => {
+    const traceId = randomUUID();
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: randomUUID(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: `${traceId}-${char}-test`,
+            timestamp: new Date().toISOString(),
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+
+    await waitForExpect(async () => {
+      const trace = await getTraceById(`${traceId}-${char}-test`, projectId);
+      expect(trace).toBeDefined();
+      expect(trace!.id).toBe(`${traceId}-${char}-test`);
+      expect(trace!.projectId).toBe(projectId);
+      expect(trace!.environment).toEqual("default");
+    });
+  });
+
+  it("should fail for \\r in id", async () => {
+    const traceId = v4();
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: `${v4()}-\r-test`,
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+    expect("errors" in response.body).toBe(true);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(response.body.errors.length).toBe(1);
+    expect(response.body.errors[0].message).toBe("Invalid request data");
+    expect(response.body.errors[0].error).toContain(
+      "ID cannot contain carriage return characters",
+    );
+  });
+
   it("should fail for long trace name", async () => {
     const traceId = v4();
 
@@ -275,6 +400,36 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(response.body.errors[0].message).toBe("Invalid request data");
   });
 
+  it("should fail for float in usageDetails", async () => {
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "generation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: randomUUID(),
+            traceId: randomUUID(),
+            parentObservationId: randomUUID(),
+            startTime: new Date().toISOString(),
+            model: "gpt-4",
+            input: { text: "input" },
+            output: { text: "output" },
+            usageDetails: {
+              key: 0.1,
+            },
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+    expect("errors" in response.body).toBe(true);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(response.body.errors.length).toBe(1);
+    expect(response.body.errors[0].message).toBe("Invalid request data");
+  });
+
   it.each([
     "langfuse-test",
     ".invalidcharacter!",
@@ -342,6 +497,88 @@ describe("/api/public/ingestion API Endpoint", () => {
     });
   });
 
+  it.each([
+    ["string", { testId: "this is a string metadata" }],
+    ["big-number", { testId: "1983516295378495150" }],
+    ["small-number", { testId: 5 }],
+    ["float-number", { testId: 5.5 }],
+  ])(
+    "#6123: should treat %s metadata for traces as such",
+    async (_type, metadataValue) => {
+      const traceId = randomUUID();
+
+      const entity = {
+        id: randomUUID(),
+        type: "trace-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: traceId,
+          timestamp: new Date().toISOString(),
+          metadata: metadataValue,
+        },
+      };
+
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [entity],
+      });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const trace = await getTraceById(traceId, projectId);
+        expect(trace).toBeDefined();
+        expect(trace!.id).toBe(traceId);
+        expect(JSON.stringify(trace!.metadata)).toBe(
+          JSON.stringify(metadataValue),
+        );
+      });
+    },
+  );
+
+  it.each([
+    ["string", { testId: "this is a string metadata" }],
+    ["big-number", { testId: "1983516295378495150" }],
+    ["small-number", { testId: 5 }],
+    ["float-number", { testId: 5.5 }],
+  ])(
+    "#6123: should treat %s metadata for observations as such",
+    async (_type, metadataValue) => {
+      const observationId = randomUUID();
+      const traceId = randomUUID();
+
+      const entity = {
+        id: randomUUID(),
+        type: "span-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: observationId,
+          traceId: traceId,
+          startTime: new Date().toISOString(),
+          metadata: metadataValue,
+        },
+      };
+
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [entity],
+      });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const observation = await getObservationById(
+          observationId,
+          projectId,
+          true,
+        );
+        expect(observation).toBeDefined();
+        expect(observation!.id).toBe(observationId);
+        expect(JSON.stringify(observation!.metadata)).toBe(
+          JSON.stringify(metadataValue),
+        );
+      });
+    },
+  );
+
   it("#4900: should clear score comment on update with `null`", async () => {
     const scoreId = randomUUID();
     const score1 = {

web/src/__tests__/async/repositories/dashboard-repository.servertest.ts

@@ -1,5 +1,12 @@
-import { prepareUsageDataForTimeseriesChart } from "@/src/features/dashboard/components/ModelUsageChart";
-import { orderByTimeSeries } from "@langfuse/shared/src/server";
+import {
+  orderByTimeSeries,
+  getObservationUsageByTypeByTime,
+  createOrgProjectAndApiKey,
+  createTrace,
+  createTracesCh,
+  createObservationsCh,
+  createObservation,
+} from "@langfuse/shared/src/server";
 
 describe("orderByTimeSeries", () => {
   it("should return correct bucket size and query for 1 hour time range", () => {
@@ -89,55 +96,102 @@ describe("orderByTimeSeries", () => {
     );
   });
 
-  describe("aggregate time series for model cost and usage", () => {
-    it("should aggregate time series for model cost and usage", async () => {
-      const metricHistory = prepareUsageDataForTimeseriesChart(
-        ["gpt-4o-mini", "text-embedding-ada-002"],
-        [
-          {
-            startTime: "2025-02-10T13:30:00.000Z",
-            units: {
-              input: 422,
-              output: 61,
-              total: 483,
-            },
-            cost: {
-              input: 0.0000633,
-              output: 0.0000366,
-              total: 0.0000999,
-            },
-            model: "gpt-4o-mini",
-          },
-          {
-            startTime: "2025-02-10T13:30:00.000Z",
-            units: {
-              input: 6,
-              total: 6,
-            },
-            cost: {
-              total: 6e-7,
-            },
-            model: "text-embedding-ada-002",
-          },
-        ],
+  describe("getObservationUsageByTypeByTime", () => {
+    const mockFilter = [
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: ">=" as const,
+        value: new Date("2024-01-01T00:00:00Z"),
+      },
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: "<=" as const,
+        value: new Date("2024-01-02T01:00:00Z"),
+      },
+    ];
+
+    it("should return usage data grouped by time and type", async () => {
+      const { projectId } = await createOrgProjectAndApiKey();
+
+      const trace = createTrace({
+        name: "trace-name",
+        project_id: projectId,
+        timestamp: new Date("2024-01-01T01:00:00Z").getTime(),
+      });
+
+      const trace2 = createTrace({
+        name: "trace-name",
+        project_id: projectId,
+        timestamp: new Date("2024-01-01T04:00:00Z").getTime(),
+      });
+
+      await createTracesCh([trace, trace2]);
+
+      const obs1 = createObservation({
+        trace_id: trace.id,
+        project_id: trace.project_id,
+        usage_details: { input: 1, output: 2, total: 3 },
+        start_time: new Date("2024-01-01T01:00:00Z").getTime(),
+      });
+
+      const obs2 = createObservation({
+        trace_id: trace.id,
+        project_id: trace.project_id,
+        usage_details: { input: 4, output: 5, total: 9 },
+        start_time: new Date("2024-01-01T01:00:00Z").getTime(),
+      });
+
+      const obs3 = createObservation({
+        trace_id: trace2.id,
+        project_id: trace.project_id,
+        usage_details: { input: 400, output: 500, total: 900 },
+        start_time: new Date("2024-01-01T04:00:00Z").getTime(),
+      });
+
+      await createObservationsCh([obs1, obs2, obs3]);
+
+      const result = await getObservationUsageByTypeByTime(
+        projectId,
+        mockFilter,
       );
 
-      expect(metricHistory.get("total")).toEqual([
-        {
-          startTime: "2025-02-10T13:30:00.000Z",
-          units: 483,
-          cost: 0.0000999,
-          model: "gpt-4o-mini",
-          usageType: "total",
-        },
-        {
-          startTime: "2025-02-10T13:30:00.000Z",
-          units: 6,
-          cost: 6e-7,
-          model: "text-embedding-ada-002",
-          usageType: "total",
-        },
-      ]);
+      // Verify the structure of the returned data
+      expect(result).toEqual(
+        expect.arrayContaining([
+          {
+            intervalStart: new Date("2024-01-01T01:00:00Z"),
+            key: "input",
+            sum: 5,
+          },
+          {
+            intervalStart: new Date("2024-01-01T01:00:00Z"),
+            key: "output",
+            sum: 7,
+          },
+          {
+            intervalStart: new Date("2024-01-01T01:00:00Z"),
+            key: "total",
+            sum: 12,
+          },
+          {
+            intervalStart: new Date("2024-01-01T04:00:00Z"),
+            key: "input",
+            sum: 400,
+          },
+          {
+            intervalStart: new Date("2024-01-01T04:00:00Z"),
+            key: "output",
+            sum: 500,
+          },
+          {
+            intervalStart: new Date("2024-01-01T04:00:00Z"),
+            key: "total",
+            sum: 900,
+          },
+        ]),
+      );
     });
   });
 });

web/src/__tests__/async/repositories/environment-repository.servertest.ts

@@ -0,0 +1,46 @@
+import {
+  createTracesCh,
+  createTrace,
+  getEnvironmentsForProject,
+} from "@langfuse/shared/src/server";
+import { randomUUID } from "crypto";
+
+describe("Clickhouse Project Repository Test", () => {
+  it("should return default if no environments are found", async () => {
+    const projectId = randomUUID();
+    const environments = await getEnvironmentsForProject({ projectId });
+    expect(environments).toHaveLength(1);
+    expect(environments[0].environment).toEqual("default");
+  });
+
+  it("should return environment from project_environments table after new trace was inserted", async () => {
+    const projectId = randomUUID();
+    const environmentId1 = randomUUID();
+    const environmentId2 = randomUUID();
+    await createTracesCh([
+      createTrace({
+        project_id: projectId,
+        environment: environmentId1,
+      }),
+      createTrace({
+        project_id: projectId,
+        environment: environmentId1,
+      }),
+      createTrace({
+        project_id: projectId,
+        environment: environmentId2,
+      }),
+    ]);
+
+    const environments = await getEnvironmentsForProject({ projectId });
+
+    expect(environments).toHaveLength(3);
+    expect(environments).toEqual(
+      expect.arrayContaining([
+        { environment: environmentId1 },
+        { environment: environmentId2 },
+        { environment: "default" },
+      ]),
+    );
+  });
+});