chore: release v2.85.0

fix

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx
+ignore-words-list = afterall,vertx,notIn
 

.env.dev.example

@@ -34,11 +34,25 @@ EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
 # S3 storage
-S3_ENDPOINT=
-S3_ACCESS_KEY_ID=
-S3_SECRET_ACCESS_KEY=
-S3_BUCKET_NAME=
-S3_REGION=
+S3_ENDPOINT=http://localhost:9090
+S3_ACCESS_KEY_ID=minio
+S3_SECRET_ACCESS_KEY=miniosecret
+S3_BUCKET_NAME=mybucket
+S3_REGION=us-east-1
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=false
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=mybucket
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 # Set during docker build of application
 # Used to disable environment verification at build time
@@ -53,4 +67,4 @@ LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.prod.example

@@ -67,6 +67,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_CLIENT_ID=
+# AUTH_GITLAB_CLIENT_SECRET=
+# AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_ISSUER=
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=

.github/workflows/ci.yml.template

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start containers
         run: docker compose -f "docker-compose.yml" up -d --build
@@ -31,7 +31,7 @@ jobs:
         uses: pnpm/action-setup@v2.2.4
 
       - name: Setup Node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main", "production", "v*"]
+  pull_request:
+    branches: ["main", "production", "v*"]
+  schedule:
+    - cron: "31 13 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: javascript-typescript
+            build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # If the analyze step fails for one of the languages you are analyzing with
+      # "We were unable to automatically build your code", modify the matrix above
+      # to set the build mode to "manual" for that language. Then modify this step
+      # to build your code.
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      - if: matrix.build-mode == 'manual'
+        shell: bash
+        run: |
+          echo 'If you are using a "manual" build mode for one or more of the' \
+            'languages you are analyzing, replace this with the commands to build' \
+            'your code, for example:'
+          echo '  make bootstrap'
+          echo '  make release'
+          exit 1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/licencecheck.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/pipeline.yml

@@ -16,11 +16,11 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -35,10 +35,11 @@ jobs:
         run: pnpm run lint
 
   test-docker-build:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set NEXT_PUBLIC_BUILD_ID
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
@@ -48,14 +49,6 @@ jobs:
           docker compose -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Check server health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
-
-      - name: Check worker health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
-
       - name: Ensure no unhealthy status
         run: |
           if docker-compose ps | grep "(unhealthy)"; then
@@ -65,6 +58,14 @@ jobs:
             echo "All services are healthy"
           fi
 
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
   tests-web:
     timeout-minutes: 20
     runs-on: ubuntu-latest
@@ -79,12 +80,12 @@ jobs:
         with:
           swap-size-gb: 10
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -97,7 +98,7 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
 
       - name: Run + migrate
         run: |
@@ -144,13 +145,13 @@ jobs:
         with:
           swap-size-gb: 10
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -201,11 +202,11 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -243,11 +244,11 @@ jobs:
   e2e-server-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -321,13 +322,13 @@ jobs:
         with:
           version: 9.5.0
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache-dependency-path: "pnpm-lock.yaml"
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set NEXT_PUBLIC_BUILD_ID
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV

docker-compose.dev.yml

@@ -17,6 +17,38 @@ services:
     depends_on:
       - postgres
 
+  minio:
+    image: minio/minio
+    container_name: minio
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 5s
+
+  miniocreatebucket:
+    image: minio/mc
+    container_name: miniocreatebucket
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mc alias set minio http://minio:9000 minio miniosecret && 
+       mc rm -r --force minio/mybucket || true &&
+       mc mb minio/mybucket && 
+       mc policy set download minio/mybucket"
+    depends_on:
+      minio:
+        condition: service_healthy
+
   redis:
     image: redis:7.2.4
     restart: always
@@ -45,3 +77,5 @@ volumes:
     driver: local
   langfuse_clickhouse_logs:
     driver: local
+  langfuse_minio_data:
+    driver: local

ee/package.json

@@ -25,8 +25,9 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "next": "^14.2.6",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "axios": "^1.7.7",
+    "next": "^14.2.15",
     "next-auth": "^4.24.7",
     "zod": "^3.23.8"
   },
@@ -39,7 +40,7 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"

ee/src/env.ts

@@ -1,9 +1,12 @@
 import { z } from "zod";
-import { env as sharedEnv } from "@langfuse/shared";
+import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };
+export const env = {
+  ...sharedEnv,
+  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.82.0",
+  "version": "2.85.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -33,7 +33,7 @@
     "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
     "husky": "^9.0.11",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "release-it": "^17.3.0",
     "turbo": "^1.13.4"
   },

packages/config-eslint/package.json

@@ -11,7 +11,7 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.6",
+    "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",

packages/shared/package.json

@@ -55,20 +55,22 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.627.0",
-    "@aws-sdk/lib-storage": "^3.645.0",
+    "@aws-sdk/lib-storage": "^3.667.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@clickhouse/client": "^1.4.0",
     "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.3",
+    "@langchain/aws": "^0.1.0",
+    "@langchain/core": "^0.3.9",
     "@langchain/openai": "^0.3.0",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "@prisma/client": "^5.17.0",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "@prisma/client": "^5.20.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
+    "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.12.10",
-    "dd-trace": "^5.19.0",
+    "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
     "ioredis": "^5.4.1",
@@ -76,37 +78,36 @@
     "langchain": "^0.3.2",
     "lodash": "^4.17.21",
     "next-auth": "^4.24.7",
-    "nodemailer": "^6.9.13",
+    "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
     "winston": "^3.14.2",
-    "winston-transport": "^4.7.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.2"
   },
   "devDependencies": {
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
     "@types/node": "^20.11.29",
-    "@types/nodemailer": "^6.4.15",
-    "@types/pg": "^8.11.6",
+    "@types/nodemailer": "^6.4.16",
+    "@types/pg": "^8.11.10",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.1.3",
-    "prettier": "^3.2.5",
-    "prisma": "^5.17.0",
+    "kysely-codegen": "^0.16.8",
+    "nodemon": "^3.1.7",
+    "prettier": "^3.3.3",
+    "prisma": "^5.20.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.3"
+    "vitest": "^2.1.2"
   },
   "peerDependencies": {
     "@types/react": "~18.2.79",

packages/shared/prisma/generated/types.ts

@@ -37,6 +37,16 @@ export const ScoreDataType = {
     BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
+export const AnnotationQueueStatus = {
+    PENDING: "PENDING",
+    COMPLETED: "COMPLETED"
+} as const;
+export type AnnotationQueueStatus = (typeof AnnotationQueueStatus)[keyof typeof AnnotationQueueStatus];
+export const AnnotationQueueObjectType = {
+    TRACE: "TRACE",
+    OBSERVATION: "OBSERVATION"
+} as const;
+export type AnnotationQueueObjectType = (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -81,6 +91,30 @@ export type Account = {
     id_token: string | null;
     session_state: string | null;
     refresh_token_expires_in: number | null;
+    created_at: number | null;
+};
+export type AnnotationQueue = {
+    id: string;
+    name: string;
+    description: string | null;
+    score_config_ids: Generated<string[]>;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type AnnotationQueueItem = {
+    id: string;
+    queue_id: string;
+    object_id: string;
+    object_type: AnnotationQueueObjectType;
+    status: Generated<AnnotationQueueStatus>;
+    locked_at: Timestamp | null;
+    locked_by_user_id: string | null;
+    annotator_user_id: string | null;
+    completed_at: Timestamp | null;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type ApiKey = {
     id: string;
@@ -245,6 +279,7 @@ export type LlmApiKeys = {
     base_url: string | null;
     custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
+    config: unknown | null;
     project_id: string;
 };
 export type MembershipInvitation = {
@@ -410,6 +445,7 @@ export type Score = {
     observation_id: string | null;
     config_id: string | null;
     string_value: string | null;
+    queue_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     data_type: Generated<ScoreDataType>;
@@ -506,6 +542,8 @@ export type VerificationToken = {
 };
 export type DB = {
     Account: Account;
+    annotation_queue_items: AnnotationQueueItem;
+    annotation_queues: AnnotationQueue;
     api_keys: ApiKey;
     audit_logs: AuditLog;
     batch_exports: BatchExport;

packages/shared/prisma/migrations/20241009042557_auth_add_created_at_for_gitlab/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "created_at" INTEGER;

packages/shared/prisma/migrations/20241009110720_scores_add_nullable_queue_id_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "queue_id" TEXT;

packages/shared/prisma/migrations/20241009113245_add_annotation_queue/migration.sql

@@ -0,0 +1,75 @@
+-- CreateEnum
+CREATE TYPE "AnnotationQueueStatus" AS ENUM ('PENDING', 'COMPLETED');
+
+-- CreateEnum
+CREATE TYPE "AnnotationQueueObjectType" AS ENUM ('TRACE', 'OBSERVATION');
+
+-- CreateTable
+CREATE TABLE "annotation_queues" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT,
+    "score_config_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queues_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "annotation_queue_items" (
+    "id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "object_type" "AnnotationQueueObjectType" NOT NULL,
+    "status" "AnnotationQueueStatus" NOT NULL DEFAULT 'PENDING',
+    "locked_at" TIMESTAMP(3),
+    "locked_by_user_id" TEXT,
+    "annotator_user_id" TEXT,
+    "completed_at" TIMESTAMP(3),
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_items_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_id_project_id_idx" ON "annotation_queues"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_project_id_created_at_idx" ON "annotation_queues"("project_id", "created_at");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queues_project_id_name_key" ON "annotation_queues"("project_id", "name");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_id_project_id_idx" ON "annotation_queue_items"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_project_id_queue_id_status_idx" ON "annotation_queue_items"("project_id", "queue_id", "status");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_object_id_object_type_project_id_que_idx" ON "annotation_queue_items"("object_id", "object_type", "project_id", "queue_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_annotator_user_id_idx" ON "annotation_queue_items"("annotator_user_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_created_at_idx" ON "annotation_queue_items"("created_at");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queues" ADD CONSTRAINT "annotation_queues_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_locked_by_user_id_fkey" FOREIGN KEY ("locked_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_annotator_user_id_fkey" FOREIGN KEY ("annotator_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241010120245_llm_keys_add_config/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "llm_api_keys" ADD COLUMN "config" JSONB;

packages/shared/prisma/seed.ts

@@ -5,6 +5,7 @@ import {
   ObservationType,
   ScoreSource,
   ScoreDataType,
+  AnnotationQueueObjectType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -77,6 +78,9 @@ async function main() {
     create: {
       id: seedOrgId,
       name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
     },
   });
 
@@ -250,6 +254,11 @@ async function main() {
       project2,
     ]);
 
+    const queueIds = await generateQueuesForProject(
+      [project1, project2],
+      configIdsAndNames
+    );
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -257,19 +266,27 @@ async function main() {
 
     const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;
 
-    const { traces, observations, scores, sessions, events, comments } =
-      createObjects(
-        traceVolume,
-        envTags,
-        colorTags,
-        project1,
-        project2,
-        promptIds,
-        configIdsAndNames,
-      );
+    const {
+      traces,
+      observations,
+      scores,
+      sessions,
+      events,
+      comments,
+      queueItems,
+    } = createObjects(
+      traceVolume,
+      envTags,
+      colorTags,
+      project1,
+      project2,
+      promptIds,
+      queueIds,
+      configIdsAndNames
+    );
 
     logger.info(
-      `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`,
+      `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
     );
 
     await uploadObjects(
@@ -279,6 +296,7 @@ async function main() {
       sessions,
       events,
       comments,
+      queueItems
     );
 
     // If openai key is in environment, add it to the projects LLM API keys
@@ -296,7 +314,7 @@ async function main() {
       });
     } else {
       logger.warn(
-        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key.",
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
       );
     }
 
@@ -431,7 +449,7 @@ async function main() {
 
         for (const datasetItemId of datasetItemIds) {
           const relevantObservations = observations.filter(
-            (o) => o.projectId === project2.id,
+            (o) => o.projectId === project2.id
           );
           const observation =
             relevantObservations[
@@ -474,6 +492,7 @@ async function uploadObjects(
   sessions: Prisma.TraceSessionCreateManyInput[],
   events: Prisma.ObservationCreateManyInput[],
   comments: Prisma.CommentCreateManyInput[],
+  queueItems: Prisma.AnnotationQueueItemCreateManyInput[]
 ) {
   let promises: Prisma.PrismaPromise<unknown>[] = [];
 
@@ -487,14 +506,14 @@ async function uploadObjects(
         },
         create: chunk[0]!,
         update: {},
-      }),
+      })
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -505,13 +524,13 @@ async function uploadObjects(
     promises.push(
       prisma.trace.createMany({
         data: chunk,
-      }),
+      })
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -521,14 +540,14 @@ async function uploadObjects(
     promises.push(
       prisma.observation.createMany({
         data: chunk,
-      }),
+      })
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -538,14 +557,14 @@ async function uploadObjects(
     promises.push(
       prisma.observation.createMany({
         data: chunk,
-      }),
+      })
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -555,13 +574,13 @@ async function uploadObjects(
     promises.push(
       prisma.score.createMany({
         data: chunk,
-      }),
+      })
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -571,13 +590,29 @@ async function uploadObjects(
     promises.push(
       prisma.comment.createMany({
         data: chunk,
-      }),
+      })
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`,
+        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`
+      );
+    await promises[i];
+  }
+
+  promises = [];
+  chunk(queueItems, chunkSize).forEach((chunk) => {
+    promises.push(
+      prisma.annotationQueueItem.createMany({
+        data: chunk,
+      })
+    );
+  });
+  for (let i = 0; i < promises.length; i++) {
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      logger.info(
+        `Seeding of Annotation Queue Items ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
@@ -590,6 +625,7 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
+  queueIds: Map<string, string[]>,
   configParams: Map<
     string,
     {
@@ -598,7 +634,7 @@ function createObjects(
       dataType: ScoreDataType;
       categories: ConfigCategory[] | null;
     }[]
-  >,
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -607,12 +643,13 @@ function createObjects(
   const events: Prisma.ObservationCreateManyInput[] = [];
   const configs: Prisma.ScoreConfigCreateManyInput[] = [];
   const comments: Prisma.CommentCreateManyInput[] = [];
+  const queueItems: Prisma.AnnotationQueueItemCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
     // random date within last 90 days, with a linear bias towards more recent dates
     const traceTs = new Date(
-      Date.now() - Math.floor(Math.random() ** 1.5 * 90 * 24 * 60 * 60 * 1000),
+      Date.now() - Math.floor(Math.random() ** 1.5 * 90 * 24 * 60 * 60 * 1000)
     );
 
     const envTag = envTags[Math.floor(Math.random() * envTags.length)];
@@ -689,6 +726,21 @@ function createObjects(
       }),
     };
 
+    const queueItem = [
+      ...(Math.random() > 0.9 && queueIds.get(projectId)?.[0]
+        ? [
+            {
+              queueId: queueIds.get(projectId)?.[0] as string,
+              objectId: trace.id,
+              objectType: AnnotationQueueObjectType.TRACE,
+              projectId,
+            },
+          ]
+        : []),
+    ];
+
+    queueItems.push(...queueItem);
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
@@ -753,11 +805,11 @@ function createObjects(
     for (let j = 0; j < Math.floor(Math.random() * 10) + 1; j++) {
       // add between 1 and 30 ms to trace timestamp
       const spanTsStart = new Date(
-        traceTs.getTime() + Math.floor(Math.random() * 30),
+        traceTs.getTime() + Math.floor(Math.random() * 30)
       );
       // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 5000),
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
       );
 
       const span = {
@@ -792,22 +844,22 @@ function createObjects(
         const generationTsStart = new Date(
           spanTsStart.getTime() +
             Math.floor(
-              Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime()),
-            ),
+              Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime())
+            )
         );
         const generationTsEnd = new Date(
           generationTsStart.getTime() +
             Math.floor(
               Math.random() *
-                (spanTsEnd.getTime() - generationTsStart.getTime()),
-            ),
+                (spanTsEnd.getTime() - generationTsStart.getTime())
+            )
         );
         // somewhere in the middle
         const generationTsCompletionStart = new Date(
           generationTsStart.getTime() +
             Math.floor(
-              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3,
-            ),
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
+            )
         );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
@@ -828,7 +880,7 @@ function createObjects(
         const promptId =
           promptIds.get(projectId)![
             Math.floor(
-              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2),
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
             )
           ];
 
@@ -910,8 +962,8 @@ function createObjects(
           const eventTs = new Date(
             spanTsStart.getTime() +
               Math.floor(
-                Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime()),
-              ),
+                Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime())
+              )
           );
 
           events.push({
@@ -933,7 +985,7 @@ function createObjects(
   }
   // find unique sessions by id and projectid
   const uniqueSessions: Prisma.TraceSessionCreateManyInput[] = Array.from(
-    new Set(sessions.map((session) => JSON.stringify(session))),
+    new Set(sessions.map((session) => JSON.stringify(session)))
   ).map((session) => JSON.parse(session) as Prisma.TraceSessionCreateManyInput);
 
   return {
@@ -941,6 +993,7 @@ function createObjects(
     observations,
     scores,
     configs,
+    queueItems,
     sessions: uniqueSessions,
     events,
     comments,
@@ -954,7 +1007,7 @@ async function generatePromptsForProject(projects: Project[]) {
     projects.map(async (project) => {
       const promptIdsForProject = await generatePrompts(project);
       promptIds.set(project.id, promptIdsForProject);
-    }),
+    })
   );
   return promptIds;
 }
@@ -1140,7 +1193,7 @@ async function generateConfigsForProject(projects: Project[]) {
     projects.map(async (project) => {
       const configNameAndId = await generateConfigs(project);
       projectIdsToConfigs.set(project.id, configNameAndId);
-    }),
+    })
   );
   return projectIdsToConfigs;
 }
@@ -1218,6 +1271,7 @@ async function generateConfigs(project: Project) {
 
   return configNameAndId;
 }
+
 function getGenerationInputOutput(): {
   input: Prisma.InputJsonValue;
   output: Prisma.InputJsonValue;
@@ -1278,3 +1332,64 @@ function getGenerationInputOutput(): {
 
   return { input, output };
 }
+
+async function generateQueuesForProject(
+  projects: Project[],
+  configIdsAndNames: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
+) {
+  const projectIdsToQueues: Map<string, string[]> = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const queueIds = await generateQueues(
+        project,
+        configIdsAndNames.get(project.id) ?? []
+      );
+      projectIdsToQueues.set(project.id, queueIds);
+    })
+  );
+  return projectIdsToQueues;
+}
+
+async function generateQueues(
+  project: Project,
+  configIdsAndNames: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[]
+) {
+  const queue = {
+    id: `queue-${v4()}`,
+    name: "Default",
+    description: "Default queue",
+    scoreConfigIds: configIdsAndNames.map((config) => config.id),
+    projectId: project.id,
+  };
+
+  await prisma.annotationQueue.upsert({
+    where: {
+      projectId_name: {
+        projectId: queue.projectId,
+        name: queue.name,
+      },
+    },
+    create: {
+      ...queue,
+    },
+    update: {
+      id: queue.id,
+    },
+  });
+
+  return [queue.id];
+}

packages/shared/src/env.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
   NODE_ENV: z
@@ -21,7 +22,7 @@ const EnvSchema = z.object({
     .string()
     .length(
       64,
-      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
+      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
     )
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
@@ -45,4 +46,4 @@ const EnvSchema = z.object({
   LANGFUSE_LOG_FORMAT: z.enum(["text", "json"]).default("text"),
 });
 
-export const env = EnvSchema.parse(process.env);
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/annotation/types.ts

@@ -32,6 +32,7 @@ const CreateAnnotationScoreBase = z.object({
   configId: z.string().optional(),
   observationId: z.string().optional(),
   comment: z.string().nullish(),
+  queueId: z.string().nullish(),
 });
 
 const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
@@ -57,3 +58,15 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
   UpdateAnnotationScoreBase.merge(CategoricalData),
   UpdateAnnotationScoreBase.merge(BooleanData),
 ]);
+
+// annotation queues
+
+export const CreateQueueData = z.object({
+  name: z.string().min(1).max(35),
+  description: z.string().max(1000).optional(),
+  scoreConfigIds: z.array(z.string()).min(1, {
+    message: "At least 1 score config must be selected",
+  }),
+});
+
+export type CreateQueue = z.infer<typeof CreateQueueData>;

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,4 @@
 import z from "zod";
-import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,18 +76,6 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
-  [LLMAdapter.OpenAI]: [
-    "gpt-4o",
-    "gpt-4o-mini",
-    "gpt-4o-2024-08-06", // cheaper than gpt-4o
-    "gpt-4-turbo-preview",
-    "gpt-3.5-turbo",
-  ] as const,
-  [LLMAdapter.Anthropic]: [] as const,
-  [LLMAdapter.Azure]: [] as const,
-};
-
 export const OutputSchema = z.object({
   reasoning: z.string(),
   score: z.string(),

packages/shared/src/index.ts

@@ -3,6 +3,7 @@ export * from "./interfaces/filters";
 export * from "./interfaces/orderBy";
 export * from "./interfaces/cloudConfigSchema";
 export * from "./interfaces/parseDbOrg";
+export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
 export * from "./tracesTable";
@@ -18,7 +19,6 @@ export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";
-export * from "./server/llm/fetchLLMCompletion";
 
 // evals
 export * from "./features/evals/types";
@@ -40,3 +40,5 @@ export { type DB } from "../prisma/generated/types";
 
 // errors
 export * from "./errors/index";
+
+export * from "./utils/environment";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+export const BedrockConfigSchema = z.object({ region: z.string() });
+export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
+
+export const BedrockCredentialSchema = z
+  .object({
+    accessKeyId: z.string(),
+    secretAccessKey: z.string(),
+  })
+  .optional();
+export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -10,6 +10,7 @@ import {
   traceEvent,
   scoreEvent,
   sdkLogEvent,
+  ingestionEvent,
 } from "../types";
 import { validateAndInflateScore } from "../validateAndInflateScore";
 import { Trace, Observation, Score, Prisma, Model } from "@prisma/client";
@@ -19,16 +20,41 @@ import { jsonSchema } from "../../../utils/zod";
 import { prisma } from "../../../db";
 import { LegacyIngestionAccessScope } from ".";
 import { logger } from "../../logger";
-import { ApiAccessScope } from "../../auth/types";
 
 export interface EventProcessor {
   auth(apiScope: LegacyIngestionAccessScope): void;
 
   process(
-    apiScope: LegacyIngestionAccessScope
+    apiScope: LegacyIngestionAccessScope,
   ): Promise<Trace | Observation | Score> | undefined;
 }
 
+export const getProcessorForEvent = (
+  event: z.infer<typeof ingestionEvent>,
+  calculateTokenDelegate: (p: {
+    model: Model;
+    text: unknown;
+  }) => number | undefined,
+): EventProcessor => {
+  switch (event.type) {
+    case eventTypes.TRACE_CREATE:
+      return new TraceProcessor(event);
+    case eventTypes.OBSERVATION_CREATE:
+    case eventTypes.OBSERVATION_UPDATE:
+    case eventTypes.EVENT_CREATE:
+    case eventTypes.SPAN_CREATE:
+    case eventTypes.SPAN_UPDATE:
+    case eventTypes.GENERATION_CREATE:
+    case eventTypes.GENERATION_UPDATE:
+      return new ObservationProcessor(event, calculateTokenDelegate);
+    case eventTypes.SCORE_CREATE: {
+      return new ScoreProcessor(event);
+    }
+    case eventTypes.SDK_LOG:
+      return new SdkLogProcessor(event);
+  }
+};
+
 export class ObservationProcessor implements EventProcessor {
   event: ObservationEvent;
   calculateTokenDelegate: (p: {
@@ -41,7 +67,7 @@ export class ObservationProcessor implements EventProcessor {
     calculateTokenDelegate: (p: {
       model: Model;
       text: unknown;
-    }) => number | undefined
+    }) => number | undefined,
   ) {
     this.event = event;
     this.calculateTokenDelegate = calculateTokenDelegate;
@@ -49,7 +75,7 @@ export class ObservationProcessor implements EventProcessor {
 
   async convertToObservation(
     apiScope: LegacyIngestionAccessScope,
-    existingObservation: Omit<Observation, "input" | "output"> | null
+    existingObservation: Omit<Observation, "input" | "output"> | null,
   ): Promise<{
     id: string;
     create: Prisma.ObservationUncheckedCreateInput;
@@ -79,7 +105,7 @@ export class ObservationProcessor implements EventProcessor {
       !existingObservation
     ) {
       throw new LangfuseNotFoundError(
-        `Observation with id ${this.event.id} not found`
+        `Observation with id ${this.event.id} not found`,
       );
     }
 
@@ -126,7 +152,7 @@ export class ObservationProcessor implements EventProcessor {
             this.event.body,
             this.calculateTokenDelegate,
             internalModel ?? undefined,
-            existingObservation ?? undefined
+            existingObservation ?? undefined,
           )
         : [undefined, undefined];
 
@@ -162,7 +188,7 @@ export class ObservationProcessor implements EventProcessor {
     const calculatedCosts = ObservationProcessor.calculateTokenCosts(
       internalModel,
       userProvidedTokenCosts,
-      tokenCounts
+      tokenCounts,
     );
 
     // merge metadata from existingObservation.metadata and metadata
@@ -170,7 +196,7 @@ export class ObservationProcessor implements EventProcessor {
       existingObservation?.metadata
         ? jsonSchema.parse(existingObservation.metadata)
         : undefined,
-      this.event.body.metadata ?? undefined
+      this.event.body.metadata ?? undefined,
     );
 
     const prompt =
@@ -323,7 +349,7 @@ export class ObservationProcessor implements EventProcessor {
       text: unknown;
     }) => number | undefined,
     model?: Model,
-    existingObservation?: Omit<Observation, "input" | "output">
+    existingObservation?: Omit<Observation, "input" | "output">,
   ) {
     let newPromptTokens = body.usage?.input;
     if (newPromptTokens === undefined && model && model.tokenizerId) {
@@ -334,7 +360,7 @@ export class ObservationProcessor implements EventProcessor {
         });
       } else {
         logger.info(
-          `No input provided, trying to calculate for id: ${existingObservation?.id}`
+          `No input provided, trying to calculate for id: ${existingObservation?.id}`,
         );
         const observationInput = await prisma.observation.findFirst({
           where: { id: existingObservation?.id, projectId: projectId },
@@ -360,7 +386,7 @@ export class ObservationProcessor implements EventProcessor {
         });
       } else {
         logger.info(
-          `No output provided, trying to calculate for id: ${existingObservation?.id}`
+          `No output provided, trying to calculate for id: ${existingObservation?.id}`,
         );
         const observationOutput = await prisma.observation.findFirst({
           where: { id: existingObservation?.id, projectId: projectId },
@@ -385,7 +411,7 @@ export class ObservationProcessor implements EventProcessor {
       outputCost?: Decimal | null;
       totalCost?: Decimal | null;
     },
-    tokenCounts: { input?: number; output?: number; total?: number }
+    tokenCounts: { input?: number; output?: number; total?: number },
   ): {
     inputCost?: Decimal | null;
     outputCost?: Decimal | null;
@@ -402,7 +428,7 @@ export class ObservationProcessor implements EventProcessor {
         totalCost:
           userProvidedCosts.totalCost ??
           (userProvidedCosts.inputCost ?? new Decimal(0)).add(
-            userProvidedCosts.outputCost ?? new Decimal(0)
+            userProvidedCosts.outputCost ?? new Decimal(0),
           ),
       };
     }
@@ -488,7 +514,7 @@ export class ObservationProcessor implements EventProcessor {
       existingObservation.projectId !== apiScope.projectId
     ) {
       throw new ForbiddenError(
-        `Access denied for observation creation ${existingObservation.projectId} `
+        `Access denied for observation creation ${existingObservation.projectId} `,
       );
     }
 
@@ -505,6 +531,7 @@ export class ObservationProcessor implements EventProcessor {
     });
   }
 }
+
 export class TraceProcessor implements EventProcessor {
   event: z.infer<typeof traceEvent>;
 
@@ -518,7 +545,7 @@ export class TraceProcessor implements EventProcessor {
   }
 
   async process(
-    apiScope: LegacyIngestionAccessScope
+    apiScope: LegacyIngestionAccessScope,
   ): Promise<Trace | Observation | Score> {
     const { body } = this.event;
 
@@ -527,7 +554,7 @@ export class TraceProcessor implements EventProcessor {
     const internalId = body.id ?? v4();
 
     logger.debug(
-      `Trying to create trace, project ${apiScope.projectId}, id: ${internalId}`
+      `Trying to create trace, project ${apiScope.projectId}, id: ${internalId}`,
     );
 
     const existingTrace = await prisma.trace.findFirst({
@@ -538,7 +565,7 @@ export class TraceProcessor implements EventProcessor {
 
     if (existingTrace && existingTrace.projectId !== apiScope.projectId) {
       throw new ForbiddenError(
-        `Access denied for trace creation ${existingTrace.projectId}`
+        `Access denied for trace creation ${existingTrace.projectId}`,
       );
     }
 
@@ -546,7 +573,7 @@ export class TraceProcessor implements EventProcessor {
       existingTrace?.metadata
         ? jsonSchema.parse(existingTrace.metadata)
         : undefined,
-      body.metadata ?? undefined
+      body.metadata ?? undefined,
     );
 
     const mergedTags =
@@ -614,6 +641,7 @@ export class TraceProcessor implements EventProcessor {
     return upsertedTrace;
   }
 }
+
 export class ScoreProcessor implements EventProcessor {
   event: z.infer<typeof scoreEvent>;
 
@@ -624,12 +652,12 @@ export class ScoreProcessor implements EventProcessor {
   auth(apiScope: LegacyIngestionAccessScope) {
     if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
       throw new ForbiddenError(
-        `Access denied for score creation, ${apiScope.accessLevel}`
+        `Access denied for score creation, ${apiScope.accessLevel}`,
       );
   }
 
   async process(
-    apiScope: LegacyIngestionAccessScope
+    apiScope: LegacyIngestionAccessScope,
   ): Promise<Trace | Observation | Score> {
     const { body } = this.event;
 
@@ -647,7 +675,7 @@ export class ScoreProcessor implements EventProcessor {
     });
     if (existingScore && existingScore.projectId !== apiScope.projectId) {
       throw new ForbiddenError(
-        `Access denied for score creation ${existingScore.projectId}`
+        `Access denied for score creation ${existingScore.projectId}`,
       );
     }
 

packages/shared/src/server/ingestion/legacy/index.ts

@@ -2,14 +2,8 @@ import { env } from "node:process";
 import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, ingestionEvent } from "../types";
-import {
-  EventProcessor,
-  TraceProcessor,
-  ObservationProcessor,
-  ScoreProcessor,
-  SdkLogProcessor,
-} from "./EventProcessor";
-import { EventBodyType, EventName, TraceUpsertEventType } from "../../queues";
+import { getProcessorForEvent } from "./EventProcessor";
+import { TraceUpsertEventType } from "../../queues";
 import {
   convertTraceUpsertEventsToRedisEvents,
   getTraceUpsertQueue,
@@ -144,40 +138,18 @@ const handleSingleEvent = async (
 
   const cleanedEvent = ingestionEvent.parse(cleanEvent(event));
 
-  const { type } = cleanedEvent;
-
-  let processor: EventProcessor;
-  switch (type) {
-    case eventTypes.TRACE_CREATE:
-      processor = new TraceProcessor(cleanedEvent);
-      break;
-    case eventTypes.OBSERVATION_CREATE:
-    case eventTypes.OBSERVATION_UPDATE:
-    case eventTypes.EVENT_CREATE:
-    case eventTypes.SPAN_CREATE:
-    case eventTypes.SPAN_UPDATE:
-    case eventTypes.GENERATION_CREATE:
-    case eventTypes.GENERATION_UPDATE:
-      processor = new ObservationProcessor(
-        cleanedEvent,
-        calculateTokenDelegate,
-      );
-      break;
-    case eventTypes.SCORE_CREATE: {
-      processor = new ScoreProcessor(cleanedEvent);
-      break;
-    }
-    case eventTypes.SDK_LOG:
-      processor = new SdkLogProcessor(cleanedEvent);
-  }
-
   // Deny access to non-score events if the access level is not "all"
   // This is an additional safeguard to auth checks in EventProcessor
-  if (apiScope.accessLevel !== "all" && type !== eventTypes.SCORE_CREATE) {
+  if (
+    apiScope.accessLevel !== "all" &&
+    cleanedEvent.type !== eventTypes.SCORE_CREATE
+  ) {
     throw new ForbiddenError("Access denied. Event type not allowed.");
   }
 
-  return await processor.process(apiScope);
+  return getProcessorForEvent(cleanedEvent, calculateTokenDelegate).process(
+    apiScope,
+  );
 };
 
 // cleans NULL characters from the event
@@ -208,7 +180,7 @@ export const isNotNullOrUndefined = <T>(
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
 
-export const sendToWorkerIfEnvironmentConfigured = async (
+export const addTracesToTraceUpsertQueue = async (
   batchResults: BatchResult[],
   projectId: string,
 ): Promise<void> => {
@@ -235,32 +207,6 @@ export const sendToWorkerIfEnvironmentConfigured = async (
       }
 
       await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    } else if (
-      env.LANGFUSE_WORKER_HOST &&
-      env.LANGFUSE_WORKER_PASSWORD &&
-      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
-    ) {
-      logger.info(`Sending ${traceEvents.length} events to worker via HTTP`);
-      const body: EventBodyType = {
-        name: EventName.TraceUpsert,
-        payload: traceEvents,
-      };
-
-      if (traceEvents.length > 0) {
-        await fetch(`${env.LANGFUSE_WORKER_HOST}/api/events`, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            Authorization:
-              "Basic " +
-              Buffer.from(
-                "admin" + ":" + env.LANGFUSE_WORKER_PASSWORD,
-              ).toString("base64"),
-          },
-          body: JSON.stringify(body),
-          signal: AbortSignal.timeout(8 * 1000),
-        });
-      }
     }
   } catch (error) {
     logger.error("Error sending events to worker", error);

packages/shared/src/server/ingestion/types.ts

@@ -115,7 +115,7 @@ export const CreateGenerationBody = CreateSpanBody.extend({
           z.array(z.string()),
           z.record(z.string()),
         ])
-        .nullish()
+        .nullish(),
     )
     .nullish(),
   usage: usage,
@@ -143,7 +143,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
           z.array(z.string()),
           z.record(z.string()),
         ])
-        .nullish()
+        .nullish(),
     )
     .nullish(),
   usage: usage,
@@ -174,14 +174,14 @@ export const ScoreBody = z.discriminatedUnion("dataType", [
       value: z.number(),
       dataType: z.literal("NUMERIC"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -191,14 +191,14 @@ export const ScoreBody = z.discriminatedUnion("dataType", [
       }),
       dataType: z.literal("BOOLEAN"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.union([z.string(), z.number()]),
       dataType: z.undefined(),
       configId: z.string().nullish(),
-    })
+    }),
   ),
 ]);
 
@@ -243,7 +243,7 @@ export const LegacyGenerationsCreateSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish()
+      z.union([z.string(), z.number(), z.boolean()]).nullish(),
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -267,7 +267,7 @@ export const LegacyGenerationPatchSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish()
+      z.union([z.string(), z.number(), z.boolean()]).nullish(),
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -291,7 +291,7 @@ export const LegacyObservationBody = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish()
+      z.union([z.string(), z.number(), z.boolean()]).nullish(),
     )
     .nullish(),
   input: jsonSchema.nullish(),
@@ -306,6 +306,7 @@ export const LegacyObservationBody = z.object({
 
 export const SdkLogEvent = z.object({
   log: jsonSchema,
+  id: z.string().nullish(), // Not used, but makes downstream processing easier.
 });
 
 // definitions for the ingestion API
@@ -402,7 +403,7 @@ export const ingestionBatchEvent = z.array(ingestionEvent);
 export type IngestionBatchEventType = z.infer<typeof ingestionBatchEvent>;
 
 export const ingestionEventWithProjectId = ingestionEvent.and(
-  z.object({ projectId: z.string() })
+  z.object({ projectId: z.string() }),
 );
 export type IngestionEventWithProjectIdType = z.infer<
   typeof ingestionEventWithProjectId

packages/shared/src/server/instrumentation/index.ts

@@ -86,18 +86,6 @@ export function instrumentSync<T>(
 
 export const getCurrentSpan = () => opentelemetry.trace.getActiveSpan();
 
-export const addTraceContext = <T extends Record<string, any>>(
-  input: T,
-): T & { _tracecontext?: TCarrier } => {
-  const context = {};
-  opentelemetry.propagation.inject(opentelemetry.context.active(), context);
-
-  return {
-    ...input,
-    _tracecontext: context,
-  };
-};
-
 export const traceException = (
   ex: unknown,
   span?: opentelemetry.Span,

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,20 +1,26 @@
-import { type ZodSchema } from "zod";
+import type { ZodSchema } from "zod";
 
 import { ChatAnthropic } from "@langchain/anthropic";
+import { ChatBedrockConverse } from "@langchain/aws";
 import {
   AIMessage,
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
-import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 import {
   BytesOutputParser,
   StringOutputParser,
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
+import {
+  BedrockConfigSchema,
+  BedrockCredentialSchema,
+} from "../../interfaces/customLLMProviderConfigSchemas";
 
-import { ChatMessage, ChatMessageRole, ModelParams, LLMAdapter } from "./types";
+import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
+
+import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
 type LLMCompletionParams = {
   messages: ChatMessage[];
@@ -22,8 +28,9 @@ type LLMCompletionParams = {
   structuredOutputSchema?: ZodSchema;
   callbacks?: BaseCallbackHandler[];
   baseURL?: string;
-  apiKey?: string;
+  apiKey: string;
   maxRetries?: number;
+  config?: Record<string, string> | null;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -61,6 +68,7 @@ export async function fetchLLMCompletion(
     apiKey,
     baseURL,
     maxRetries,
+    config,
   } = params;
 
   const finalMessages = messages.map((message) => {
@@ -72,7 +80,7 @@ export async function fetchLLMCompletion(
     return new AIMessage(message.content);
   });
 
-  let chatModel: ChatOpenAI | ChatAnthropic;
+  let chatModel: ChatOpenAI | ChatAnthropic | ChatBedrockConverse;
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
@@ -109,6 +117,20 @@ export async function fetchLLMCompletion(
       callbacks,
       maxRetries,
     });
+  } else if (modelParams.adapter === LLMAdapter.Bedrock) {
+    const { region } = BedrockConfigSchema.parse(config);
+    const credentials = BedrockCredentialSchema.parse(JSON.parse(apiKey));
+
+    chatModel = new ChatBedrockConverse({
+      model: modelParams.model,
+      region,
+      credentials,
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+      maxRetries,
+    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;

packages/shared/src/server/llm/types.ts

@@ -1,5 +1,6 @@
 import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
+import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
 
@@ -14,6 +15,7 @@ export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
   Azure = "azure",
+  Bedrock = "bedrock",
 }
 
 export enum ChatMessageRole {
@@ -90,6 +92,7 @@ export const supportedModels = {
   [LLMAdapter.Anthropic]: anthropicModels,
   [LLMAdapter.OpenAI]: openAIModels,
   [LLMAdapter.Azure]: [],
+  [LLMAdapter.Bedrock]: [],
 } as const;
 
 export type LLMFunctionCall = {
@@ -111,6 +114,7 @@ export const LLMApiKeySchema = z
     baseURL: z.string().nullable(),
     customModels: z.array(z.string()),
     withDefaultModels: z.boolean(),
+    config: BedrockConfigSchema.nullish(), // currently only Bedrock has additional config
   })
   // strict mode to prevent extra keys. Thorws error otherwise
   // https://github.com/colinhacks/zod?tab=readme-ov-file#strict

packages/shared/src/server/logger.ts

@@ -1,26 +1,21 @@
 import { env } from "../env";
 import winston from "winston";
-import Transport from "winston-transport";
 import { getCurrentSpan } from "./instrumentation";
 
-class TracedTransport extends Transport {
-  constructor(opts: Transport.TransportStreamOptions = {}) {
-    super(opts);
-  }
-
-  log(info: Record<string, any>, callback: () => void): void {
-    setImmediate(() => {
-      this.emit("logged", info);
-    });
-
-    const currentSpan = getCurrentSpan();
-    info.trace_id = currentSpan?.spanContext().traceId;
-    info.span_id = currentSpan?.spanContext().spanId;
-    console.log(JSON.stringify(info));
-
-    callback();
-  }
-}
+const tracingFormat = function () {
+  return winston.format((info) => {
+    const span = getCurrentSpan();
+    if (span) {
+      const { spanId, traceId } = span.spanContext();
+      const traceIdEnd = traceId.slice(traceId.length / 2);
+      info["dd.trace_id"] = BigInt(`0x${traceIdEnd}`).toString();
+      info["dd.span_id"] = BigInt(`0x${spanId}`).toString();
+      info["trace_id"] = traceId;
+      info["span_id"] = spanId;
+    }
+    return info;
+  })();
+};
 
 const getWinstonLogger = (
   nodeEnv: "development" | "production" | "test",
@@ -39,19 +34,16 @@ const getWinstonLogger = (
   const jsonLoggerFormat = winston.format.combine(
     winston.format.errors({ stack: true }),
     winston.format.timestamp(),
+    tracingFormat(),
     winston.format.json(),
   );
 
   const format =
     env.LANGFUSE_LOG_FORMAT === "text" ? textLoggerFormat : jsonLoggerFormat;
-  const transport =
-    env.LANGFUSE_LOG_FORMAT === "text"
-      ? new winston.transports.Console()
-      : new TracedTransport();
   return winston.createLogger({
     level: minLevel,
     format: format,
-    transports: [transport],
+    transports: [new winston.transports.Console()],
   });
 };
 

packages/shared/src/server/queues.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { ingestionBatchEvent, TCarrier } from ".";
+import { eventTypes, ingestionBatchEvent, TCarrier } from ".";
 
 export enum EventName {
   TraceUpsert = "TraceUpsert",
@@ -9,7 +9,8 @@ export enum EventName {
   CloudUsageMetering = "CloudUsageMetering",
 }
 
-export const LegacyIngestionEvent = z.object({
+export const LegacyIngestionEventFull = z.object({
+  useS3EventStore: z.literal(false),
   data: ingestionBatchEvent,
   authCheck: z.object({
     validKey: z.literal(true),
@@ -20,6 +21,29 @@ export const LegacyIngestionEvent = z.object({
   }),
 });
 
+export const LegacyIngestionEventMeta = z.object({
+  useS3EventStore: z.literal(true),
+  data: z.array(
+    z.object({
+      type: z.nativeEnum(eventTypes),
+      eventBodyId: z.string(),
+      eventId: z.string(),
+    })
+  ),
+  authCheck: z.object({
+    validKey: z.literal(true),
+    scope: z.object({
+      projectId: z.string(),
+      accessLevel: z.enum(["all", "scores"]),
+    }),
+  }),
+});
+
+export const LegacyIngestionEvent = z.discriminatedUnion("useS3EventStore", [
+  LegacyIngestionEventFull,
+  LegacyIngestionEventMeta,
+]);
+
 export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
@@ -31,6 +55,7 @@ export const TraceUpsertEventSchema = z.object({
 export const EvalExecutionEvent = z.object({
   projectId: z.string(),
   jobExecutionId: z.string(),
+  delay: z.number().nullish(),
 });
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;

packages/shared/src/server/services/S3StorageService.ts

@@ -25,11 +25,10 @@ export class S3StorageService {
     bucketName: string;
     endpoint: string | undefined;
     region: string | undefined;
+    forcePathStyle: boolean;
   }) {
-    const { accessKeyId, secretAccessKey, bucketName, endpoint, region } =
-      params;
-
     // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
+    const { accessKeyId, secretAccessKey } = params;
     const credentials =
       accessKeyId !== undefined && secretAccessKey !== undefined
         ? {
@@ -40,10 +39,11 @@ export class S3StorageService {
 
     this.client = new S3Client({
       credentials,
-      endpoint,
-      region,
+      endpoint: params.endpoint,
+      region: params.region,
+      forcePathStyle: params.forcePathStyle,
     });
-    this.bucketName = bucketName;
+    this.bucketName = params.bucketName;
   }
 
   public async uploadFile({
@@ -72,7 +72,7 @@ export class S3StorageService {
     }
   }
 
-  public async uploadJson(path: string, body: Record<string, unknown>) {
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
     const putCommand = new PutObjectCommand({
       Bucket: this.bucketName,
       Key: path,
@@ -88,6 +88,21 @@ export class S3StorageService {
     }
   }
 
+  public async download(path: string): Promise<string> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+
+    try {
+      const response = await this.client.send(getCommand);
+      return (await response.Body?.transformToString()) ?? "";
+    } catch (err) {
+      logger.error(`Failed to download file from S3 ${path}`, err);
+      throw Error("Failed to download file from S3");
+    }
+  }
+
   private async getSignedUrl(
     fileName: string,
     ttlSeconds: number,

packages/shared/src/server/services/email/organizationInvitation/sendMembershipInvitationEmail.ts

@@ -36,7 +36,7 @@ export const sendMembershipInvitationEmail = async ({
 }: SendMembershipInvitationParams) => {
   if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
     logger.error(
-      "Missing environment variables for sending membership invitation email.",
+      "Missing environment variables for sending membership invitation email."
     );
     return;
   }
@@ -51,7 +51,7 @@ export const sendMembershipInvitationEmail = async ({
   const authUrl = getAuthURL();
   if (!authUrl) {
     logger.error(
-      "Missing NEXTAUTH_URL or NEXT_PUBLIC_LANGFUSE_CLOUD_REGION environment variable.",
+      "Missing NEXTAUTH_URL or NEXT_PUBLIC_LANGFUSE_CLOUD_REGION environment variable."
     );
     return;
   }
@@ -68,7 +68,7 @@ export const sendMembershipInvitationEmail = async ({
         inviteLink: authUrl,
         emailFromAddress: env.EMAIL_FROM_ADDRESS,
         langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
-      }),
+      })
     );
 
     await mailer.sendMail({

packages/shared/src/utils/environment.ts

@@ -0,0 +1,10 @@
+// https://github.com/t3-oss/t3-env/blob/e7e21095e00a477e37608783defda5a6a99586d0/packages/core/src/index.ts#L228
+// unfortunately, we are not able to install t3-env in all our packaging as some rely on commonjs.
+export const removeEmptyEnvVariables = (runtimeEnv: any) => {
+  for (const [key, value] of Object.entries(runtimeEnv)) {
+    if (value === "") {
+      delete runtimeEnv[key];
+    }
+  }
+  return runtimeEnv;
+};

web/Dockerfile

@@ -106,7 +106,7 @@ RUN apk add --no-cache dumb-init
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
-RUN npm install -g --no-package-lock --no-save prisma@5.13.0
+RUN npm install -g --no-package-lock --no-save prisma@5.20.0
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.82.0",
+  "version": "2.85.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.1",
+    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.3",
     "@baselime/trpc-opentelemetry-middleware": "^0.1.2",
     "@codemirror/lang-json": "^6.0.1",
     "@codemirror/lint": "^6.8.0",
@@ -31,17 +31,17 @@
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
     "@dnd-kit/utilities": "^3.2.2",
-    "@headlessui/react": "1.7.18",
+    "@headlessui/react": "2.1.9",
     "@headlessui/tailwindcss": "0.2.1",
-    "@heroicons/react": "^2.1.3",
+    "@heroicons/react": "^2.1.5",
     "@hookform/resolvers": "^3.3.4",
     "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.3",
+    "@langchain/core": "^0.3.9",
     "@langchain/openai": "^0.3.0",
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
-    "@mui/x-tree-view": "^7.6.2",
+    "@mui/x-tree-view": "^7.19.0",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/core": "^1.26.0",
@@ -51,52 +51,51 @@
     "@opentelemetry/instrumentation-http": "^0.53.0",
     "@opentelemetry/instrumentation-ioredis": "^0.43.0",
     "@opentelemetry/instrumentation-winston": "^0.40.0",
-    "@opentelemetry/resource-detector-aws": "^1.6.1",
-    "@opentelemetry/resource-detector-container": "^0.4.1",
+    "@opentelemetry/resource-detector-aws": "^1.6.2",
+    "@opentelemetry/resource-detector-container": "^0.4.2",
     "@opentelemetry/resources": "^1.26.0",
     "@opentelemetry/sdk-node": "^0.53.0",
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/sdk-trace-node": "^1.26.0",
-    "@opentelemetry/winston-transport": "^0.6.0",
     "@prisma/instrumentation": "^5.20.0",
-    "@radix-ui/react-accordion": "^1.2.0",
-    "@radix-ui/react-alert-dialog": "^1.1.1",
-    "@radix-ui/react-avatar": "^1.1.0",
-    "@radix-ui/react-checkbox": "^1.1.1",
-    "@radix-ui/react-collapsible": "^1.1.0",
-    "@radix-ui/react-dialog": "^1.1.1",
-    "@radix-ui/react-dropdown-menu": "^2.1.1",
-    "@radix-ui/react-hover-card": "^1.1.1",
+    "@radix-ui/react-accordion": "^1.2.1",
+    "@radix-ui/react-alert-dialog": "^1.1.2",
+    "@radix-ui/react-avatar": "^1.1.1",
+    "@radix-ui/react-checkbox": "^1.1.2",
+    "@radix-ui/react-collapsible": "^1.1.1",
+    "@radix-ui/react-dialog": "^1.1.2",
+    "@radix-ui/react-dropdown-menu": "^2.1.2",
+    "@radix-ui/react-hover-card": "^1.1.2",
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.1.0",
-    "@radix-ui/react-popover": "^1.1.1",
-    "@radix-ui/react-scroll-area": "^1.1.0",
-    "@radix-ui/react-select": "^2.1.1",
+    "@radix-ui/react-popover": "^1.1.2",
+    "@radix-ui/react-scroll-area": "^1.2.0",
+    "@radix-ui/react-select": "^2.1.2",
     "@radix-ui/react-separator": "^1.1.0",
-    "@radix-ui/react-slider": "^1.2.0",
+    "@radix-ui/react-slider": "^1.2.1",
     "@radix-ui/react-slot": "^1.1.0",
-    "@radix-ui/react-switch": "^1.1.0",
-    "@radix-ui/react-tabs": "^1.1.0",
+    "@radix-ui/react-switch": "^1.1.1",
+    "@radix-ui/react-tabs": "^1.1.1",
     "@radix-ui/react-toggle": "^1.1.0",
     "@radix-ui/react-toggle-group": "^1.1.0",
-    "@radix-ui/react-tooltip": "^1.1.2",
+    "@radix-ui/react-tooltip": "^1.1.3",
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.31.0",
-    "@t3-oss/env-nextjs": "^0.10.1",
+    "@sentry/nextjs": "^8.33.1",
+    "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
-    "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-table": "^8.20.5",
     "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.23.0",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
+    "@uiw/codemirror-theme-tokyo-night": "^4.23.5",
     "@uiw/react-codemirror": "^4.21.25",
-    "ai": "^3.0.23",
+    "ai": "^3.4.9",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.12.10",
     "class-variance-authority": "^0.7.0",
@@ -105,25 +104,25 @@
     "core-js": "^3.38.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
-    "dd-trace": "^5.21.0",
+    "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "dompurify": "^3.1.5",
     "graphql": "^16.9.0",
     "ioredis": "^5.4.1",
     "ip-address": "^9.0.5",
-    "js-tiktoken": "^1.0.12",
+    "js-tiktoken": "^1.0.15",
     "kysely": "^0.27.4",
     "langchain": "^0.3.2",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.436.0",
-    "next": "^14.2.6",
+    "lucide-react": "^0.447.0",
+    "next": "^14.2.15",
     "next-auth": "^4.24.7",
-    "next-query-params": "^5.0.0",
+    "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.161.3",
     "posthog-node": "^3.6.3",
     "prexit": "^2.2.0",
-    "prisma": "^5.13.0",
+    "prisma": "^5.20.0",
     "rate-limiter-flexible": "^5.0.3",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
@@ -149,8 +148,8 @@
   },
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
-    "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.43.1",
+    "@mermaid-js/mermaid-cli": "^11.2.0",
+    "@playwright/test": "^1.47.2",
     "@tailwindcss/forms": "^0.5.7",
     "@testing-library/jest-dom": "^6.4.6",
     "@testing-library/react": "^15.0.7",
@@ -159,7 +158,7 @@
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
     "@types/node": "20.10.5",
     "@types/react": "~18.2.79",
     "@types/react-dom": "~18.2.25",
@@ -170,17 +169,17 @@
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.2.6",
+    "eslint-config-next": "^14.2.15",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",
-    "postcss": "^8.4.38",
-    "prettier": "^3.2.5",
+    "postcss": "^8.4.47",
+    "prettier": "^3.3.3",
     "prettier-plugin-tailwindcss": "^0.6.6",
-    "tailwindcss": "^3.4.10",
+    "tailwindcss": "^3.4.13",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.19.0",
+    "tsx": "^4.19.1",
     "typescript": "^5.4.5",
     "wait-for-expect": "^3.0.2"
   },

web/src/__tests__/generation-exports.servertest.ts

@@ -14,7 +14,7 @@ describe("observations.export RPC", () => {
 
   beforeAll(async () => {
     // Disable S3 upload
-    process.env.S3_ENDPOINT = "";
+    process.env.S3_BUCKET_NAME = "";
 
     await pruneDatabase();
     const traceId = "trace-1";

web/src/__tests__/rate-limit.servertest.ts

@@ -39,6 +39,8 @@ describe("RateLimitService", () => {
       rateLimitOverrides: [],
     };
 
+    expect(redis).toBeDefined();
+
     const rateLimitService = new RateLimitService(redis!);
     const result = await rateLimitService.rateLimitRequest(scope, "public-api");
 

web/src/components/layouts/ClickhouseAdminToggle.tsx

@@ -0,0 +1,40 @@
+import { Switch } from "@/src/components/ui/switch";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { useSession } from "next-auth/react";
+
+export const useClickhouse = () => {
+  const session = useSession();
+  const [isEnabled] = useLocalStorage<boolean>("useClickhouseQueries", false);
+
+  return isEnabled && session.data?.user?.admin === true;
+};
+
+export function ClickhouseAdminToggle() {
+  const [isEnabled, setIsEnabled] = useLocalStorage<boolean>(
+    "useClickhouseQueries",
+    false,
+  );
+
+  const handleToggle = () => {
+    setIsEnabled((prev) => !prev);
+    // You can add any additional logic here, such as API calls or analytics tracking
+  };
+
+  return (
+    <div className="ml-auto flex items-center space-x-1">
+      <div
+        title={
+          isEnabled ? "Disable Clickhouse Queries" : "Enable Clickhouse Queries"
+        }
+      >
+        <Switch
+          id="clickhouse-toggle"
+          checked={isEnabled}
+          onCheckedChange={() => {
+            handleToggle();
+          }}
+        />
+      </div>
+    </div>
+  );
+}

web/src/components/layouts/layout.tsx

@@ -34,6 +34,7 @@ import { useQueryProjectOrOrganization } from "@/src/features/projects/hooks";
 import { useOrgEntitlements } from "@/src/features/entitlements/hooks";
 import { useUiCustomization } from "@/src/ee/features/ui-customization/useUiCustomization";
 import { hasOrganizationAccess } from "@/src/features/rbac/utils/checkOrganizationAccess";
+import { ClickhouseAdminToggle } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 const signOutUser = async () => {
   localStorage.clear();
@@ -42,17 +43,30 @@ const signOutUser = async () => {
   await signOut();
 };
 
-const userNavigation = [
-  {
-    name: "Theme",
-    onClick: () => {},
-    content: <ThemeToggle />,
-  },
-  {
-    name: "Sign out",
-    onClick: signOutUser,
-  },
-];
+const getUserNavigation = (isAdmin: boolean) => {
+  const navigationItems = [
+    {
+      name: "Theme",
+      onClick: () => {},
+      content: <ThemeToggle />,
+    },
+    {
+      name: "Sign out",
+      onClick: signOutUser,
+    },
+  ];
+
+  return isAdmin
+    ? [
+        {
+          name: "CH Query",
+          onClick: () => {},
+          content: <ClickhouseAdminToggle />,
+        },
+        ...navigationItems,
+      ]
+    : navigationItems;
+};
 
 const pathsWithoutNavigation: string[] = [
   "/onboarding",
@@ -186,7 +200,7 @@ export default function Layout(props: PropsWithChildren) {
 
     const href = (
       route.customizableHref
-        ? uiCustomization?.[route.customizableHref] ?? route.pathname
+        ? (uiCustomization?.[route.customizableHref] ?? route.pathname)
         : route.pathname
     )
       ?.replace("[projectId]", routerProjectId ?? "")
@@ -441,14 +455,14 @@ export default function Layout(props: PropsWithChildren) {
                 leaveFrom="transform opacity-100 scale-100"
                 leaveTo="transform opacity-0 scale-95"
               >
-                <Menu.Items className="absolute -top-full bottom-1 right-0 z-10 overflow-hidden rounded-md bg-background py-2 shadow-lg ring-1 ring-border focus:outline-none">
+                <Menu.Items className="absolute bottom-1 right-0 z-10 overflow-hidden rounded-md bg-background py-2 shadow-lg ring-1 ring-border focus:outline-none">
                   <span
                     className="block max-w-52 overflow-hidden truncate border-b px-3 pb-2 text-sm leading-6 text-muted-foreground"
                     title={session.data?.user?.email ?? undefined}
                   >
                     {session.data?.user?.email}
                   </span>
-                  {userNavigation.map((item) => (
+                  {getUserNavigation(cloudAdmin).map((item) => (
                     <Menu.Item key={item.name}>
                       {({ active }) => (
                         <a
@@ -516,7 +530,7 @@ export default function Layout(props: PropsWithChildren) {
                 >
                   {session.data?.user?.email}
                 </span>
-                {userNavigation.map((item) => (
+                {getUserNavigation(cloudAdmin).map((item) => (
                   <Menu.Item key={item.name}>
                     {({ active }) => (
                       <a

web/src/components/layouts/routes.tsx

@@ -8,12 +8,13 @@ import {
   type LucideIcon,
   Settings,
   UsersIcon,
-  PenSquareIcon,
   LibraryBig,
   TerminalIcon,
   Lightbulb,
   Grid2X2,
   Sparkle,
+  ClipboardPen,
+  FileJson,
 } from "lucide-react";
 import { LangfuseIcon } from "@/src/components/LangfuseLogo";
 import { type ReactNode } from "react";
@@ -86,6 +87,14 @@ export const ROUTES: Route[] = [
       },
     ],
   },
+  {
+    name: "Annotate",
+    pathname: `/project/[projectId]/annotation-queues`,
+    icon: ClipboardPen,
+    label: "Beta",
+    projectRbacScope: "annotationQueues:read",
+    entitlement: "annotation-queues",
+  },
   {
     name: "Evaluation",
     icon: Lightbulb,
@@ -120,7 +129,7 @@ export const ROUTES: Route[] = [
   {
     name: "Prompts",
     pathname: "/project/[projectId]/prompts",
-    icon: PenSquareIcon,
+    icon: FileJson,
     projectRbacScope: "prompts:read",
   },
   {

web/src/components/layouts/status-badge.tsx

@@ -11,32 +11,35 @@ const statusCategories = {
 export type Status =
   (typeof statusCategories)[keyof typeof statusCategories][number];
 
-export const StatusBadge = (props: {
-  className?: string;
+export const StatusBadge = ({
+  type,
+  isLive = true,
+  className,
+}: {
   type: Status | (string & {});
+  isLive?: boolean;
+  className?: string;
 }) => {
   let badgeColor = "bg-muted-gray text-primary";
   let dotColor = "bg-muted-foreground";
   let dotPingColor = "bg-muted-foreground";
-  let showDot = false;
+  let showDot = isLive;
 
-  if (statusCategories.active.includes(props.type)) {
+  if (statusCategories.active.includes(type)) {
     badgeColor = "bg-light-green text-dark-green";
     dotColor = "animate-ping bg-dark-green";
     dotPingColor = "bg-dark-green";
-    showDot = true;
-  } else if (statusCategories.pending.includes(props.type)) {
+  } else if (statusCategories.pending.includes(type)) {
     badgeColor = "bg-light-yellow text-dark-yellow";
     dotColor = "animate-ping bg-dark-yellow";
     dotPingColor = "bg-dark-yellow";
-    showDot = true;
-  } else if (statusCategories.error.includes(props.type)) {
+  } else if (statusCategories.error.includes(type)) {
     badgeColor = "bg-light-red text-dark-red";
     dotColor = "animate-ping bg-dark-red";
     dotPingColor = "bg-dark-red";
-    showDot = true;
-  } else if (statusCategories.completed.includes(props.type)) {
+  } else if (statusCategories.completed.includes(type)) {
     badgeColor = "bg-light-green text-dark-green";
+    showDot = false;
   }
 
   return (
@@ -44,26 +47,26 @@ export const StatusBadge = (props: {
       className={cn(
         "inline-flex items-center gap-2 rounded-sm px-2 py-1 text-xs",
         badgeColor,
-        props.className,
+        className,
       )}
     >
       {showDot && (
         <span className="relative inline-flex h-2 w-2">
           <span
             className={cn(
-              "absolute inline-flex h-full w-full  rounded-full opacity-75",
+              "absolute inline-flex h-full w-full rounded-full opacity-75",
               dotColor,
             )}
           ></span>
           <span
             className={cn(
-              "relative inline-flex h-2 w-2 rounded-full ",
+              "relative inline-flex h-2 w-2 rounded-full",
               dotPingColor,
             )}
           ></span>
         </span>
       )}
-      <span>{props.type}</span>
+      <span>{type}</span>
     </div>
   );
 };

web/src/components/publish-object-switch.tsx

@@ -93,7 +93,7 @@ const Base = (props: {
           "Loading.."
         ) : props.isPublic ? (
           <div
-            className="text-dark-green flex cursor-pointer items-center gap-1"
+            className="flex cursor-pointer items-center gap-1 text-dark-green"
             onClick={() => copyUrl()}
           >
             {isCopied ? "Link copied ..." : "Public"}

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -1,4 +1,4 @@
-import { ChevronDown, Trash } from "lucide-react";
+import { ChevronDown, ClipboardPen, Trash } from "lucide-react";
 
 import {
   DropdownMenu,
@@ -20,6 +20,30 @@ import {
 } from "@/src/components/ui/dialog";
 import { useState } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import {
+  Select,
+  SelectTrigger,
+  SelectContent,
+  SelectItem,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import { z } from "zod";
+import { useSession } from "next-auth/react";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
+
+const addToQueueFormSchema = z.object({
+  queueId: z.string(),
+});
 
 export function TraceTableMultiSelectAction({
   selectedTraceIds,
@@ -31,7 +55,9 @@ export function TraceTableMultiSelectAction({
   onDeleteSuccess: () => void;
 }) {
   const utils = api.useUtils();
-  const [open, setOpen] = useState(false);
+  const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
+  const [addToQueueDialogOpen, setAddToQueueDialogOpen] = useState(false);
+  const session = useSession();
   const capture = usePostHogClientCapture();
 
   const hasDeleteAccess = useHasProjectAccess({
@@ -45,6 +71,35 @@ export function TraceTableMultiSelectAction({
     },
   });
 
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
+  const hasAddToQueueAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const mutAddToQueue = api.annotationQueueItems.createMany.useMutation({
+    onSuccess: (data) => {
+      showSuccessToast({
+        title: "Traces added to queue",
+        description: `${selectedTraceIds.length} traces added to queue "${data.queueName}".`,
+        link: {
+          href: `/project/${projectId}/annotation-queues/${data.queueId}`,
+          text: `View queue "${data.queueName}"`,
+        },
+      });
+    },
+  });
+
+  const form = useForm<z.infer<typeof addToQueueFormSchema>>({
+    resolver: zodResolver(addToQueueFormSchema),
+  });
+
+  const queues = api.annotationQueues.allNamesAndIds.useQuery(
+    {
+      projectId,
+    },
+    { enabled: session.status === "authenticated" && hasEntitlement },
+  );
+
   return (
     <>
       <DropdownMenu>
@@ -62,19 +117,31 @@ export function TraceTableMultiSelectAction({
                 count: selectedTraceIds.length,
                 source: "table-multi-select",
               });
-              setOpen(true);
+              setDeleteDialogOpen(true);
             }}
           >
             <Trash className="mr-2 h-4 w-4" />
             <span>Delete</span>
           </DropdownMenuItem>
+          {hasEntitlement && (
+            <DropdownMenuItem
+              disabled={!hasAddToQueueAccess}
+              onClick={() => {
+                setAddToQueueDialogOpen(true);
+              }}
+            >
+              <ClipboardPen className="mr-2 h-4 w-4" />
+              <span>Add to Annotation Queue</span>
+            </DropdownMenuItem>
+          )}
         </DropdownMenuContent>
       </DropdownMenu>
+
       <Dialog
-        open={open}
+        open={deleteDialogOpen}
         onOpenChange={(isOpen) => {
           if (!isOpen) {
-            setOpen(false);
+            setDeleteDialogOpen(false);
           }
         }}
       >
@@ -99,7 +166,7 @@ export function TraceTableMultiSelectAction({
                     projectId,
                   })
                   .then(() => {
-                    setOpen(false);
+                    setDeleteDialogOpen(false);
                   });
                 capture("trace:delete_form_submit", {
                   count: selectedTraceIds.length,
@@ -112,6 +179,79 @@ export function TraceTableMultiSelectAction({
           </DialogFooter>
         </DialogContent>
       </Dialog>
+
+      <Dialog
+        open={addToQueueDialogOpen}
+        onOpenChange={(isOpen) => {
+          if (!isOpen) {
+            setAddToQueueDialogOpen(false);
+          }
+        }}
+      >
+        <DialogContent className="sm:max-w-md">
+          <Form {...form}>
+            <form
+              className="space-y-6"
+              onSubmit={form.handleSubmit((data) => {
+                if (data.queueId) {
+                  void mutAddToQueue
+                    .mutateAsync({
+                      projectId,
+                      queueId: data.queueId,
+                      objectIds: selectedTraceIds,
+                      objectType: "TRACE",
+                    })
+                    .then(() => {
+                      setAddToQueueDialogOpen(false);
+                    });
+                }
+              })}
+            >
+              <DialogHeader>
+                <DialogTitle>Add to Annotation Queue</DialogTitle>
+                <DialogDescription>
+                  Select an annotation queue to add the selected traces to.
+                </DialogDescription>
+              </DialogHeader>
+              <FormField
+                control={form.control}
+                name="queueId"
+                render={({ field }) => (
+                  <FormItem>
+                    <Select
+                      onValueChange={field.onChange}
+                      defaultValue={field.value}
+                    >
+                      <FormControl>
+                        <SelectTrigger>
+                          <SelectValue placeholder="Select a queue" />
+                        </SelectTrigger>
+                      </FormControl>
+                      <SelectContent>
+                        {queues?.data?.map((queue) => (
+                          <SelectItem key={queue.id} value={queue.id}>
+                            {queue.name}
+                          </SelectItem>
+                        ))}
+                      </SelectContent>
+                    </Select>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <DialogFooter className="sm:justify-start">
+                <Button
+                  type="submit"
+                  loading={mutAddToQueue.isLoading}
+                  disabled={mutAddToQueue.isLoading}
+                >
+                  Add {selectedTraceIds.length} trace(s) to queue
+                </Button>
+              </DialogFooter>
+            </form>
+          </Form>
+        </DialogContent>
+      </Dialog>
     </>
   );
 }

web/src/components/table/use-cases/generations.tsx

@@ -284,7 +284,7 @@ export default function GenerationsTable({
         return typeof observationId === "string" &&
           typeof traceId === "string" ? (
           <TableLink
-            path={`/project/${projectId}/traces/${traceId}?observation=${observationId}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(traceId)}?observation=${encodeURIComponent(observationId)}`}
             value={observationId}
           />
         ) : null;

web/src/components/table/use-cases/scores.tsx

@@ -174,7 +174,7 @@ export default function ScoresTable({
         return typeof value === "string" ? (
           <>
             <TableLink
-              path={`/project/${projectId}/traces/${value}`}
+              path={`/project/${projectId}/traces/${encodeURIComponent(value)}`}
               value={value}
             />
           </>
@@ -194,7 +194,7 @@ export default function ScoresTable({
         const traceId = row.getValue("traceId") as ScoresTableRow["traceId"];
         return traceId && observationId ? (
           <TableLink
-            path={`/project/${projectId}/traces/${traceId}?observation=${observationId}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(traceId)}?observation=${encodeURIComponent(observationId)}`}
             value={observationId}
           />
         ) : undefined;
@@ -236,7 +236,7 @@ export default function ScoresTable({
         return typeof value === "string" ? (
           <>
             <TableLink
-              path={`/project/${projectId}/users/${value}`}
+              path={`/project/${projectId}/users/${encodeURIComponent(value)}`}
               value={value}
             />
           </>
@@ -396,7 +396,7 @@ export default function ScoresTable({
           ? score.value % 1 === 0
             ? String(score.value)
             : score.value.toFixed(4)
-          : score.stringValue ?? "",
+          : (score.stringValue ?? ""),
       author: {
         userId: score.authorUserId ?? undefined,
         image: score.authorUserImage ?? undefined,

web/src/components/table/use-cases/traces.tsx

@@ -291,7 +291,7 @@ export default function TracesTable({
         const value: TracesTableRow["id"] = row.getValue("id");
         return value && typeof value === "string" ? (
           <TableLink
-            path={`/project/${projectId}/traces/${value}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(value)}`}
             value={value}
           />
         ) : undefined;
@@ -692,7 +692,7 @@ export default function TracesTable({
 
   const rows = useMemo(() => {
     return traces.isSuccess
-      ? traceRowData?.rows?.map((trace) => {
+      ? (traceRowData?.rows?.map((trace) => {
           return {
             bookmarked: trace.bookmarked,
             id: trace.id,
@@ -721,7 +721,7 @@ export default function TracesTable({
             outputCost: trace.calculatedOutputCost ?? undefined,
             totalCost: trace.calculatedTotalCost ?? undefined,
           };
-        }) ?? []
+        }) ?? [])
       : [];
   }, [traces, traceRowData, scoreKeysAndProps]);
 

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,9 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type APIScore, type ScoreSource } from "@langfuse/shared";
+import {
+  AnnotationQueueObjectType,
+  type APIScore,
+  type ScoreSource,
+} from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -8,7 +12,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { Badge } from "@/src/components/ui/badge";
-import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -23,16 +26,31 @@ import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/compon
 import { AnnotateDrawer } from "@/src/features/scores/components/AnnotateDrawer";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { CommentDrawerButton } from "@/src/features/comments/CommentDrawerButton";
+import { cn } from "@/src/utils/tailwind";
+import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
+import { CreateNewAnnotationQueueItem } from "@/src/ee/features/annotation-queues/components/CreateNewAnnotationQueueItem";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
 import { calculateDisplayTotalCost } from "@/src/components/trace/lib/helpers";
 import { useMemo } from "react";
 
-export const ObservationPreview = (props: {
+export const ObservationPreview = ({
+  observations,
+  projectId,
+  scores,
+  currentObservationId,
+  traceId,
+  commentCounts,
+  viewType = "detailed",
+  className,
+}: {
   observations: Array<ObservationReturnType>;
   projectId: string;
   scores: APIScore[];
   currentObservationId: string;
   traceId: string;
   commentCounts?: Map<string, number>;
+  viewType?: "focused" | "detailed";
+  className?: string;
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -41,15 +59,16 @@ export const ObservationPreview = (props: {
   const [emptySelectedConfigIds, setEmptySelectedConfigIds] = useLocalStorage<
     string[]
   >("emptySelectedConfigIds", []);
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
 
   const observationWithInputAndOutput = api.observations.byId.useQuery({
-    observationId: props.currentObservationId,
-    traceId: props.traceId,
-    projectId: props.projectId,
+    observationId: currentObservationId,
+    traceId: traceId,
+    projectId: projectId,
   });
 
-  const preloadedObservation = props.observations.find(
-    (o) => o.id === props.currentObservationId,
+  const preloadedObservation = observations.find(
+    (o) => o.id === currentObservationId,
   );
 
   const thisCost = preloadedObservation
@@ -61,15 +80,15 @@ export const ObservationPreview = (props: {
   const totalCost = useMemo(
     () =>
       calculateDisplayTotalCost({
-        allObservations: props.observations,
-        rootObservationId: props.currentObservationId,
+        allObservations: observations,
+        rootObservationId: currentObservationId,
       }),
-    [props.observations, props.currentObservationId],
+    [observations, currentObservationId],
   );
 
   if (!preloadedObservation) return <div className="flex-1">Not found</div>;
 
-  const observationScores = props.scores.filter(
+  const observationScores = scores.filter(
     (s) => s.observationId === preloadedObservation.id,
   );
   const observationScoresBySource = observationScores.reduce((acc, score) => {
@@ -81,29 +100,36 @@ export const ObservationPreview = (props: {
   }, new Map<ScoreSource, APIScore[]>());
 
   return (
-    <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
-      <div className="flex flex-shrink-0 flex-row justify-end gap-2">
-        <Tabs
-          value={selectedTab}
-          onValueChange={setSelectedTab}
-          className="flex w-full justify-end border-b bg-background"
-        >
-          <TabsList className="bg-background py-0">
-            <TabsTrigger
-              value="preview"
-              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
-            >
-              Preview
-            </TabsTrigger>
-            <TabsTrigger
-              value="scores"
-              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
-            >
-              Scores
-            </TabsTrigger>
-          </TabsList>
-        </Tabs>
-      </div>
+    <Card
+      className={cn(
+        "col-span-2 flex max-h-full flex-col overflow-hidden",
+        className,
+      )}
+    >
+      {viewType === "detailed" && (
+        <div className="flex flex-shrink-0 flex-row justify-end gap-2">
+          <Tabs
+            value={selectedTab}
+            onValueChange={setSelectedTab}
+            className="flex w-full justify-end border-b bg-background"
+          >
+            <TabsList className="bg-background py-0">
+              <TabsTrigger
+                value="preview"
+                className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+              >
+                Preview
+              </TabsTrigger>
+              <TabsTrigger
+                value="scores"
+                className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+              >
+                Scores
+              </TabsTrigger>
+            </TabsList>
+          </Tabs>
+        </div>
+      )}
       <div className="flex w-full flex-col overflow-y-auto">
         <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
           <div className="flex flex-col gap-1">
@@ -116,107 +142,124 @@ export const ObservationPreview = (props: {
             <CardDescription className="flex gap-2">
               {preloadedObservation.startTime.toLocaleString()}
             </CardDescription>
-            <div className="flex flex-wrap gap-2">
-              {preloadedObservation.promptId ? (
-                <PromptBadge
-                  promptId={preloadedObservation.promptId}
-                  projectId={preloadedObservation.projectId}
-                />
-              ) : undefined}
-              {preloadedObservation.timeToFirstToken ? (
-                <Badge variant="outline">
-                  Time to first token:{" "}
-                  {formatIntervalSeconds(preloadedObservation.timeToFirstToken)}
-                </Badge>
-              ) : null}
-              {preloadedObservation.endTime ? (
-                <Badge variant="outline">
-                  Latency:{" "}
-                  {formatIntervalSeconds(
-                    (preloadedObservation.endTime.getTime() -
-                      preloadedObservation.startTime.getTime()) /
-                      1000,
-                  )}
-                </Badge>
-              ) : null}
-              {preloadedObservation.type === "GENERATION" && (
-                <Badge variant="outline">
-                  {preloadedObservation.promptTokens} prompt →{" "}
-                  {preloadedObservation.completionTokens} completion (∑{" "}
-                  {preloadedObservation.totalTokens})
-                </Badge>
-              )}
-              {preloadedObservation.version ? (
-                <Badge variant="outline">
-                  Version: {preloadedObservation.version}
-                </Badge>
-              ) : undefined}
-              {preloadedObservation.model ? (
-                <Badge variant="outline">{preloadedObservation.model}</Badge>
-              ) : null}
-              {thisCost ? (
-                <Badge variant="outline">
-                  {usdFormatter(thisCost.toNumber())}
-                </Badge>
-              ) : undefined}
-              {totalCost && totalCost !== thisCost ? (
-                <Badge variant="outline">
-                  ∑ {usdFormatter(totalCost.toNumber())}
-                </Badge>
-              ) : undefined}
+            {viewType === "detailed" && (
+              <div className="flex flex-wrap gap-2">
+                {preloadedObservation.promptId ? (
+                  <PromptBadge
+                    promptId={preloadedObservation.promptId}
+                    projectId={preloadedObservation.projectId}
+                  />
+                ) : undefined}
+                {preloadedObservation.timeToFirstToken ? (
+                  <Badge variant="outline">
+                    Time to first token:{" "}
+                    {formatIntervalSeconds(
+                      preloadedObservation.timeToFirstToken,
+                    )}
+                  </Badge>
+                ) : null}
+                {preloadedObservation.endTime ? (
+                  <Badge variant="outline">
+                    Latency:{" "}
+                    {formatIntervalSeconds(
+                      (preloadedObservation.endTime.getTime() -
+                        preloadedObservation.startTime.getTime()) /
+                        1000,
+                    )}
+                  </Badge>
+                ) : null}
+                {preloadedObservation.type === "GENERATION" && (
+                  <Badge variant="outline">
+                    {preloadedObservation.promptTokens} prompt →{" "}
+                    {preloadedObservation.completionTokens} completion (∑{" "}
+                    {preloadedObservation.totalTokens})
+                  </Badge>
+                )}
+                {preloadedObservation.version ? (
+                  <Badge variant="outline">
+                    Version: {preloadedObservation.version}
+                  </Badge>
+                ) : undefined}
+                {preloadedObservation.model ? (
+                  <Badge variant="outline">{preloadedObservation.model}</Badge>
+                ) : null}
+                {thisCost ? (
+                  <Badge variant="outline">
+                    {usdFormatter(thisCost.toNumber())}
+                  </Badge>
+                ) : undefined}
+                {totalCost && totalCost !== thisCost ? (
+                  <Badge variant="outline">
+                    ∑ {usdFormatter(totalCost.toNumber())}
+                  </Badge>
+                ) : undefined}
 
-              {preloadedObservation.modelParameters &&
-              typeof preloadedObservation.modelParameters === "object"
-                ? Object.entries(preloadedObservation.modelParameters)
-                    .filter(Boolean)
-                    .map(([key, value]) => (
-                      <Badge variant="outline" key={key}>
-                        {key}:{" "}
-                        {Object.prototype.toString.call(value) ===
-                        "[object Object]"
-                          ? JSON.stringify(value)
-                          : value?.toString()}
-                      </Badge>
-                    ))
-                : null}
-            </div>
-          </div>
-          <div className="flex flex-wrap gap-2">
-            <CommentDrawerButton
-              projectId={preloadedObservation.projectId}
-              objectId={preloadedObservation.id}
-              objectType="OBSERVATION"
-              count={props.commentCounts?.get(preloadedObservation.id)}
-            />
-            <AnnotateDrawer
-              projectId={props.projectId}
-              traceId={preloadedObservation.traceId}
-              observationId={preloadedObservation.id}
-              scores={props.scores}
-              emptySelectedConfigIds={emptySelectedConfigIds}
-              setEmptySelectedConfigIds={setEmptySelectedConfigIds}
-              type="observation"
-              key={"annotation-drawer" + preloadedObservation.id}
-            />
-            {observationWithInputAndOutput.data?.type === "GENERATION" && (
-              <JumpToPlaygroundButton
-                source="generation"
-                generation={observationWithInputAndOutput.data}
-                analyticsEventName="trace_detail:test_in_playground_button_click"
-              />
+                {preloadedObservation.modelParameters &&
+                typeof preloadedObservation.modelParameters === "object"
+                  ? Object.entries(preloadedObservation.modelParameters)
+                      .filter(Boolean)
+                      .map(([key, value]) => (
+                        <Badge variant="outline" key={key}>
+                          {key}:{" "}
+                          {Object.prototype.toString.call(value) ===
+                          "[object Object]"
+                            ? JSON.stringify(value)
+                            : value?.toString()}
+                        </Badge>
+                      ))
+                  : null}
+              </div>
             )}
-            {observationWithInputAndOutput.data ? (
-              <NewDatasetItemFromTrace
-                traceId={preloadedObservation.traceId}
-                observationId={preloadedObservation.id}
-                projectId={props.projectId}
-                input={observationWithInputAndOutput.data.input}
-                output={observationWithInputAndOutput.data.output}
-                metadata={preloadedObservation.metadata}
-                key={preloadedObservation.id}
-              />
-            ) : null}
           </div>
+          {viewType === "detailed" && (
+            <div className="flex flex-wrap gap-2">
+              <CommentDrawerButton
+                projectId={preloadedObservation.projectId}
+                objectId={preloadedObservation.id}
+                objectType="OBSERVATION"
+                count={commentCounts?.get(preloadedObservation.id)}
+              />
+              <div className="flex items-start">
+                <AnnotateDrawer
+                  key={"annotation-drawer" + preloadedObservation.id}
+                  projectId={projectId}
+                  traceId={traceId}
+                  observationId={preloadedObservation.id}
+                  scores={scores}
+                  emptySelectedConfigIds={emptySelectedConfigIds}
+                  setEmptySelectedConfigIds={setEmptySelectedConfigIds}
+                  type="observation"
+                  hasGroupedButton={hasEntitlement}
+                />
+                {hasEntitlement && (
+                  <CreateNewAnnotationQueueItem
+                    projectId={projectId}
+                    objectId={preloadedObservation.id}
+                    objectType={AnnotationQueueObjectType.OBSERVATION}
+                  />
+                )}
+              </div>
+
+              {observationWithInputAndOutput.data?.type === "GENERATION" && (
+                <JumpToPlaygroundButton
+                  source="generation"
+                  generation={observationWithInputAndOutput.data}
+                  analyticsEventName="trace_detail:test_in_playground_button_click"
+                />
+              )}
+              {observationWithInputAndOutput.data ? (
+                <NewDatasetItemFromTrace
+                  traceId={preloadedObservation.traceId}
+                  observationId={preloadedObservation.id}
+                  projectId={projectId}
+                  input={observationWithInputAndOutput.data.input}
+                  output={observationWithInputAndOutput.data.output}
+                  metadata={preloadedObservation.metadata}
+                  key={preloadedObservation.id}
+                />
+              ) : null}
+            </div>
+          )}
         </CardHeader>
         <CardContent className="flex flex-col gap-4">
           {selectedTab === "preview" && (
@@ -241,12 +284,14 @@ export const ObservationPreview = (props: {
                   json={observationWithInputAndOutput.data.metadata}
                 />
               ) : null}
-              <ScoresPreview itemScoresBySource={observationScoresBySource} />
+              {viewType === "detailed" && (
+                <ScoresPreview itemScoresBySource={observationScoresBySource} />
+              )}
             </>
           )}
           {selectedTab === "scores" && (
             <ScoresTable
-              projectId={props.projectId}
+              projectId={projectId}
               omittedFilter={["Observation ID"]}
               observationId={preloadedObservation.id}
               hiddenColumns={[

web/src/components/trace/ObservationTree.tsx

@@ -31,6 +31,7 @@ export const ObservationTree = (props: {
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics: boolean;
   showScores: boolean;
+  colorCodeMetrics: boolean;
   observationCommentCounts?: Map<string, number>;
   traceCommentCounts?: Map<string, number>;
   className?: string;
@@ -70,7 +71,7 @@ export const ObservationTree = (props: {
         setCurrentObservationId={props.setCurrentObservationId}
         showMetrics={props.showMetrics}
         showScores={props.showScores}
-        colorCodeMetrics={props.observations.length >= 3}
+        colorCodeMetrics={props.colorCodeMetrics}
         parentTotalCost={totalCost}
         parentTotalDuration={
           props.trace.latency ? props.trace.latency * 1000 : undefined

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,10 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type APIScore, type Trace, type ScoreSource } from "@langfuse/shared";
+import {
+  type APIScore,
+  type Trace,
+  type ScoreSource,
+  AnnotationQueueObjectType,
+} from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -13,7 +18,6 @@ import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
 import { formatIntervalSeconds } from "@/src/utils/dates";
-import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
@@ -21,6 +25,10 @@ import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/scores/components/AnnotateDrawer";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { CommentDrawerButton } from "@/src/features/comments/CommentDrawerButton";
+import { cn } from "@/src/utils/tailwind";
+import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
+import { CreateNewAnnotationQueueItem } from "@/src/ee/features/annotation-queues/components/CreateNewAnnotationQueueItem";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
 import { useMemo } from "react";
 import { usdFormatter } from "@/src/utils/numbers";
 import { calculateDisplayTotalCost } from "@/src/components/trace/lib/helpers";
@@ -30,11 +38,15 @@ export const TracePreview = ({
   observations,
   scores,
   commentCounts,
+  viewType = "detailed",
+  className,
 }: {
   trace: Trace & { latency?: number };
   observations: ObservationReturnType[];
   scores: APIScore[];
   commentCounts?: Map<string, number>;
+  viewType?: "detailed" | "focused";
+  className?: string;
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -43,6 +55,7 @@ export const TracePreview = ({
   const [emptySelectedConfigIds, setEmptySelectedConfigIds] = useLocalStorage<
     string[]
   >("emptySelectedConfigIds", []);
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
 
   const traceScores = scores.filter((s) => s.observationId === null);
   const traceScoresBySource = traceScores.reduce((acc, score) => {
@@ -62,29 +75,36 @@ export const TracePreview = ({
   );
 
   return (
-    <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
-      <div className="flex flex-shrink-0 flex-row justify-end gap-2">
-        <Tabs
-          value={selectedTab}
-          onValueChange={setSelectedTab}
-          className="flex w-full justify-end border-b bg-background"
-        >
-          <TabsList className="bg-background py-0">
-            <TabsTrigger
-              value="preview"
-              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
-            >
-              Preview
-            </TabsTrigger>
-            <TabsTrigger
-              value="scores"
-              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
-            >
-              Scores
-            </TabsTrigger>
-          </TabsList>
-        </Tabs>
-      </div>
+    <Card
+      className={cn(
+        "col-span-2 flex max-h-full flex-col overflow-hidden",
+        className,
+      )}
+    >
+      {viewType === "detailed" && (
+        <div className="flex flex-shrink-0 flex-row justify-end gap-2">
+          <Tabs
+            value={selectedTab}
+            onValueChange={setSelectedTab}
+            className="flex w-full justify-end border-b bg-background"
+          >
+            <TabsList className="bg-background py-0">
+              <TabsTrigger
+                value="preview"
+                className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+              >
+                Preview
+              </TabsTrigger>
+              <TabsTrigger
+                value="scores"
+                className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+              >
+                Scores
+              </TabsTrigger>
+            </TabsList>
+          </Tabs>
+        </div>
+      )}
       <div className="flex w-full flex-col overflow-y-auto">
         <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
           <div className="flex flex-col gap-1">
@@ -97,50 +117,64 @@ export const TracePreview = ({
             <CardDescription>
               {trace.timestamp.toLocaleString()}
             </CardDescription>
+            {viewType === "detailed" && (
+              <div className="flex flex-wrap gap-2">
+                {!!trace.latency && (
+                  <Badge variant="outline">
+                    {formatIntervalSeconds(trace.latency)}
+                  </Badge>
+                )}
+                <AggUsageBadge observations={observations} />
+                {!!trace.release && (
+                  <Badge variant="outline">Release: {trace.release}</Badge>
+                )}
+                {!!trace.version && (
+                  <Badge variant="outline">Version: {trace.version}</Badge>
+                )}
+                {totalCost && (
+                  <Badge variant="outline">
+                    ∑ {usdFormatter(totalCost.toNumber())}
+                  </Badge>
+                )}
+              </div>
+            )}
+          </div>
+          {viewType === "detailed" && (
             <div className="flex flex-wrap gap-2">
-              {!!trace.latency && (
-                <Badge variant="outline">
-                  {formatIntervalSeconds(trace.latency)}
-                </Badge>
-              )}
-              <AggUsageBadge observations={observations} />
-              {!!trace.release && (
-                <Badge variant="outline">Release: {trace.release}</Badge>
-              )}
-              {!!trace.version && (
-                <Badge variant="outline">Version: {trace.version}</Badge>
-              )}
-              {totalCost && (
-                <Badge variant="outline">
-                  ∑ {usdFormatter(totalCost.toNumber())}
-                </Badge>
-              )}
+              <CommentDrawerButton
+                projectId={trace.projectId}
+                objectId={trace.id}
+                objectType="TRACE"
+                count={commentCounts?.get(trace.id)}
+              />
+              <div className="flex items-start">
+                <AnnotateDrawer
+                  key={"annotation-drawer" + trace.id}
+                  projectId={trace.projectId}
+                  traceId={trace.id}
+                  scores={scores}
+                  emptySelectedConfigIds={emptySelectedConfigIds}
+                  setEmptySelectedConfigIds={setEmptySelectedConfigIds}
+                  hasGroupedButton={hasEntitlement}
+                />
+                {hasEntitlement && (
+                  <CreateNewAnnotationQueueItem
+                    projectId={trace.projectId}
+                    objectId={trace.id}
+                    objectType={AnnotationQueueObjectType.TRACE}
+                  />
+                )}
+              </div>
+              <NewDatasetItemFromTrace
+                traceId={trace.id}
+                projectId={trace.projectId}
+                input={trace.input}
+                output={trace.output}
+                metadata={trace.metadata}
+                key={trace.id}
+              />
             </div>
-          </div>
-          <div className="flex flex-wrap gap-2">
-            <CommentDrawerButton
-              projectId={trace.projectId}
-              objectId={trace.id}
-              objectType="TRACE"
-              count={commentCounts?.get(trace.id)}
-            />
-            <AnnotateDrawer
-              projectId={trace.projectId}
-              traceId={trace.id}
-              scores={scores}
-              emptySelectedConfigIds={emptySelectedConfigIds}
-              setEmptySelectedConfigIds={setEmptySelectedConfigIds}
-              key={"annotation-drawer" + trace.id}
-            />
-            <NewDatasetItemFromTrace
-              traceId={trace.id}
-              projectId={trace.projectId}
-              input={trace.input}
-              output={trace.output}
-              metadata={trace.metadata}
-              key={trace.id}
-            />
-          </div>
+          )}
         </CardHeader>
         <CardContent className="flex flex-col gap-4">
           {selectedTab === "preview" && (
@@ -155,7 +189,9 @@ export const TracePreview = ({
                 title="Metadata"
                 json={trace.metadata}
               />
-              <ScoresPreview itemScoresBySource={traceScoresBySource} />
+              {viewType === "detailed" && (
+                <ScoresPreview itemScoresBySource={traceScoresBySource} />
+              )}
             </>
           )}
           {selectedTab === "scores" && (

web/src/components/trace/index.tsx

@@ -24,6 +24,7 @@ import {
   ChevronsUpDown,
   ListTree,
   Network,
+  Percent,
 } from "lucide-react";
 import { usdFormatter } from "@/src/utils/numbers";
 import { useCallback, useState } from "react";
@@ -41,7 +42,11 @@ export function Trace(props: {
   trace: Trace;
   scores: APIScore[];
   projectId: string;
+  viewType?: "detailed" | "focused";
+  isValidObservationId?: boolean;
 }) {
+  const viewType = props.viewType ?? "detailed";
+  const isValidObservationId = props.isValidObservationId ?? true;
   const capture = usePostHogClientCapture();
   const [currentObservationId, setCurrentObservationId] = useQueryParam(
     "observation",
@@ -53,6 +58,10 @@ export function Trace(props: {
     "scoresOnObservationTree",
     true,
   );
+  const [
+    colorCodeMetricsOnObservationTree,
+    setColorCodeMetricsOnObservationTree,
+  ] = useLocalStorage("colorCodeMetricsOnObservationTree", true);
 
   const [collapsedObservations, setCollapsedObservations] = useState<string[]>(
     [],
@@ -151,8 +160,9 @@ export function Trace(props: {
             observations={props.observations}
             scores={props.scores}
             commentCounts={traceCommentCounts.data}
+            viewType={viewType}
           />
-        ) : (
+        ) : isValidObservationId ? (
           <ObservationPreview
             observations={props.observations}
             scores={props.scores}
@@ -160,8 +170,9 @@ export function Trace(props: {
             currentObservationId={currentObservationId}
             traceId={props.trace.id}
             commentCounts={observationCommentCounts.data}
+            viewType={viewType}
           />
-        )}
+        ) : null}
       </div>
       <div className="md:col-span-2 md:flex md:h-full md:flex-col md:overflow-hidden">
         <div className="mb-2 flex flex-shrink-0 flex-row justify-end gap-2">
@@ -195,6 +206,14 @@ export function Trace(props: {
               <ChevronsUpDown className="h-4 w-4" />
             )}
           </Toggle>
+          <Toggle
+            pressed={colorCodeMetricsOnObservationTree}
+            onPressedChange={(e) => setColorCodeMetricsOnObservationTree(e)}
+            size="xs"
+            title="Color code metrics (>50% yellow, >75% red)"
+          >
+            <Percent className="h-4 w-4" />
+          </Toggle>
         </div>
 
         <ObservationTree
@@ -209,6 +228,7 @@ export function Trace(props: {
           setCurrentObservationId={setCurrentObservationId}
           showMetrics={metricsOnObservationTree}
           showScores={scoresOnObservationTree}
+          colorCodeMetrics={colorCodeMetricsOnObservationTree}
           observationCommentCounts={observationCommentCounts.data}
           traceCommentCounts={traceCommentCounts.data}
           className="flex w-full flex-col overflow-y-auto"

web/src/components/ui/MarkdownViewer.tsx

@@ -108,14 +108,14 @@ function MarkdownRenderer({
           if (isChecklist(children))
             return <ul className="list-none">{children}</ul>;
 
-          return <ul className="list-outside list-disc pl-4">{children}</ul>;
+          return <ul className="list-inside list-disc">{children}</ul>;
         },
         ol({ children }) {
-          return <ol className="list-outside list-decimal pl-4">{children}</ol>;
+          return <ol className="list-inside list-decimal">{children}</ol>;
         },
         li({ children }) {
           return (
-            <li className="mb-1 list-item">
+            <li className="mt-1 [&>ol]:pl-4 [&>ul]:pl-4">
               {transformListItemChildren(children)}
             </li>
           );

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.82.0";
+export const VERSION = "v2.85.0";

web/src/ee/features/annotation-queues/components/AnnotationQueueItemPage.tsx

@@ -0,0 +1,415 @@
+import { Trace } from "@/src/components/trace";
+import { ObservationPreview } from "@/src/components/trace/ObservationPreview";
+import { TracePreview } from "@/src/components/trace/TracePreview";
+import { Button } from "@/src/components/ui/button";
+import { Card } from "@/src/components/ui/card";
+import {
+  ResizableHandle,
+  ResizablePanel,
+  ResizablePanelGroup,
+} from "@/src/components/ui/resizable";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import useSessionStorage from "@/src/components/useSessionStorage";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { AnnotateDrawerContent } from "@/src/features/scores/components/AnnotateDrawerContent";
+import { api } from "@/src/utils/api";
+import { type RouterOutput } from "@/src/utils/types";
+import {
+  type AnnotationQueueItem,
+  AnnotationQueueObjectType,
+  AnnotationQueueStatus,
+  isPresent,
+  type ValidatedScoreConfig,
+} from "@langfuse/shared";
+import {
+  ArrowLeft,
+  ArrowRight,
+  SearchXIcon,
+  TriangleAlertIcon,
+} from "lucide-react";
+import { useSession } from "next-auth/react";
+import { useRouter } from "next/router";
+import { useEffect, useMemo, useState } from "react";
+import { StringParam, useQueryParam } from "use-query-params";
+
+const AnnotateIOView = ({
+  item,
+  configs,
+  view,
+}: {
+  item: AnnotationQueueItem & {
+    parentTraceId?: string | null;
+    lockedByUser: { name: string | null | undefined } | null;
+  };
+  configs: ValidatedScoreConfig[];
+  view: "showTree" | "hideTree";
+}) => {
+  const router = useRouter();
+  const session = useSession();
+  const traceId = item.parentTraceId ?? item.objectId;
+  const projectId = router.query.projectId as string;
+  const [panelSize, setPanelSize] = useSessionStorage(
+    `annotationQueuePanelSize-${projectId}`,
+    65,
+  );
+
+  const [currentObservationId, setCurrentObservationId] = useQueryParam(
+    "observation",
+    StringParam,
+  );
+  useEffect(() => {
+    if (
+      view === "showTree" &&
+      item.objectType === AnnotationQueueObjectType.OBSERVATION
+    ) {
+      setCurrentObservationId(item.objectId);
+    } else setCurrentObservationId(undefined);
+  }, [view, item, setCurrentObservationId]);
+
+  const isLockedByOtherUser = item.lockedByUserId !== session.data?.user?.id;
+
+  const trace = api.traces.byIdWithObservationsAndScores.useQuery(
+    { traceId, projectId },
+    {
+      retry(failureCount, error) {
+        if (error.data?.code === "UNAUTHORIZED") return false;
+        return failureCount < 3;
+      },
+    },
+  );
+
+  const emptySelectedConfigIds = useMemo(() => {
+    return configs.map((c) => c.id);
+  }, [configs]);
+
+  if (trace.isLoading || !trace.data) return <div>Loading...</div>;
+
+  let isValidObservationId = false;
+
+  if (
+    currentObservationId &&
+    trace.data.observations.some(({ id }) => id === currentObservationId)
+  ) {
+    isValidObservationId = true;
+  }
+
+  return (
+    <ResizablePanelGroup
+      direction="horizontal"
+      className="h-full overflow-hidden"
+      onLayout={(sizes) => {
+        setPanelSize(sizes[0]);
+      }}
+    >
+      <ResizablePanel
+        className="col-span-1 h-full !overflow-y-auto"
+        minSize={30}
+        defaultSize={panelSize}
+      >
+        {view === "hideTree" ? (
+          item.objectType === AnnotationQueueObjectType.TRACE ? (
+            <TracePreview
+              key={trace.data.id}
+              trace={trace.data}
+              scores={trace.data.scores}
+              observations={trace.data.observations}
+              viewType="focused"
+              className="h-full"
+            />
+          ) : (
+            <ObservationPreview
+              observations={trace.data.observations}
+              scores={trace.data.scores}
+              projectId={item.projectId}
+              currentObservationId={item.objectId}
+              traceId={traceId}
+              viewType="focused"
+              className="h-full"
+            />
+          )
+        ) : (
+          <Card className="col-span-2 flex h-full flex-col overflow-hidden p-2">
+            <div className="overflow-x-auto md:overflow-hidden">
+              <Trace
+                key={trace.data.id}
+                trace={trace.data}
+                scores={trace.data.scores}
+                projectId={trace.data.projectId}
+                observations={trace.data.observations}
+                viewType="focused"
+                isValidObservationId={isValidObservationId}
+              />
+            </div>
+          </Card>
+        )}
+      </ResizablePanel>
+      <ResizableHandle withHandle className="ml-4 bg-transparent" />
+      <ResizablePanel
+        className="col-span-1 h-full md:flex md:flex-col md:overflow-hidden"
+        minSize={30}
+      >
+        <Card className="col-span-2 flex h-full flex-col overflow-hidden">
+          <AnnotateDrawerContent
+            key={"annotation-drawer-content" + item.objectId}
+            traceId={traceId}
+            scores={trace.data?.scores ?? []}
+            observationId={item.parentTraceId ? item.objectId : undefined}
+            configs={configs}
+            emptySelectedConfigIds={emptySelectedConfigIds}
+            setEmptySelectedConfigIds={() => {}}
+            projectId={item.projectId}
+            type={item.objectType.toLowerCase() as "trace" | "observation"}
+            isSelectHidden
+            queueId={item.queueId}
+            actionButtons={
+              isLockedByOtherUser && isPresent(item.lockedByUser?.name) ? (
+                <div className="flex items-center justify-center rounded-sm border border-dark-red bg-light-red p-1">
+                  <TriangleAlertIcon className="mr-1 h-4 w-4 text-dark-red" />
+                  <span className="text-xs text-dark-red">
+                    Currently edited by {item.lockedByUser.name}
+                  </span>
+                </div>
+              ) : undefined
+            }
+          />
+        </Card>
+      </ResizablePanel>
+    </ResizablePanelGroup>
+  );
+};
+
+export const AnnotationQueueItemPage: React.FC<{
+  annotationQueueId: string;
+  projectId: string;
+  view: "showTree" | "hideTree";
+  queryItemId?: string;
+}> = ({ annotationQueueId, projectId, view, queryItemId }) => {
+  const router = useRouter();
+  const isSingleItem = router.query.singleItem === "true";
+  const [nextItemData, setNextItemData] = useState<
+    RouterOutput["annotationQueues"]["fetchAndLockNext"] | null
+  >(null);
+  const [seenItemIds, setSeenItemIds] = useState<string[]>([]);
+  const [progressIndex, setProgressIndex] = useState(0);
+
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const itemId = isSingleItem ? queryItemId : seenItemIds[progressIndex];
+
+  const seenItemData = api.annotationQueueItems.byId.useQuery(
+    { projectId, itemId: itemId as string },
+    { enabled: !!itemId, refetchOnMount: false },
+  );
+
+  const fetchAndLockNextMutation =
+    api.annotationQueues.fetchAndLockNext.useMutation();
+
+  useEffect(() => {
+    async function fetchNextItem() {
+      if (!itemId && !isSingleItem) {
+        const nextItem = await fetchAndLockNextMutation.mutateAsync({
+          queueId: annotationQueueId,
+          projectId,
+          seenItemIds,
+        });
+        setNextItemData(nextItem);
+      }
+    }
+    fetchNextItem();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const queueData = api.annotationQueues.byId.useQuery(
+    {
+      queueId: annotationQueueId,
+      projectId,
+    },
+    {
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+    },
+  );
+
+  const unseenPendingItemCount =
+    api.annotationQueueItems.unseenPendingItemCountByQueueId.useQuery(
+      {
+        queueId: annotationQueueId,
+        projectId,
+        seenItemIds,
+      },
+      { refetchOnWindowFocus: false },
+    );
+
+  const utils = api.useUtils();
+  const completeMutation = api.annotationQueueItems.complete.useMutation({
+    onSuccess: async () => {
+      utils.annotationQueueItems.invalidate();
+      showSuccessToast({
+        title: "Item marked as complete",
+        description: "The item is successfully marked as complete.",
+      });
+      if (isSingleItem) {
+        return;
+      }
+
+      if (progressIndex >= seenItemIds.length - 1) {
+        const nextItem = await fetchAndLockNextMutation.mutateAsync({
+          queueId: annotationQueueId,
+          projectId,
+          seenItemIds,
+        });
+        setNextItemData(nextItem);
+      }
+
+      if (progressIndex + 1 < totalItems) {
+        setProgressIndex(Math.max(progressIndex + 1, 0));
+      }
+    },
+  });
+
+  const totalItems = useMemo(() => {
+    return seenItemIds.length + (unseenPendingItemCount.data ?? 0);
+  }, [unseenPendingItemCount.data, seenItemIds.length]);
+
+  const configs = queueData.data?.scoreConfigs ?? [];
+
+  const relevantItem = useMemo(() => {
+    if (isSingleItem) return seenItemData.data;
+    else
+      return progressIndex < seenItemIds.length
+        ? seenItemData.data
+        : nextItemData;
+  }, [
+    progressIndex,
+    seenItemIds.length,
+    seenItemData.data,
+    nextItemData,
+    isSingleItem,
+  ]);
+
+  useEffect(() => {
+    if (relevantItem && router.query.itemId !== relevantItem.id) {
+      router.push(
+        {
+          pathname: `/project/${projectId}/annotation-queues/${annotationQueueId}/items/${relevantItem.id}`,
+        },
+        undefined,
+      );
+    }
+  }, [relevantItem, router, projectId, annotationQueueId]);
+
+  useEffect(() => {
+    if (
+      relevantItem &&
+      !seenItemIds.includes(relevantItem.id) &&
+      !isSingleItem
+    ) {
+      setSeenItemIds((prev) => [...prev, relevantItem.id]);
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [relevantItem]);
+
+  if (
+    (seenItemData.isLoading && itemId) ||
+    (fetchAndLockNextMutation.isLoading && !itemId) ||
+    unseenPendingItemCount.isLoading
+  ) {
+    return <Skeleton className="h-full w-full" />;
+  }
+
+  if (!relevantItem && !(itemId && seenItemIds.includes(itemId))) {
+    return <div>No more items left to annotate!</div>;
+  }
+
+  const isNextItemAvailable = totalItems > progressIndex + 1;
+
+  return (
+    <div className="grid h-full grid-rows-[1fr,auto] gap-4 overflow-hidden">
+      {relevantItem ? (
+        <AnnotateIOView item={relevantItem} configs={configs} view={view} />
+      ) : (
+        <Card className="flex h-full w-full flex-col items-center justify-center overflow-hidden">
+          <SearchXIcon className="mb-2 h-8 w-8 text-muted-foreground" />
+          <span className="max-w-96 text-wrap text-sm text-muted-foreground">
+            Item has been <strong>deleted from annotation queue</strong>.
+            Previously added scores and underlying reference trace are
+            unaffected by this action.
+          </span>
+        </Card>
+      )}
+      <div className="grid h-full w-full grid-cols-1 justify-end gap-2 sm:grid-cols-[auto,min-content]">
+        {!isSingleItem && (
+          <div className="flex max-h-10 flex-row gap-2">
+            <span className="grid h-9 min-w-16 items-center rounded-md bg-muted p-1 text-center text-sm">
+              {progressIndex + 1} / {totalItems}
+            </span>
+            <Button
+              onClick={() => {
+                setProgressIndex(progressIndex - 1);
+              }}
+              variant="outline"
+              disabled={progressIndex === 0 || !hasAccess}
+              size="lg"
+              className="px-4"
+            >
+              <ArrowLeft className="mr-1 h-4 w-4" />
+              Back
+            </Button>
+          </div>
+        )}
+        <div className="flex w-full min-w-[265px] justify-end gap-2">
+          {!isSingleItem && (
+            <Button
+              onClick={async () => {
+                if (progressIndex >= seenItemIds.length - 1) {
+                  const nextItem = await fetchAndLockNextMutation.mutateAsync({
+                    queueId: annotationQueueId,
+                    projectId,
+                    seenItemIds,
+                  });
+                  setNextItemData(nextItem);
+                }
+                setProgressIndex(Math.max(progressIndex + 1, 0));
+              }}
+              disabled={!isNextItemAvailable || !hasAccess} // Disable button during loading
+              size="lg"
+              className={`px-4 ${!relevantItem ? "w-full" : ""}`}
+              variant="outline"
+            >
+              {relevantItem?.status === AnnotationQueueStatus.PENDING
+                ? "Skip"
+                : "Next"}
+              <ArrowRight className="ml-1 h-4 w-4" />
+            </Button>
+          )}
+          {!!relevantItem &&
+            (relevantItem.status === AnnotationQueueStatus.PENDING ? (
+              <Button
+                onClick={async () => {
+                  await completeMutation.mutateAsync({
+                    itemId: relevantItem.id,
+                    projectId,
+                  });
+                }}
+                size="lg"
+                className="w-full"
+                disabled={completeMutation.isLoading || !hasAccess}
+              >
+                {isSingleItem || progressIndex + 1 === totalItems
+                  ? "Complete"
+                  : "Complete + Next"}
+              </Button>
+            ) : (
+              <div className="text-dark-gree inline-flex h-9 w-full items-center justify-center rounded-md border border-dark-green bg-light-green px-8 text-sm font-medium">
+                Completed
+              </div>
+            ))}
+        </div>
+      </div>
+    </div>
+  );
+};

web/src/ee/features/annotation-queues/components/AnnotationQueueItemsTable.tsx

@@ -0,0 +1,409 @@
+import { DataTable } from "@/src/components/table/data-table";
+import TableLink from "@/src/components/table/table-link";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { api } from "@/src/utils/api";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { type AnnotationQueueStatus } from "@langfuse/shared";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import { ChevronDown, ListTree, Trash } from "lucide-react";
+import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
+import {
+  Avatar,
+  AvatarFallback,
+  AvatarImage,
+} from "@/src/components/ui/avatar";
+import { type RouterOutput } from "@/src/utils/types";
+import { type RowSelectionState } from "@tanstack/react-table";
+import { useState } from "react";
+import {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+} from "@/src/components/ui/dropdown-menu";
+import { Button } from "@/src/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/src/components/ui/dialog";
+import { Checkbox } from "@/src/components/ui/checkbox";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { StatusBadge } from "@/src/components/layouts/status-badge";
+
+const QueueItemTableMultiSelectAction = ({
+  selectedItemIds,
+  projectId,
+  onDeleteSuccess,
+}: {
+  selectedItemIds: string[];
+  projectId: string;
+  onDeleteSuccess: () => void;
+}) => {
+  const utils = api.useUtils();
+  const [open, setOpen] = useState(false);
+
+  const hasDeleteAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const mutDeleteItems = api.annotationQueueItems.deleteMany.useMutation({
+    onSuccess: () => {
+      onDeleteSuccess();
+      void utils.annotationQueueItems.itemsByQueueId.invalidate();
+    },
+  });
+
+  return (
+    <>
+      <DropdownMenu>
+        <DropdownMenuTrigger asChild>
+          <Button disabled={selectedItemIds.length < 1}>
+            Actions ({selectedItemIds.length} selected)
+            <ChevronDown className="h-5 w-5" />
+          </Button>
+        </DropdownMenuTrigger>
+        <DropdownMenuContent>
+          <DropdownMenuItem
+            disabled={!hasDeleteAccess}
+            onClick={() => {
+              setOpen(true);
+            }}
+          >
+            <Trash className="mr-2 h-4 w-4" />
+            <span>Delete</span>
+          </DropdownMenuItem>
+        </DropdownMenuContent>
+      </DropdownMenu>
+      <Dialog
+        open={open}
+        onOpenChange={(isOpen) => {
+          if (!isOpen) {
+            setOpen(false);
+          }
+        }}
+      >
+        <DialogContent className="sm:max-w-md">
+          <DialogHeader>
+            <DialogTitle>Delete queue items</DialogTitle>
+            <DialogDescription>
+              This action cannot be undone and removes the selected annotation
+              queue item(s), but
+              <strong> does not delete associated scores.</strong>
+            </DialogDescription>
+          </DialogHeader>
+          <DialogFooter className="sm:justify-start">
+            <Button
+              type="button"
+              variant="destructive"
+              loading={mutDeleteItems.isLoading}
+              disabled={mutDeleteItems.isLoading}
+              onClick={() => {
+                void mutDeleteItems
+                  .mutateAsync({
+                    itemIds: selectedItemIds,
+                    projectId,
+                  })
+                  .then(() => {
+                    setOpen(false);
+                  });
+              }}
+            >
+              Delete {selectedItemIds.length} item(s)
+            </Button>
+          </DialogFooter>
+        </DialogContent>
+      </Dialog>
+    </>
+  );
+};
+
+export type QueueItemRowData = {
+  id: string;
+  source: {
+    traceId: string;
+    observationId?: string;
+  };
+  status: AnnotationQueueStatus;
+  completedAt: string;
+  annotatorUser: {
+    userId?: string;
+    userName?: string;
+    image?: string;
+  };
+};
+
+export function AnnotationQueueItemsTable({
+  projectId,
+  queueId,
+}: {
+  projectId: string;
+  queueId: string;
+}) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
+
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage("queueItems", "s");
+  const items = api.annotationQueueItems.itemsByQueueId.useQuery({
+    projectId,
+    queueId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  const columns: LangfuseColumnDef<QueueItemRowData>[] = [
+    {
+      id: "select",
+      accessorKey: "select",
+      size: 30,
+      isPinned: true,
+      header: ({ table }) => {
+        return (
+          <Checkbox
+            checked={
+              table.getIsAllPageRowsSelected()
+                ? true
+                : table.getIsSomePageRowsSelected()
+                  ? "indeterminate"
+                  : false
+            }
+            onCheckedChange={(value) => {
+              table.toggleAllPageRowsSelected(!!value);
+              if (!value) {
+                setSelectedRows({});
+              }
+            }}
+            aria-label="Select all"
+            className="mt-1 opacity-60 data-[state=checked]:mt-[6px] data-[state=indeterminate]:mt-[6px]"
+          />
+        );
+      },
+      cell: ({ row }) => {
+        return (
+          <Checkbox
+            checked={row.getIsSelected()}
+            onCheckedChange={(value) => row.toggleSelected(!!value)}
+            aria-label="Select row"
+            className="mt-1 opacity-60 data-[state=checked]:mt-[5px]"
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "id",
+      header: "Id",
+      id: "id",
+      size: 70,
+      isPinned: true,
+      cell: ({ row }) => {
+        const id: QueueItemRowData["id"] = row.getValue("id");
+        return (
+          <TableLink
+            path={`/project/${projectId}/annotation-queues/${queueId}/items/${id}?singleItem=true`}
+            value={id}
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "source",
+      header: "Source",
+      headerTooltip: {
+        description:
+          "Link to the source trace based on which this item was added",
+      },
+      id: "source",
+      size: 50,
+      cell: ({ row }) => {
+        const source: QueueItemRowData["source"] = row.getValue("source");
+        if (!source) return null;
+
+        if (!!source.observationId) {
+          return (
+            <TableLink
+              path={`/project/${projectId}/traces/${source.traceId}?observation=${source.observationId}`}
+              value={source.observationId}
+              icon={<ListTree className="h-4 w-4" />}
+            />
+          );
+        } else {
+          return (
+            <TableLink
+              path={`/project/${projectId}/traces/${source.traceId}`}
+              value={source.traceId}
+              icon={<ListTree className="h-4 w-4" />}
+            />
+          );
+        }
+      },
+    },
+    {
+      accessorKey: "status",
+      header: "Status",
+      id: "status",
+      size: 60,
+      cell: ({ row }) => {
+        const status: QueueItemRowData["status"] = row.getValue("status");
+        return (
+          <StatusBadge
+            className="capitalize"
+            type={status.toLowerCase()}
+            isLive={false}
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "completedAt",
+      header: "Completed At",
+      id: "completedAt",
+      defaultHidden: true,
+      enableHiding: true,
+      size: 60,
+    },
+    {
+      accessorKey: "annotatorUser",
+      header: "Completed by",
+      id: "annotatorUser",
+      enableHiding: true,
+      size: 80,
+      cell: ({ row }) => {
+        const annotatorUser: QueueItemRowData["annotatorUser"] =
+          row.getValue("annotatorUser");
+        if (!annotatorUser || !annotatorUser.userId) return null;
+
+        const { userId, userName, image } = annotatorUser;
+        return (
+          <div className="flex items-center space-x-2">
+            <Avatar className="h-7 w-7">
+              <AvatarImage
+                src={image ?? undefined}
+                alt={userName ?? "User Avatar"}
+              />
+              <AvatarFallback>
+                {userName
+                  ? userName
+                      .split(" ")
+                      .map((word) => word[0])
+                      .slice(0, 2)
+                      .concat("")
+                  : null}
+              </AvatarFallback>
+            </Avatar>
+            <span>{userName ?? userId}</span>
+          </div>
+        );
+      },
+    },
+  ];
+
+  const convertToTableRow = (
+    item: RouterOutput["annotationQueueItems"]["itemsByQueueId"]["queueItems"][number],
+  ): QueueItemRowData => {
+    return {
+      id: item.id,
+      completedAt: item.completedAt?.toLocaleString() ?? "",
+      status: item.status,
+      source:
+        item.objectType === "OBSERVATION"
+          ? {
+              traceId: item.parentTraceId ?? "",
+              observationId: item.objectId,
+            }
+          : {
+              traceId: item.objectId,
+            },
+      annotatorUser: {
+        userId: item.annotatorUserId ?? undefined,
+        userName: item.annotatorUserName ?? undefined,
+        image: item.annotatorUserImage ?? undefined,
+      },
+    };
+  };
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<QueueItemRowData>(
+      `queueItemsColumnVisibility-${projectId}`,
+      columns,
+    );
+
+  const [columnOrder, setColumnOrder] = useColumnOrder<QueueItemRowData>(
+    "queueItemsColumnOrder",
+    columns,
+  );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        columnOrder={columnOrder}
+        setColumnOrder={setColumnOrder}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+        actionButtons={[
+          Object.keys(selectedRows).filter((itemId) =>
+            items.data?.queueItems.map((item) => item.id).includes(itemId),
+          ).length > 0 ? (
+            <QueueItemTableMultiSelectAction
+              // Exclude items that are not in the current page
+              selectedItemIds={Object.keys(selectedRows).filter((itemId) =>
+                items.data?.queueItems.map((item) => item.id).includes(itemId),
+              )}
+              projectId={projectId}
+              onDeleteSuccess={() => {
+                setSelectedRows({});
+              }}
+            />
+          ) : null,
+        ]}
+      />
+      <DataTable
+        columns={columns}
+        data={
+          items.isLoading
+            ? { isLoading: true, isError: false }
+            : items.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: items.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: items.data.queueItems.map((item) =>
+                    convertToTableRow(item),
+                  ),
+                }
+        }
+        help={{
+          description:
+            "Add traces and/or observations to your annotation queue to have them annotated by your team across predefined dimensions.",
+          href: "https://langfuse.com/docs/scores/model-based-evals",
+        }}
+        pagination={{
+          totalCount: items.data?.totalItems ?? null,
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        rowSelection={selectedRows}
+        setRowSelection={setSelectedRows}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+        columnOrder={columnOrder}
+        onColumnOrderChange={setColumnOrder}
+        rowHeight={rowHeight}
+      />
+    </>
+  );
+}

web/src/ee/features/annotation-queues/components/AnnotationQueuesItem.tsx

@@ -0,0 +1,93 @@
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import Header from "@/src/components/layouts/header";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import useSessionStorage from "@/src/components/useSessionStorage";
+import { SupportOrUpgradePage } from "@/src/ee/features/billing/components/SupportOrUpgradePage";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { AnnotationQueueItemPage } from "@/src/ee/features/annotation-queues/components/AnnotationQueueItemPage";
+import { api } from "@/src/utils/api";
+import { Goal, Network } from "lucide-react";
+
+export const AnnotationQueuesItem = ({
+  annotationQueueId,
+  projectId,
+  itemId,
+}: {
+  annotationQueueId: string;
+  projectId: string;
+  itemId?: string;
+}) => {
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:read",
+  });
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
+
+  const queue = api.annotationQueues.byId.useQuery(
+    {
+      queueId: annotationQueueId,
+      projectId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+
+  const [view, setView] = useSessionStorage<"hideTree" | "showTree">(
+    `annotationQueueView-${projectId}`,
+    "hideTree",
+  );
+
+  if (!hasAccess || !hasEntitlement) return <SupportOrUpgradePage />;
+
+  return (
+    <FullScreenPage>
+      <>
+        <Header
+          title={`${queue.data?.name ?? annotationQueueId}`}
+          breadcrumb={[
+            {
+              name: "Annotation Queues",
+              href: `/project/${projectId}/annotation-queues`,
+            },
+            {
+              name: queue.data?.name ?? annotationQueueId,
+              href: `/project/${projectId}/annotation-queues/${annotationQueueId}`,
+            },
+          ]}
+          actionButtons={
+            <Tabs
+              value={view}
+              onValueChange={(view: string) => {
+                setView(view as "hideTree" | "showTree");
+              }}
+            >
+              <TabsList>
+                <TabsTrigger value="hideTree">
+                  <Goal className="mr-1 h-4 w-4"></Goal>
+                  Focused
+                </TabsTrigger>
+                <TabsTrigger value="showTree">
+                  <Network className="mr-1 h-4 w-4"></Network>
+                  Detailed
+                </TabsTrigger>
+              </TabsList>
+            </Tabs>
+          }
+        />
+        <AnnotationQueueItemPage
+          projectId={projectId}
+          annotationQueueId={annotationQueueId}
+          view={view}
+          queryItemId={itemId}
+        />
+      </>
+    </FullScreenPage>
+  );
+};

web/src/ee/features/annotation-queues/components/AnnotationQueuesTable.tsx

@@ -0,0 +1,273 @@
+import { DataTable } from "@/src/components/table/data-table";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { api } from "@/src/utils/api";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+import { type RouterOutput } from "@/src/utils/types";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
+import { CreateOrEditAnnotationQueueButton } from "@/src/ee/features/annotation-queues/components/CreateOrEditAnnotationQueueButton";
+import { type ScoreDataType } from "@langfuse/shared";
+import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuLabel,
+  DropdownMenuTrigger,
+} from "@/src/components/ui/dropdown-menu";
+import { ClipboardPen, Lock, MoreVertical } from "lucide-react";
+import { Button } from "@/src/components/ui/button";
+import { cn } from "@/src/utils/tailwind";
+import TableLink from "@/src/components/table/table-link";
+import Link from "next/link";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { DeleteAnnotationQueueButton } from "@/src/ee/features/annotation-queues/components/DeleteAnnotationQueueButton";
+
+type RowData = {
+  key: {
+    id: string;
+    name: string;
+  };
+  description?: string;
+  countCompletedItems: number;
+  countPendingItems: number;
+  scoreConfigs: { id: string; name: string; dataType: ScoreDataType }[];
+  createdAt: string;
+};
+
+export function AnnotationQueuesTable({ projectId }: { projectId: string }) {
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "annotationQueues",
+    "s",
+  );
+
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const queues = api.annotationQueues.all.useQuery({
+    projectId: projectId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "annotationQueues:CUD",
+  });
+
+  const columns: LangfuseColumnDef<RowData>[] = [
+    {
+      accessorKey: "key",
+      header: "Name",
+      id: "key",
+      size: 150,
+      isPinned: true,
+      cell: ({ row }) => {
+        const key: RowData["key"] = row.getValue("key");
+        return key && "id" in key && typeof key.id === "string" ? (
+          <TableLink
+            path={`/project/${projectId}/annotation-queues/${key.id}`}
+            value={key.name}
+          />
+        ) : undefined;
+      },
+    },
+    {
+      accessorKey: "description",
+      header: "Description",
+      id: "description",
+      enableHiding: true,
+      size: 200,
+      cell: ({ row }) => {
+        const description: RowData["description"] = row.getValue("description");
+        return (
+          <span
+            className={cn(
+              "grid h-full items-center overflow-auto",
+              rowHeight === "s" && "leading-3",
+            )}
+          >
+            {description}
+          </span>
+        );
+      },
+    },
+    {
+      accessorKey: "countCompletedItems",
+      header: "Completed Items",
+      id: "countCompletedItems",
+      enableHiding: true,
+      size: 90,
+    },
+    {
+      accessorKey: "countPendingItems",
+      header: "Pending Items",
+      id: "countPendingItems",
+      enableHiding: true,
+      size: 90,
+    },
+    {
+      accessorKey: "scoreConfigs",
+      header: "Score Configs",
+      id: "scoreConfigs",
+      enableHiding: true,
+      size: 200,
+      cell: ({ row }) => {
+        const scoreConfigs: RowData["scoreConfigs"] =
+          row.getValue("scoreConfigs");
+
+        return (
+          <span
+            className={cn(
+              "grid h-full items-center overflow-auto",
+              rowHeight === "s" && "leading-3",
+            )}
+          >
+            {scoreConfigs
+              .map(
+                (config) =>
+                  `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
+              )
+              .join(", ")}
+          </span>
+        );
+      },
+    },
+    {
+      accessorKey: "createdAt",
+      header: "Created",
+      id: "createdAt",
+      enableHiding: true,
+      size: 150,
+    },
+    {
+      accessorKey: "processAction",
+      header: "Process",
+      id: "processAction",
+      isPinned: true,
+      cell: ({ row }) => {
+        const key: RowData["key"] = row.getValue("key");
+        return !hasAccess ? (
+          <Button size="sm" disabled>
+            <Lock className="mr-1 h-3 w-3" />
+            <span className="text-xs">Process queue</span>
+          </Button>
+        ) : (
+          <Button size="sm" asChild>
+            <Link
+              href={`/project/${projectId}/annotation-queues/${key.id}/items`}
+            >
+              <ClipboardPen className="mr-1 h-3 w-3" />
+              <span className="text-xs">Process queue</span>
+            </Link>
+          </Button>
+        );
+      },
+    },
+    {
+      accessorKey: "actions",
+      header: "Actions",
+      id: "actions",
+      size: 70,
+      isPinned: true,
+      cell: ({ row }) => {
+        const key: RowData["key"] = row.getValue("key");
+        return (
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button variant="ghost" className="h-8 w-8 p-0">
+                <span className="sr-only">Open menu</span>
+                <MoreVertical className="h-4 w-4" />
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent align="end">
+              <DropdownMenuLabel>Actions</DropdownMenuLabel>
+              <div className="flex flex-col space-y-0.5">
+                <CreateOrEditAnnotationQueueButton
+                  projectId={projectId}
+                  queueId={key.id}
+                  variant="ghost"
+                />
+                <DeleteAnnotationQueueButton
+                  projectId={projectId}
+                  queueId={key.id}
+                />
+              </div>
+            </DropdownMenuContent>
+          </DropdownMenu>
+        );
+      },
+    },
+  ];
+
+  const convertToTableRow = (
+    item: RouterOutput["annotationQueues"]["all"]["queues"][number],
+  ): RowData => {
+    return {
+      key: { id: item.id, name: item.name },
+      description: item.description ?? undefined,
+      scoreConfigs: item.scoreConfigs,
+      createdAt: item.createdAt.toLocaleString(),
+      countCompletedItems: item.countCompletedItems,
+      countPendingItems: item.countPendingItems,
+    };
+  };
+
+  const [columnVisibility, setColumnVisibility] = useColumnVisibility<RowData>(
+    "queuesColumnVisibility",
+    columns,
+  );
+
+  const [columnOrder, setColumnOrder] = useColumnOrder<RowData>(
+    "queuesColumnOrder",
+    columns,
+  );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        columnOrder={columnOrder}
+        setColumnOrder={setColumnOrder}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+        actionButtons={
+          <CreateOrEditAnnotationQueueButton projectId={projectId} />
+        }
+      />
+      <DataTable
+        columns={columns}
+        data={
+          queues.isLoading
+            ? { isLoading: true, isError: false }
+            : queues.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: queues.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: queues.data.queues.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          totalCount: queues.data?.totalCount ?? null,
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+        columnOrder={columnOrder}
+        onColumnOrderChange={setColumnOrder}
+        rowHeight={rowHeight}
+      />
+    </>
+  );
+}

web/src/ee/features/annotation-queues/components/CreateNewAnnotationQueueItem.tsx

@@ -0,0 +1,194 @@
+import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
+import {
+  DropdownMenuItem,
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuSeparator,
+  DropdownMenuCheckboxItem,
+  DropdownMenuLabel,
+} from "@/src/components/ui/dropdown-menu";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { api } from "@/src/utils/api";
+import { cn } from "@/src/utils/tailwind";
+import {
+  AnnotationQueueStatus,
+  type AnnotationQueueObjectType,
+} from "@langfuse/shared";
+import { ChevronDown, ExternalLink } from "lucide-react";
+import { useSession } from "next-auth/react";
+import Link from "next/link";
+import { useState, useCallback } from "react";
+
+export const CreateNewAnnotationQueueItem = ({
+  projectId,
+  objectId,
+  objectType,
+}: {
+  projectId: string;
+  objectId: string;
+  objectType: AnnotationQueueObjectType;
+}) => {
+  const [isDropdownOpen, setIsDropdownOpen] = useState(false);
+  const session = useSession();
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const queues = api.annotationQueues.byObjectId.useQuery(
+    {
+      projectId,
+      objectId,
+      objectType,
+    },
+    { enabled: session.status === "authenticated" },
+  );
+  const utils = api.useUtils();
+  const addToQueueMutation = api.annotationQueueItems.createMany.useMutation();
+  const removeFromQueueMutation =
+    api.annotationQueueItems.deleteMany.useMutation();
+
+  const handleQueueItemToggle = useCallback(
+    async (queueId: string, queueName: string, itemId?: string) => {
+      try {
+        if (!itemId) {
+          await addToQueueMutation.mutateAsync({
+            projectId,
+            objectIds: [objectId],
+            objectType,
+            queueId,
+          });
+        } else {
+          const confirmRemoval = confirm(
+            `Are you sure you want to remove this item from the queue "${queueName}"?`,
+          );
+          if (confirmRemoval) {
+            await removeFromQueueMutation.mutateAsync({
+              projectId,
+              itemIds: [itemId],
+            });
+          }
+        }
+        // Manually invalidate the query to refresh the data
+        await utils.annotationQueues.byObjectId.invalidate({
+          projectId,
+          objectId,
+          objectType,
+        });
+      } catch (error) {
+        console.error("Error toggling queue item:", error);
+      }
+    },
+    [
+      addToQueueMutation,
+      removeFromQueueMutation,
+      projectId,
+      objectId,
+      objectType,
+      utils.annotationQueues,
+    ],
+  );
+
+  if (session.status !== "authenticated" || queues.isLoading) {
+    return (
+      <Button
+        variant="secondary"
+        disabled
+        className="rounded-l-none rounded-r-md border-l-2"
+      >
+        <ChevronDown className="h-3 w-3" />
+      </Button>
+    );
+  }
+
+  return (
+    <DropdownMenu
+      key="queue"
+      open={isDropdownOpen}
+      onOpenChange={() => {
+        if (hasAccess) {
+          setIsDropdownOpen(!isDropdownOpen);
+        }
+      }}
+    >
+      <DropdownMenuTrigger asChild>
+        <Button
+          variant="secondary"
+          disabled={!hasAccess}
+          className="rounded-l-none rounded-r-md border-l-2"
+        >
+          {queues.data?.totalCount ? (
+            <span className="relative mr-1 text-xs">
+              <ChevronDown className="h-3 w-3 text-secondary-foreground" />
+              <span className="absolute -top-1.5 left-2.5 flex max-h-[0.8rem] min-w-[0.8rem] items-center justify-center rounded-full border border-muted-foreground bg-accent-light-blue px-[0.2rem] text-[8px]">
+                {queues.data?.totalCount > 99 ? "99+" : queues.data?.totalCount}
+              </span>
+            </span>
+          ) : (
+            <span className="relative mr-1 text-xs">
+              <ChevronDown className="h-3 w-3" />
+            </span>
+          )}
+        </Button>
+      </DropdownMenuTrigger>
+      <DropdownMenuContent>
+        <DropdownMenuLabel>In queue(s)</DropdownMenuLabel>
+        {queues.data?.queues.length ? (
+          queues.data?.queues.map((queue) => (
+            <DropdownMenuCheckboxItem
+              key={queue.id}
+              className="hover:bg-accent"
+              checked={!!queue.itemId}
+              onSelect={(event) => {
+                event.preventDefault();
+              }}
+              onClick={(event) => {
+                event.preventDefault();
+                event.stopPropagation();
+                handleQueueItemToggle(queue.id, queue.name, queue.itemId);
+              }}
+            >
+              {queue.name}
+              {queue.status && (
+                <Badge
+                  className={cn(
+                    "ml-2 px-1 py-0.5 text-[10px] capitalize",
+                    queue.status === AnnotationQueueStatus.COMPLETED
+                      ? "bg-light-green text-dark-green"
+                      : "bg-light-yellow text-dark-yellow",
+                  )}
+                  variant="outline"
+                >
+                  {queue.status.toLowerCase()}
+                </Badge>
+              )}
+            </DropdownMenuCheckboxItem>
+          ))
+        ) : (
+          <DropdownMenuItem
+            onClick={(event) => {
+              event.preventDefault();
+              event.stopPropagation();
+            }}
+          >
+            No queues defined
+          </DropdownMenuItem>
+        )}
+        <DropdownMenuSeparator />
+        <DropdownMenuItem
+          key="manage-queues"
+          className="hover:bg-accent"
+          asChild
+        >
+          <div>
+            <ExternalLink className="mr-2 h-4 w-4" />
+            <Link href={`/project/${projectId}/annotation-queues`}>
+              Manage queues
+            </Link>
+          </div>
+        </DropdownMenuItem>
+      </DropdownMenuContent>
+    </DropdownMenu>
+  );
+};

web/src/ee/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx

@@ -0,0 +1,332 @@
+import { Button } from "@/src/components/ui/button";
+import React, { useEffect, useState } from "react";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
+import {
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { Input } from "@/src/components/ui/input";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { CircleAlert, Edit, LockIcon, PlusIcon } from "lucide-react";
+import { useForm } from "react-hook-form";
+import { Form } from "@/src/components/ui/form";
+import { Textarea } from "@/src/components/ui/textarea";
+import {
+  type CreateQueue,
+  CreateQueueData,
+  type ValidatedScoreConfig,
+} from "@langfuse/shared";
+import { api } from "@/src/utils/api";
+import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
+import { CommandItem } from "@/src/components/ui/command";
+import { useRouter } from "next/router";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { useOrganizationPlan } from "@/src/features/entitlements/hooks";
+import {
+  HoverCard,
+  HoverCardContent,
+  HoverCardTrigger,
+} from "@/src/components/ui/hover-card";
+
+export const CreateOrEditAnnotationQueueButton = ({
+  projectId,
+  queueId,
+  variant = "secondary",
+}: {
+  projectId: string;
+  queueId?: string;
+  variant?: "secondary" | "ghost";
+}) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const router = useRouter();
+  const plan = useOrganizationPlan();
+  const capture = usePostHogClientCapture();
+
+  const queueQuery = api.annotationQueues.byId.useQuery(
+    { projectId, queueId: queueId as string },
+    { enabled: !!queueId && hasAccess },
+  );
+
+  const form = useForm<CreateQueue>({
+    resolver: zodResolver(CreateQueueData),
+  });
+
+  useEffect(() => {
+    if (queueId && queueQuery.data) {
+      form.reset({
+        name: queueQuery.data.name,
+        description: queueQuery.data.description || undefined,
+        scoreConfigIds: queueQuery.data.scoreConfigs.map(
+          (config: ValidatedScoreConfig) => config.id,
+        ),
+      });
+    } else {
+      form.reset({
+        name: "",
+        scoreConfigIds: [],
+      });
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [queueId, queueQuery.data]);
+
+  const utils = api.useUtils();
+
+  const createQueueMutation = api.annotationQueues.create.useMutation({
+    onSuccess: async () => {
+      await Promise.all([utils.annotationQueues.invalidate()]);
+      form.reset();
+      setIsOpen(false);
+    },
+  });
+  const editQueueMutation = api.annotationQueues.update.useMutation({
+    onSuccess: async () => {
+      await Promise.all([utils.annotationQueues.invalidate()]);
+      form.reset();
+      setIsOpen(false);
+    },
+  });
+
+  const queueCountData = api.annotationQueues.count.useQuery(
+    { projectId },
+    { enabled: hasAccess },
+  );
+
+  const configsData = api.scoreConfigs.all.useQuery(
+    {
+      projectId,
+    },
+    {
+      enabled: hasAccess && isOpen,
+    },
+  );
+
+  const configs = configsData.data?.configs ?? [];
+
+  if (!hasAccess) {
+    return (
+      <Button variant={variant} disabled={true} className="justify-start">
+        <LockIcon className="-ml-0.5 mr-1.5 h-4 w-4" aria-hidden="true" />
+        <span className="text-sm">{queueId ? "Edit" : "New queue"}</span>
+      </Button>
+    );
+  }
+
+  if (queueCountData.isLoading) return null;
+
+  // gate cloud hobby usage of annotation queue
+  if (plan === "cloud:hobby" && !!queueCountData.data && !queueId) {
+    return (
+      <HoverCard>
+        <HoverCardTrigger asChild>
+          <Button
+            variant={variant}
+            className="relative grid grid-flow-row items-start justify-start overflow-hidden py-0 disabled:cursor-default"
+            disabled
+          >
+            <div className="mt-2 flex h-6 flex-row items-center justify-center">
+              <PlusIcon className="mr-1.5 h-4 w-4" aria-hidden="true" />
+              <span className="text-sm">New queue</span>
+            </div>
+            <div className="absolute top-0 flex h-3 w-full items-center justify-center bg-primary-accent">
+              <CircleAlert
+                className="mr-1 h-2 w-2 text-white"
+                aria-hidden="true"
+              />
+              <span className="text-xs text-white">At usage limit</span>
+            </div>
+          </Button>
+        </HoverCardTrigger>
+        <HoverCardContent className="w-80" align="start" side="right">
+          <div className="flex justify-between space-x-4">
+            <div className="space-y-1">
+              <h4 className="text-sm font-semibold">Usage Limit Reached</h4>
+              <p className="text-xs">
+                You have reached the maximum number of annotation queues allowed
+                on the Hobby plan. Upgrade your plan to create more queues.
+              </p>
+            </div>
+          </div>
+        </HoverCardContent>
+      </HoverCard>
+    );
+  }
+
+  const onSubmit = (data: CreateQueue) => {
+    if (queueId) {
+      editQueueMutation.mutateAsync({
+        ...data,
+        projectId,
+        queueId,
+      });
+    } else {
+      createQueueMutation.mutateAsync({
+        ...data,
+        projectId,
+      });
+    }
+  };
+
+  const handleOnValueChange = (values: Record<string, string>[]) => {
+    form.setValue(
+      "scoreConfigIds",
+      values.map((value) => value.key),
+    );
+
+    if (values.length === 0) {
+      form.setError("scoreConfigIds", {
+        type: "manual",
+        message: "At least 1 score config must be selected",
+      });
+    } else {
+      form.clearErrors("scoreConfigIds");
+    }
+  };
+
+  return (
+    <Dialog open={isOpen} onOpenChange={setIsOpen}>
+      <DialogTrigger asChild>
+        <Button
+          variant={variant}
+          onClick={() => setIsOpen(true)}
+          className="justify-start"
+        >
+          {queueId ? (
+            <Edit className="-ml-0.5 mr-1.5 h-4 w-4" aria-hidden="true" />
+          ) : (
+            <PlusIcon className="-ml-0.5 mr-1.5 h-4 w-4" aria-hidden="true" />
+          )}
+          <span className="ml-1 text-sm font-normal">
+            {queueId ? "Edit" : "New queue"}
+          </span>
+        </Button>
+      </DialogTrigger>
+      {configsData.data && (
+        <DialogContent className="max-h-[90vh] overflow-y-auto">
+          <DialogHeader>
+            <DialogTitle>
+              {queueId ? "Edit" : "New"} annotation queue
+            </DialogTitle>
+            <DialogDescription>
+              {queueId ? "Edit" : "Create a new"} queue to manage your
+              annotation workflows.
+            </DialogDescription>
+          </DialogHeader>
+          <Form {...form}>
+            <form className="space-y-6" onSubmit={form.handleSubmit(onSubmit)}>
+              <FormField
+                control={form.control}
+                name="name"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Name</FormLabel>
+                    <FormControl>
+                      <Input
+                        {...field}
+                        type="text"
+                        className="text-xs"
+                        onBlur={(e) => field.onChange(e.target.value.trimEnd())}
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                control={form.control}
+                name="description"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Description (optional)</FormLabel>
+                    <FormControl>
+                      <Textarea
+                        {...field}
+                        placeholder="Add description..."
+                        className="text-xs focus:outline-none focus:ring-0 focus-visible:ring-0 focus-visible:ring-offset-0 active:ring-0"
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                control={form.control}
+                name="scoreConfigIds"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Score Configs</FormLabel>
+                    <FormDescription>
+                      Define which dimensions annotators should score for the
+                      given queue.
+                    </FormDescription>
+                    <FormControl>
+                      <MultiSelectKeyValues
+                        title="Value"
+                        align="end"
+                        className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
+                        onValueChange={handleOnValueChange}
+                        options={configs
+                          .filter((config) => !config.isArchived)
+                          .map((config) => ({
+                            key: config.id,
+                            value: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
+                            isArchived: config.isArchived,
+                          }))}
+                        values={field.value.map((configId) => {
+                          const config = configs.find(
+                            (config) => config.id === configId,
+                          );
+                          return {
+                            value: config
+                              ? `${getScoreDataTypeIcon(config.dataType)} ${config.name}`
+                              : `${configId}`,
+                            key: configId,
+                          };
+                        })}
+                        controlButtons={
+                          <CommandItem
+                            onSelect={() => {
+                              capture(
+                                "score_configs:manage_configs_item_click",
+                                { source: "AnnotationQueue" },
+                              );
+                              router.push(
+                                `/project/${projectId}/settings/scores`,
+                              );
+                            }}
+                          >
+                            Manage score configs
+                          </CommandItem>
+                        }
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <Button type="submit" className="text-xs">
+                {queueId ? "Save" : "Create"} queue
+              </Button>
+            </form>
+          </Form>
+        </DialogContent>
+      )}
+    </Dialog>
+  );
+};

web/src/ee/features/annotation-queues/components/DeleteAnnotationQueueButton.tsx

@@ -0,0 +1,88 @@
+import { Button } from "@/src/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { api } from "@/src/utils/api";
+import { LockIcon, Trash } from "lucide-react";
+import React, { useState } from "react";
+
+type DeleteAnnotationQueueButtonProps = {
+  projectId: string;
+  queueId: string;
+};
+
+export const DeleteAnnotationQueueButton = ({
+  projectId,
+  queueId,
+}: DeleteAnnotationQueueButtonProps) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const utils = api.useUtils();
+  const mutDelete = api.annotationQueues.delete.useMutation({
+    onSuccess: () => {
+      utils.annotationQueues.invalidate();
+    },
+  });
+
+  const button = (
+    <Button variant="ghost" disabled={!hasAccess}>
+      <div className="flex w-full flex-row items-center gap-1">
+        {hasAccess ? (
+          <Trash className="-ml-0.5 mr-1.5 h-4 w-4" />
+        ) : (
+          <LockIcon className="-ml-0.5 mr-1.5 h-4 w-4" aria-hidden="true" />
+        )}
+        <span className="text-sm font-normal">Delete</span>
+      </div>
+    </Button>
+  );
+
+  return hasAccess ? (
+    <Dialog
+      open={isOpen}
+      onOpenChange={(open) => {
+        if (!mutDelete.isLoading) {
+          setIsOpen(open);
+        }
+      }}
+    >
+      <DialogTrigger asChild>{button}</DialogTrigger>
+      <DialogContent className="sm:max-w-xl">
+        <DialogHeader>
+          <DialogTitle className="mb-4">Please confirm</DialogTitle>
+          <DialogDescription className="text-md p-0">
+            This action cannot be undone and removes queue items attached to
+            this queue. Scores added while annotating in this queue will not be
+            deleted.
+          </DialogDescription>
+        </DialogHeader>
+        <Button
+          variant="destructive"
+          loading={mutDelete.isLoading}
+          disabled={mutDelete.isLoading}
+          onClick={async (event) => {
+            event.preventDefault();
+            await mutDelete.mutateAsync({
+              projectId,
+              queueId,
+            });
+            setIsOpen(false);
+          }}
+        >
+          Delete Annotation Queue
+        </Button>
+      </DialogContent>
+    </Dialog>
+  ) : (
+    button
+  );
+};

web/src/ee/features/annotation-queues/pages/AnnotationQueueItems.tsx

@@ -0,0 +1,176 @@
+import Header from "@/src/components/layouts/header";
+import { api } from "@/src/utils/api";
+import { useRouter } from "next/router";
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import { AnnotationQueueItemsTable } from "@/src/ee/features/annotation-queues/components/AnnotationQueueItemsTable";
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/src/components/ui/card";
+import { Button } from "@/src/components/ui/button";
+import { ChevronRight, ClipboardPen, Lock } from "lucide-react";
+import { Separator } from "@/src/components/ui/separator";
+import { useState } from "react";
+import { Badge } from "@/src/components/ui/badge";
+import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import Link from "next/link";
+import { CreateOrEditAnnotationQueueButton } from "@/src/ee/features/annotation-queues/components/CreateOrEditAnnotationQueueButton";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
+import { SupportOrUpgradePage } from "@/src/ee/features/billing/components/SupportOrUpgradePage";
+import { Skeleton } from "@/src/components/ui/skeleton";
+
+const TableWithMetadataWrapper = ({
+  tableComponent,
+  cardTitleChildren,
+  cardContentChildren,
+}: {
+  tableComponent: React.ReactNode;
+  cardTitleChildren: React.ReactNode;
+  cardContentChildren: React.ReactNode;
+}) => {
+  const [isCollapsed, setIsCollapsed] = useState(false);
+
+  return (
+    <div
+      className={`grid h-[calc(100dvh-8rem)] ${isCollapsed ? "grid-cols-[2fr,auto]" : "grid-cols-[2fr,1fr]"} gap-4 overflow-hidden lg:h-[calc(100dvh-4rem)]`}
+    >
+      <div className="flex h-full flex-col overflow-hidden">
+        {tableComponent}
+      </div>
+      <div
+        className={`my-2 flex flex-row ${isCollapsed ? "w-8" : "w-full"} h-full overflow-hidden`}
+      >
+        <div className="grid h-full w-full grid-cols-[auto,1fr] items-start gap-2 overflow-hidden">
+          <div className="grid h-full w-full grid-rows-[auto,1fr] gap-2">
+            <Button
+              variant="outline"
+              size="icon"
+              onClick={() => setIsCollapsed(!isCollapsed)}
+            >
+              <ChevronRight
+                className={`h-4 w-4 transform ${isCollapsed ? "rotate-180" : ""}`}
+              />
+            </Button>
+            <Separator orientation="vertical" className="ml-4 h-full" />
+          </div>
+          <div
+            className={`${isCollapsed ? "hidden" : "block"} mt-8 grid h-[calc(100%-2rem)] w-full grid-rows-[auto,1fr] gap-2 overflow-hidden p-2`}
+          >
+            <Card className="flex h-full flex-col overflow-hidden">
+              <div className="flex h-full overflow-y-auto">
+                <CardHeader className="flex h-full w-full flex-col space-y-4">
+                  <CardTitle className="flex justify-between text-xl font-bold leading-7 sm:tracking-tight">
+                    {cardTitleChildren}
+                  </CardTitle>
+                  <CardContent className="flex-1 space-y-4 p-0">
+                    {cardContentChildren}
+                  </CardContent>
+                </CardHeader>
+              </div>
+            </Card>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default function QueueItems() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+  const queueId = router.query.queueId as string;
+
+  const queue = api.annotationQueues.byId.useQuery({
+    queueId,
+    projectId,
+  });
+
+  const hasReadAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:read",
+  });
+  const hasWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "annotationQueues:CUD",
+  });
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
+  if (!hasReadAccess || !hasEntitlement) return <SupportOrUpgradePage />;
+
+  return (
+    <FullScreenPage>
+      <>
+        <Header
+          title={queue.data?.name ?? queueId}
+          breadcrumb={[
+            {
+              name: "Annotation Queues",
+              href: `/project/${projectId}/annotation-queues`,
+            },
+            { name: queue.data?.name ?? queueId },
+          ]}
+          actionButtons={
+            !hasWriteAccess ? (
+              <Button disabled>
+                <Lock className="mr-1 h-4 w-4" />
+                <span className="text-sm">Process queue</span>
+              </Button>
+            ) : (
+              <Button asChild>
+                <Link
+                  href={`/project/${projectId}/annotation-queues/${queueId}/items`}
+                >
+                  <ClipboardPen className="mr-1 h-4 w-4" />
+                  <span className="text-sm">Process queue</span>
+                </Link>
+              </Button>
+            )
+          }
+        />
+        <TableWithMetadataWrapper
+          tableComponent={
+            <AnnotationQueueItemsTable
+              projectId={projectId}
+              queueId={queueId}
+            />
+          }
+          cardTitleChildren={
+            <div className="flex w-full flex-row items-center justify-between">
+              {queue.data ? (
+                <span>{queue.data.name}</span>
+              ) : (
+                <Skeleton className="h-full w-1/2" />
+              )}
+              <CreateOrEditAnnotationQueueButton
+                projectId={projectId}
+                queueId={queueId}
+              />
+            </div>
+          }
+          cardContentChildren={
+            <>
+              {queue.data?.description && (
+                <CardDescription className="text-sm">
+                  {queue.data?.description}
+                </CardDescription>
+              )}
+              <Separator orientation="horizontal" />
+              <h5 className="text-md font-bold leading-7 sm:tracking-tight">
+                Score Configs
+              </h5>
+              {queue.data?.scoreConfigs.map((scoreConfig) => (
+                <Badge key={scoreConfig.id} className="mr-2" variant="outline">
+                  {getScoreDataTypeIcon(scoreConfig.dataType)}
+                  <span className="ml-0.5">{scoreConfig.name}</span>
+                </Badge>
+              ))}
+            </>
+          }
+        />
+      </>
+    </FullScreenPage>
+  );
+}

web/src/ee/features/annotation-queues/pages/AnnotationQueues.tsx

@@ -0,0 +1,34 @@
+import Header from "@/src/components/layouts/header";
+import { useRouter } from "next/router";
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import { AnnotationQueuesTable } from "@/src/ee/features/annotation-queues/components/AnnotationQueuesTable";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
+import { SupportOrUpgradePage } from "@/src/ee/features/billing/components/SupportOrUpgradePage";
+
+export default function AnnotationQueues() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "annotationQueues:read",
+  });
+  const hasEntitlement = useHasOrgEntitlement("annotation-queues");
+  if (!hasAccess || !hasEntitlement) return <SupportOrUpgradePage />;
+
+  return (
+    <FullScreenPage>
+      <>
+        <Header
+          title="Annotation Queues"
+          help={{
+            description:
+              "Annotation queues are used to manage scoring workflows for your LLM projects. See docs to learn more.",
+            href: "https://langfuse.com/docs/scores/annotation",
+          }}
+        />
+        <AnnotationQueuesTable projectId={projectId} />
+      </>
+    </FullScreenPage>
+  );
+}

web/src/ee/features/annotation-queues/server/annotationQueueItems.ts

@@ -0,0 +1,445 @@
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
+import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import {
+  type AnnotationQueueItem,
+  AnnotationQueueObjectType,
+  AnnotationQueueStatus,
+  paginationZod,
+  Prisma,
+} from "@langfuse/shared";
+import { logger } from "@langfuse/shared/src/server";
+import { TRPCError } from "@trpc/server";
+import { z } from "zod";
+
+const isItemLocked = (item: AnnotationQueueItem) => {
+  return (
+    item.lockedByUserId &&
+    item.lockedAt &&
+    new Date(item.lockedAt) > new Date(Date.now() - 5 * 60 * 1000)
+  );
+};
+
+export const queueItemRouter = createTRPCRouter({
+  byId: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        itemId: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const item = await ctx.prisma.annotationQueueItem.findUnique({
+          where: {
+            id: input.itemId,
+            projectId: input.projectId,
+          },
+        });
+
+        // Expected behavior, non-error case: if user has seen item in given session, prior to it being deleted, we return null
+        if (!item) return null;
+        let lockedByUser: { name: string | null } | null = null;
+
+        if (isItemLocked(item)) {
+          lockedByUser = await ctx.prisma.user.findUnique({
+            where: {
+              id: item.lockedByUserId as string,
+            },
+            select: {
+              name: true,
+            },
+          });
+        }
+
+        const inflatedItem = {
+          ...item,
+          lockedByUser,
+        };
+
+        if (item.objectType === AnnotationQueueObjectType.OBSERVATION) {
+          const observation = await ctx.prisma.observation.findUnique({
+            where: {
+              id: item.objectId,
+              projectId: input.projectId,
+            },
+            select: {
+              id: true,
+              traceId: true,
+            },
+          });
+
+          return {
+            ...inflatedItem,
+            parentTraceId: observation?.traceId,
+          };
+        }
+
+        return inflatedItem;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queue item by id failed.",
+        });
+      }
+    }),
+  itemsByQueueId: protectedProjectProcedure
+    .input(
+      z.object({
+        queueId: z.string(),
+        projectId: z.string(),
+        ...paginationZod,
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const [queueItems, totalItems] = await Promise.all([
+          // queueItems
+          ctx.prisma.$queryRaw<
+            Array<{
+              id: string;
+              status: AnnotationQueueStatus;
+              objectId: string;
+              objectType: AnnotationQueueObjectType;
+              parentTraceId: string | null;
+              completedAt: string | null;
+              annotatorUserId: string | null;
+              annotatorUserImage: string | null;
+              annotatorUserName: string | null;
+            }>
+          >(Prisma.sql`
+          SELECT
+            aqi.id,
+            aqi.status,
+            aqi.object_id AS "objectId",
+            aqi.object_type AS "objectType",
+	          o.trace_id AS "parentTraceId",
+            aqi.completed_at AS "completedAt",
+            aqi.annotator_user_id AS "annotatorUserId",
+            u.image AS "annotatorUserImage", 
+            u.name AS "annotatorUserName"
+          FROM
+            annotation_queue_items aqi
+          LEFT JOIN 
+            observations o ON o.id = aqi.object_id AND aqi.object_type = 'OBSERVATION' AND o.project_id = ${input.projectId}
+          LEFT JOIN 
+            users u ON u.id = aqi.annotator_user_id AND u.id in (SELECT user_id FROM organization_memberships WHERE org_id = ${ctx.session.orgId})
+          WHERE 
+            aqi.project_id = ${input.projectId} AND aqi.queue_id = ${input.queueId}
+          ORDER BY 
+            aqi.created_at ASC,
+            aqi.object_id ASC,
+            aqi.object_type ASC
+          ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
+          ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
+        `),
+          // totalItems
+          ctx.prisma.annotationQueueItem.count({
+            where: {
+              queueId: input.queueId,
+              projectId: input.projectId,
+            },
+          }),
+        ]);
+
+        return { queueItems, totalItems };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queue items failed.",
+        });
+      }
+    }),
+  unseenPendingItemCountByQueueId: protectedProjectProcedure
+    .input(
+      z.object({
+        queueId: z.string(),
+        projectId: z.string(),
+        seenItemIds: z.array(z.string()),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const count = await ctx.prisma.annotationQueueItem.count({
+          where: {
+            queueId: input.queueId,
+            projectId: input.projectId,
+            status: AnnotationQueueStatus.PENDING,
+            id: {
+              notIn: input.seenItemIds,
+            },
+          },
+        });
+        return count;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching unseen pending item count by queueId failed.",
+        });
+      }
+    }),
+  createMany: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        queueId: z.string(),
+        objectIds: z
+          .array(z.string())
+          .min(1, "Minimum 1 object_id is required."),
+        objectType: z.nativeEnum(AnnotationQueueObjectType),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        const { count } = await ctx.prisma.annotationQueueItem.createMany({
+          data: input.objectIds.map((objectId) => ({
+            projectId: input.projectId,
+            queueId: input.queueId,
+            objectId,
+            objectType: input.objectType,
+          })),
+          skipDuplicates: true,
+        });
+
+        const createdItems = await ctx.prisma.annotationQueueItem.findMany({
+          where: {
+            projectId: input.projectId,
+            queueId: input.queueId,
+            objectId: { in: input.objectIds },
+            objectType: input.objectType,
+          },
+          orderBy: { createdAt: "desc" },
+        });
+
+        for (const item of createdItems) {
+          await auditLog(
+            {
+              session: ctx.session,
+              resourceType: "annotationQueueItem",
+              resourceId: item.id,
+              action: "create",
+              after: item,
+            },
+            ctx.prisma,
+          );
+        }
+
+        const queue = await ctx.prisma.annotationQueue.findUnique({
+          where: {
+            id: input.queueId,
+            projectId: input.projectId,
+          },
+          select: {
+            name: true,
+            id: true,
+          },
+        });
+
+        return {
+          createdCount: count,
+          queueName: queue?.name,
+          queueId: queue?.id,
+        };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Creating multiple annotation queue items failed.",
+        });
+      }
+    }),
+  deleteMany: protectedProjectProcedure
+    .input(
+      z.object({
+        itemIds: z.array(z.string()).min(1, "Minimum 1 item_id is required."),
+        projectId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        const items = await ctx.prisma.annotationQueueItem.findMany({
+          where: {
+            id: {
+              in: input.itemIds,
+            },
+            projectId: input.projectId,
+          },
+        });
+
+        for (const item of items) {
+          await auditLog({
+            resourceType: "annotationQueueItem",
+            resourceId: item.id,
+            before: item,
+            action: "delete",
+            session: ctx.session,
+          });
+        }
+
+        const { count } = await ctx.prisma.annotationQueueItem.deleteMany({
+          where: {
+            id: {
+              in: input.itemIds,
+            },
+            projectId: input.projectId,
+          },
+        });
+
+        return {
+          deletedCount: count,
+        };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Deleting annotation queue items failed.",
+        });
+      }
+    }),
+  complete: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        itemId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        const item = await ctx.prisma.annotationQueueItem.update({
+          where: {
+            id: input.itemId,
+            projectId: input.projectId,
+            status: AnnotationQueueStatus.PENDING,
+          },
+          data: {
+            status: AnnotationQueueStatus.COMPLETED,
+            completedAt: new Date(),
+            annotatorUserId: ctx.session.user.id,
+          },
+        });
+
+        await auditLog({
+          resourceType: "annotationQueueItem",
+          resourceId: item.id,
+          action: "complete",
+          after: item,
+          session: ctx.session,
+        });
+
+        return item;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        if (
+          error instanceof Prisma.PrismaClientKnownRequestError &&
+          error.code === "P2025"
+        ) {
+          throw new TRPCError({
+            code: "NOT_FOUND",
+            message:
+              "The item to complete was not found, it was likely deleted.",
+          });
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Completing annotation queue item failed.",
+        });
+      }
+    }),
+});

web/src/ee/features/annotation-queues/server/annotationQueues.ts

@@ -0,0 +1,561 @@
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
+import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import {
+  AnnotationQueueObjectType,
+  AnnotationQueueStatus,
+  CreateQueueData,
+  filterAndValidateDbScoreConfigList,
+  LangfuseNotFoundError,
+  optionalPaginationZod,
+  Prisma,
+} from "@langfuse/shared";
+import { logger } from "@langfuse/shared/src/server";
+import { TRPCError } from "@trpc/server";
+import { z } from "zod";
+
+export const queueRouter = createTRPCRouter({
+  all: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        ...optionalPaginationZod,
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const [queues, totalCount, scoreConfigs] = await Promise.all([
+          ctx.prisma.$queryRaw<
+            Array<{
+              id: string;
+              name: string;
+              description?: string | null;
+              scoreConfigIds: string[];
+              createdAt: string;
+              countCompletedItems: number;
+              countPendingItems: number;
+            }>
+          >(Prisma.sql`
+          SELECT
+            aq.id,
+            aq.name,
+            aq.description,
+            aq.score_config_ids AS "scoreConfigIds",
+            aq.created_at AS "createdAt",
+            COALESCE(SUM(CASE WHEN aqi.status = 'COMPLETED' THEN 1 ELSE 0 END), 0) AS "countCompletedItems",
+            COALESCE(SUM(CASE WHEN aqi.status = 'PENDING' THEN 1 ELSE 0 END), 0) AS "countPendingItems"
+          FROM
+            annotation_queues aq
+          LEFT JOIN
+            annotation_queue_items aqi ON aq.id = aqi.queue_id AND aqi.project_id = aq.project_id
+          WHERE
+            aq.project_id = ${input.projectId}
+          GROUP BY
+            aq.id, aq.name, aq.description, aq.created_at
+          ORDER BY
+            aq.created_at DESC
+          ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
+          ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
+        `),
+          ctx.prisma.annotationQueue.count({
+            where: {
+              projectId: input.projectId,
+            },
+          }),
+          ctx.prisma.scoreConfig.findMany({
+            where: {
+              projectId: input.projectId,
+            },
+            select: {
+              id: true,
+              name: true,
+              dataType: true,
+            },
+          }),
+        ]);
+
+        return {
+          totalCount,
+          queues: queues.map((queue) => ({
+            ...queue,
+            scoreConfigs: scoreConfigs.filter((config) =>
+              queue.scoreConfigIds.includes(config.id),
+            ),
+          })),
+        };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queues failed.",
+        });
+      }
+    }),
+  allNamesAndIds: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        const queueNamesAndIds = await ctx.prisma.annotationQueue.findMany({
+          where: {
+            projectId: input.projectId,
+          },
+          select: {
+            id: true,
+            name: true,
+          },
+        });
+
+        return queueNamesAndIds;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queues failed.",
+        });
+      }
+    }),
+  count: protectedProjectProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      try {
+        return ctx.prisma.annotationQueue.count({
+          where: { projectId: input.projectId },
+        });
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queue count failed.",
+        });
+      }
+    }),
+  byId: protectedProjectProcedure
+    .input(z.object({ queueId: z.string(), projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const queue = await ctx.prisma.annotationQueue.findUnique({
+          where: { id: input.queueId, projectId: input.projectId },
+        });
+
+        const configs = await ctx.prisma.scoreConfig.findMany({
+          where: {
+            projectId: input.projectId,
+            id: {
+              in: queue?.scoreConfigIds ?? [],
+            },
+          },
+        });
+
+        return {
+          ...queue,
+          scoreConfigs: filterAndValidateDbScoreConfigList(configs),
+        };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queue failed.",
+        });
+      }
+    }),
+  byObjectId: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        objectId: z.string(),
+        objectType: z.nativeEnum(AnnotationQueueObjectType),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:read",
+        });
+
+        const queues = await ctx.prisma.annotationQueue.findMany({
+          where: {
+            projectId: input.projectId,
+          },
+          select: {
+            id: true,
+            name: true,
+            annotationQueueItem: {
+              where: {
+                objectId: input.objectId,
+                objectType: input.objectType,
+              },
+              select: {
+                queueId: true,
+                status: true,
+                id: true,
+              },
+            },
+          },
+        });
+
+        let totalCount = 0;
+
+        return {
+          queues: queues.map((queue) => {
+            totalCount += queue.annotationQueueItem.length;
+            return {
+              id: queue.id,
+              name: queue.name,
+              itemId: queue.annotationQueueItem[0]?.id, // Safely access the first item's id
+              status: queue.annotationQueueItem[0]?.status, // Safely access the first item's status
+              // Since there may be multiple queue items in a given queue, but with the same objectId, we select only the first one
+              // to simplify the logic and because we are only interested in the first item's details.
+            };
+          }),
+          totalCount,
+          // If the given objectId has been added to the same queue more than once, the total count will reflect that, by counting each item (incl duplicates)
+        };
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching annotation queue by objectId failed.",
+        });
+      }
+    }),
+  create: protectedProjectProcedure
+    .input(
+      CreateQueueData.extend({
+        projectId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        // gate usage on cloud:hobby
+        const org = ctx.session.user.organizations.find((org) =>
+          org.projects.some((proj) => proj.id === input.projectId),
+        );
+        const plan = org?.plan ?? "oss";
+
+        if (plan === "cloud:hobby") {
+          if (
+            (await ctx.prisma.annotationQueue.count({
+              where: {
+                projectId: input.projectId,
+              },
+            })) >= 1
+          ) {
+            throw new TRPCError({
+              code: "FORBIDDEN",
+              message:
+                "Maximum number of annotation queues reached on Hobby plan.",
+            });
+          }
+        }
+
+        const existingQueue = await ctx.prisma.annotationQueue.findFirst({
+          where: {
+            projectId: input.projectId,
+            name: input.name,
+          },
+        });
+
+        if (existingQueue) {
+          throw new TRPCError({
+            code: "CONFLICT",
+            message: "A queue with this name already exists in the project",
+          });
+        }
+
+        const queue = await ctx.prisma.annotationQueue.create({
+          data: {
+            name: input.name,
+            projectId: input.projectId,
+            description: input.description,
+            scoreConfigIds: input.scoreConfigIds,
+          },
+        });
+
+        await auditLog({
+          session: ctx.session,
+          resourceType: "annotationQueue",
+          resourceId: queue.id,
+          action: "create",
+          after: queue,
+        });
+
+        return queue;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Creating annotation queue failed.",
+        });
+      }
+    }),
+  update: protectedProjectProcedure
+    .input(
+      CreateQueueData.extend({
+        projectId: z.string(),
+        queueId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        const queue = await ctx.prisma.annotationQueue.findFirst({
+          where: {
+            id: input.queueId,
+            projectId: input.projectId,
+          },
+        });
+
+        if (!queue) {
+          throw new LangfuseNotFoundError("Queue not found in project");
+        }
+
+        const updatedQueue = await ctx.prisma.annotationQueue.update({
+          where: { id: input.queueId, projectId: input.projectId },
+          data: {
+            name: input.name,
+            description: input.description,
+            scoreConfigIds: input.scoreConfigIds,
+          },
+        });
+
+        await auditLog({
+          session: ctx.session,
+          resourceType: "annotationQueue",
+          resourceId: queue.id,
+          action: "update",
+          before: queue,
+          after: updatedQueue,
+        });
+
+        return updatedQueue;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Updating annotation queue failed.",
+        });
+      }
+    }),
+  delete: protectedProjectProcedure
+    .input(z.object({ queueId: z.string(), projectId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+        const queue = await ctx.prisma.annotationQueue.delete({
+          where: { id: input.queueId, projectId: input.projectId },
+        });
+
+        await auditLog({
+          session: ctx.session,
+          resourceType: "annotationQueue",
+          resourceId: queue.id,
+          action: "delete",
+          before: queue,
+        });
+
+        return queue;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Deleting annotation queue failed.",
+        });
+      }
+    }),
+  fetchAndLockNext: protectedProjectProcedure
+    .input(
+      z.object({
+        queueId: z.string(),
+        projectId: z.string(),
+        seenItemIds: z.array(z.string()),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoEntitlement({
+          entitlement: "annotation-queues",
+          projectId: input.projectId,
+          sessionUser: ctx.session.user,
+        });
+
+        throwIfNoProjectAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "annotationQueues:CUD",
+        });
+
+        const now = new Date();
+        const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000);
+
+        const item = await ctx.prisma.annotationQueueItem.findFirst({
+          where: {
+            queueId: input.queueId,
+            projectId: input.projectId,
+            status: AnnotationQueueStatus.PENDING,
+            OR: [
+              { lockedAt: null },
+              { lockedAt: { lt: fiveMinutesAgo } },
+              { lockedByUserId: ctx.session.user.id },
+            ],
+            NOT: {
+              id: { in: input.seenItemIds },
+            },
+          },
+          orderBy: {
+            createdAt: "asc",
+          },
+        });
+
+        // Expected behavior, non-error case: all items have been seen AND/OR completed, no more unseen pending items
+        if (!item) return null;
+
+        const updatedItem = await ctx.prisma.annotationQueueItem.update({
+          where: {
+            id: item.id,
+            projectId: input.projectId,
+          },
+          data: {
+            lockedAt: now,
+            lockedByUserId: ctx.session.user.id,
+          },
+        });
+
+        const inflatedUpdatedItem = {
+          ...updatedItem,
+          lockedByUser: { name: ctx.session.user.name },
+        };
+
+        if (item.objectType === AnnotationQueueObjectType.OBSERVATION) {
+          const observation = await ctx.prisma.observation.findUnique({
+            where: {
+              id: item.objectId,
+              projectId: input.projectId,
+            },
+            select: {
+              id: true,
+              traceId: true,
+            },
+          });
+
+          return {
+            ...inflatedUpdatedItem,
+            parentTraceId: observation?.traceId,
+          };
+        }
+
+        return inflatedUpdatedItem;
+      } catch (error) {
+        logger.error(error);
+        if (error instanceof TRPCError) {
+          throw error;
+        }
+        throw new TRPCError({
+          code: "INTERNAL_SERVER_ERROR",
+          message: "Fetching and locking next annotation queue item failed.",
+        });
+      }
+    }),
+});

web/src/ee/features/billing/components/SupportOrUpgradePage.tsx

@@ -0,0 +1,9 @@
+// TODO: replace placeholder div with actual component
+export const SupportOrUpgradePage = () => {
+  return (
+    <div>
+      You have no access to this feature. Check with your system administrator
+      to get elevated access or upgrade your plan.
+    </div>
+  );
+};

web/src/ee/features/evals/components/template-form.tsx

@@ -195,7 +195,7 @@ export const InnerEvalTemplateForm = (props: {
     setModelParamEnabled,
     availableModels,
     availableProviders,
-  } = useModelParams({ evalModelsOnly: true });
+  } = useModelParams();
 
   useEffect(() => {
     if (props.preFilledFormValues?.selectedModel) {

web/src/ee/features/evals/server/router.ts

@@ -9,13 +9,19 @@ import { auditLog } from "@/src/features/audit-logs/auditLog";
 import {
   DEFAULT_TRACE_JOB_DELAY,
   EvalTargetObject,
-  LLMAdapter,
   ZodModelConfig,
   singleFilter,
   variableMapping,
+  ChatMessageRole,
 } from "@langfuse/shared";
+import { decrypt } from "@langfuse/shared/encryption";
 import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
-import { logger } from "@langfuse/shared/src/server";
+import {
+  fetchLLMCompletion,
+  LLMApiKeySchema,
+  logger,
+} from "@langfuse/shared/src/server";
+import { TRPCError } from "@trpc/server";
 
 export const CreateEvalTemplate = z.object({
   name: z.string().min(1),
@@ -343,16 +349,45 @@ export const evalRouter = createTRPCRouter({
         },
       });
 
-      if (!matchingLLMKey) {
+      const parsedKey = LLMApiKeySchema.safeParse(matchingLLMKey);
+
+      if (!matchingLLMKey || !parsedKey.success) {
         throw new Error("No matching LLM key found for provider");
       }
 
-      // check that the adapter on the api key is openai for evals
-      if (
-        matchingLLMKey.adapter !== LLMAdapter.OpenAI &&
-        matchingLLMKey.adapter !== LLMAdapter.Azure
-      ) {
-        throw new Error("Only OpenAI and Azure models are supported for evals");
+      // Make a test structured output call to validate the LLM key
+      try {
+        await fetchLLMCompletion({
+          streaming: false,
+          apiKey: decrypt(parsedKey.data.secretKey), // decrypt the secret key
+          baseURL: parsedKey.data.baseURL ?? undefined,
+          messages: [
+            {
+              role: ChatMessageRole.System,
+              content: "You are an expert at evaluating LLM outputs.",
+            },
+            { role: ChatMessageRole.User, content: input.prompt },
+          ],
+          modelParams: {
+            provider: input.provider,
+            model: input.model,
+            adapter: parsedKey.data.adapter,
+            ...input.modelParams,
+          },
+          structuredOutputSchema: z.object({
+            score: z.string(),
+            reasoning: z.string(),
+          }),
+          config: parsedKey.data.config,
+        });
+      } catch (err) {
+        logger.error(err);
+
+        throw new TRPCError({
+          code: "PRECONDITION_FAILED",
+          message:
+            "Selected model is not supported for evaluations. Test tool call failed.",
+        });
       }
 
       const latestTemplate = await ctx.prisma.evalTemplate.findFirst({

web/src/ee/features/multi-tenant-sso/types.ts

@@ -28,6 +28,18 @@ export const GithubProviderSchema = base.extend({
     .nullish(),
 });
 
+export const GitlabProviderSchema = base.extend({
+  authProvider: z.literal("gitlab"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string().optional(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const Auth0ProviderSchema = base.extend({
   authProvider: z.literal("auth0"),
   authConfig: z
@@ -92,6 +104,7 @@ export const CustomProviderSchema = base.extend({
 
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type GitlabProviderSchema = z.infer<typeof GitlabProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
@@ -101,6 +114,7 @@ export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
   GithubProviderSchema,
+  GitlabProviderSchema,
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,

web/src/ee/features/multi-tenant-sso/utils.ts

@@ -1,6 +1,7 @@
 import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
+import GitLabProvider from "next-auth/providers/gitlab";
 import OktaProvider from "next-auth/providers/okta";
 import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
@@ -157,6 +158,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "gitlab")
+    return GitLabProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else if (provider.authProvider === "auth0")
     return Auth0Provider({
       id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials

web/src/ee/features/playground/page/components/JumpToPlaygroundButton.tsx

@@ -62,13 +62,16 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
   return (
     <Button
       variant={props.variant ?? "secondary"}
+      size={props.source === "prompt" ? "icon" : "default"}
       title="Test in LLM playground"
       onClick={handleClick}
       asChild
     >
       <Link href={`/project/${projectId}/playground`}>
         <Terminal className="h-4 w-4" />
-        <span className="ml-2">Test in playground</span>
+        {props.source === "generation" && (
+          <span className="ml-2">Test in playground</span>
+        )}
       </Link>
     </Button>
   );

web/src/ee/features/playground/page/hooks/useModelParams.ts

@@ -4,17 +4,12 @@ import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
 import {
   LLMAdapter,
-  evalModelsByAdapter,
   supportedModels,
   type UIModelParams,
 } from "@langfuse/shared";
 import { type ModelParamsContext } from "@/src/components/ModelParameters";
 
-export const useModelParams = ({
-  evalModelsOnly,
-}: {
-  evalModelsOnly?: boolean;
-} = {}) => {
+export const useModelParams = () => {
   const [modelParams, setModelParams] = useState<UIModelParams>({
     ...getDefaultAdapterParams(LLMAdapter.OpenAI),
     provider: { value: "", enabled: true },
@@ -31,16 +26,10 @@ export const useModelParams = ({
   );
 
   const availableProviders = useMemo(() => {
-    const adapter =
-      (evalModelsOnly
-        ? availableLLMApiKeys.data?.data.filter(
-            (p) =>
-              p.adapter === LLMAdapter.OpenAI || p.adapter === LLMAdapter.Azure,
-          ) // for evals, we only support OpenAI
-        : availableLLMApiKeys.data?.data) ?? [];
+    const adapter = availableLLMApiKeys.data?.data ?? [];
 
     return adapter.map((key) => key.provider) ?? [];
-  }, [availableLLMApiKeys.data?.data, evalModelsOnly]);
+  }, [availableLLMApiKeys.data?.data]);
 
   const selectedProviderApiKey = availableLLMApiKeys.data?.data.find(
     (key) => key.provider === modelParams.provider.value,
@@ -50,18 +39,13 @@ export const useModelParams = ({
     () =>
       !selectedProviderApiKey
         ? []
-        : evalModelsOnly
+        : selectedProviderApiKey.withDefaultModels
           ? [
-              ...evalModelsByAdapter[selectedProviderApiKey.adapter],
               ...selectedProviderApiKey.customModels,
+              ...supportedModels[selectedProviderApiKey.adapter],
             ]
-          : selectedProviderApiKey.withDefaultModels
-            ? [
-                ...selectedProviderApiKey.customModels,
-                ...supportedModels[selectedProviderApiKey.adapter],
-              ]
-            : selectedProviderApiKey.customModels,
-    [selectedProviderApiKey, evalModelsOnly],
+          : selectedProviderApiKey.customModels,
+    [selectedProviderApiKey],
   );
 
   const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
@@ -174,5 +158,17 @@ function getDefaultAdapterParams(
         max_tokens: { value: 256, enabled: true },
         top_p: { value: 1, enabled: true },
       };
+
+    case LLMAdapter.Bedrock:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 1, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
   }
 }

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -3,8 +3,8 @@ import { NextResponse, type NextRequest } from "next/server";
 
 import {
   BaseError,
+  InternalServerError,
   InvalidRequestError,
-  fetchLLMCompletion,
 } from "@langfuse/shared";
 
 import { PosthogCallbackHandler } from "./analytics/posthogCallback";
@@ -13,7 +13,11 @@ import { validateChatCompletionBody } from "./validateChatCompletionBody";
 
 import { prisma } from "@langfuse/shared/src/db";
 import { decrypt } from "@langfuse/shared/encryption";
-import { logger } from "@langfuse/shared/src/server";
+import {
+  LLMApiKeySchema,
+  logger,
+  fetchLLMCompletion,
+} from "@langfuse/shared/src/server";
 
 export default async function chatCompletionHandler(req: NextRequest) {
   try {
@@ -34,13 +38,21 @@ export default async function chatCompletionHandler(req: NextRequest) {
         `No ${modelParams.provider} API key found in project. Please add one in the project settings.`,
       );
 
+    const parsedKey = LLMApiKeySchema.safeParse(LLMApiKey);
+    if (!parsedKey.success) {
+      throw new InternalServerError(
+        `Could not parse API key for provider ${body.modelParams.provider}: ${parsedKey.error.message}`,
+      );
+    }
+
     const stream = await fetchLLMCompletion({
       messages,
       modelParams,
       streaming: true,
       callbacks: [new PosthogCallbackHandler("playground", body, userId)],
-      apiKey: decrypt(LLMApiKey.secretKey),
-      baseURL: LLMApiKey.baseURL || undefined,
+      apiKey: decrypt(parsedKey.data.secretKey),
+      baseURL: parsedKey.data.baseURL || undefined,
+      config: parsedKey.data.config,
     });
 
     return new StreamingTextResponse(stream);

web/src/env.mjs

@@ -55,6 +55,10 @@ export const env = createEnv({
     AUTH_GITHUB_CLIENT_ID: z.string().optional(),
     AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GITLAB_CLIENT_ID: z.string().optional(),
+    AUTH_GITLAB_CLIENT_SECRET: z.string().optional(),
+    AUTH_GITLAB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GITLAB_ISSUER: z.string().optional(),
     AUTH_AZURE_AD_CLIENT_ID: z.string().optional(),
     AUTH_AZURE_AD_CLIENT_SECRET: z.string().optional(),
     AUTH_AZURE_AD_TENANT_ID: z.string().optional(),
@@ -90,20 +94,15 @@ export const env = createEnv({
       .optional()
       .default(30 * 24 * 60), // default to 30 days
     // EMAIL
-    EMAIL_FROM_ADDRESS: z
-      .string()
-      .optional()
-      .transform((v) => (v === "" ? undefined : v)),
-    SMTP_CONNECTION_URL: z
-      .string()
-      .optional()
-      .transform((v) => (v === "" ? undefined : v)),
+    EMAIL_FROM_ADDRESS: z.string().optional(),
+    SMTP_CONNECTION_URL: z.string().optional(),
     // S3
     S3_ENDPOINT: z.string().optional(),
     S3_ACCESS_KEY_ID: z.string().optional(),
     S3_SECRET_ACCESS_KEY: z.string().optional(),
     S3_BUCKET_NAME: z.string().optional(),
     S3_REGION: z.string().optional(),
+    S3_FORCE_PATH_STYLE: z.enum(["true", "false"]).default("false"),
     // Database exports
     DB_EXPORT_PAGE_SIZE: z.number().optional(),
     // Worker
@@ -126,7 +125,9 @@ export const env = createEnv({
     LANGFUSE_UI_FEEDBACK_HREF: z.string().url().optional(),
     LANGFUSE_UI_LOGO_LIGHT_MODE_HREF: z.string().url().optional(),
     LANGFUSE_UI_LOGO_DARK_MODE_HREF: z.string().url().optional(),
-    LANGFUSE_UI_DEFAULT_MODEL_ADAPTER: z.enum(["OpenAI", "Anthropic", "Azure"]).optional(),
+    LANGFUSE_UI_DEFAULT_MODEL_ADAPTER: z
+      .enum(["OpenAI", "Anthropic", "Azure"])
+      .optional(),
     LANGFUSE_UI_DEFAULT_BASE_URL_OPENAI: z.string().url().optional(),
     LANGFUSE_UI_DEFAULT_BASE_URL_ANTHROPIC: z.string().url().optional(),
     LANGFUSE_UI_DEFAULT_BASE_URL_AZURE: z.string().url().optional(),
@@ -165,6 +166,9 @@ export const env = createEnv({
     LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
     LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: z.string().optional(),
     LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: z.string().optional(),
+    LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
+      .enum(["true", "false"])
+      .default("false"),
     LANGFUSE_ASYNC_INGESTION_PROCESSING: z
       .enum(["true", "false"])
       .default("false"),
@@ -173,14 +177,12 @@ export const env = createEnv({
       .optional()
       .refine((value) => {
         if (!value) return true;
-
         const creators = value.split(",");
         const emailSchema = z.string().email();
         return creators.every(
           (creator) => emailSchema.safeParse(creator).success,
         );
-      }, "LANGFUSE_ALLOWED_ORGANIZATION_CREATORS must be a comma separated list of valid email addresses")
-      .transform((v) => (v === "" || v === undefined ? undefined : v)),
+      }, "LANGFUSE_ALLOWED_ORGANIZATION_CREATORS must be a comma separated list of valid email addresses"),
     LANGFUSE_INGESTION_BUFFER_TTL_SECONDS: z.coerce
       .number()
       .positive()
@@ -190,15 +192,17 @@ export const env = createEnv({
     SENTRY_AUTH_TOKEN: z.string().optional(),
     SENTRY_CSP_REPORT_URI: z.string().optional(),
     LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),
-    LANGFUSE_INIT_ORG_ID: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_ORG_NAME: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_PROJECT_ID: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_PROJECT_NAME: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_PROJECT_PUBLIC_KEY: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_PROJECT_SECRET_KEY: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_USER_EMAIL: z.union([z.string().email(), z.string().length(0)]).optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_USER_NAME: z.string().optional().transform((v) => v === "" ? undefined : v),
-    LANGFUSE_INIT_USER_PASSWORD: z.string().optional().transform((v) => v === "" ? undefined : v),
+    LANGFUSE_INIT_ORG_ID: z.string().optional(),
+    LANGFUSE_INIT_ORG_NAME: z.string().optional(),
+    LANGFUSE_INIT_PROJECT_ID: z.string().optional(),
+    LANGFUSE_INIT_PROJECT_NAME: z.string().optional(),
+    LANGFUSE_INIT_PROJECT_PUBLIC_KEY: z.string().optional(),
+    LANGFUSE_INIT_PROJECT_SECRET_KEY: z.string().optional(),
+    LANGFUSE_INIT_USER_EMAIL: z
+      .union([z.string().email(), z.string().length(0)])
+      .optional(),
+    LANGFUSE_INIT_USER_NAME: z.string().optional(),
+    LANGFUSE_INIT_USER_PASSWORD: z.string().optional(),
   },
 
   /**
@@ -217,7 +221,7 @@ export const env = createEnv({
       .optional(),
     NEXT_PUBLIC_DEMO_PROJECT_ID: z.string().optional(),
     NEXT_PUBLIC_DEMO_ORG_ID: z.string().optional(),
-    NEXT_PUBLIC_SIGN_UP_DISABLED: z.enum(["true", "false"]).optional(),
+    NEXT_PUBLIC_SIGN_UP_DISABLED: z.enum(["true", "false"]).default("false"),
     NEXT_PUBLIC_TURNSTILE_SITE_KEY: z.string().optional(),
     NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(),
     NEXT_PUBLIC_POSTHOG_HOST: z.string().optional(),
@@ -267,6 +271,11 @@ export const env = createEnv({
     AUTH_GITHUB_CLIENT_SECRET: process.env.AUTH_GITHUB_CLIENT_SECRET,
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GITHUB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITLAB_ISSUER: process.env.AUTH_GITLAB_ISSUER,
+    AUTH_GITLAB_CLIENT_ID: process.env.AUTH_GITLAB_CLIENT_ID,
+    AUTH_GITLAB_CLIENT_SECRET: process.env.AUTH_GITLAB_CLIENT_SECRET,
+    AUTH_GITLAB_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GITLAB_ALLOW_ACCOUNT_LINKING,
     AUTH_AZURE_AD_CLIENT_ID: process.env.AUTH_AZURE_AD_CLIENT_ID,
     AUTH_AZURE_AD_CLIENT_SECRET: process.env.AUTH_AZURE_AD_CLIENT_SECRET,
     AUTH_AZURE_AD_TENANT_ID: process.env.AUTH_AZURE_AD_TENANT_ID,
@@ -311,6 +320,7 @@ export const env = createEnv({
     S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY,
     S3_BUCKET_NAME: process.env.S3_BUCKET_NAME,
     S3_REGION: process.env.S3_REGION,
+    S3_FORCE_PATH_STYLE: process.env.S3_FORCE_PATH_STYLE,
     // S3 event upload
     LANGFUSE_S3_EVENT_UPLOAD_ENABLED:
       process.env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED,
@@ -326,6 +336,8 @@ export const env = createEnv({
       process.env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
     LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:
       process.env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+    LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:
+      process.env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE,
     // Database exports
     DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
     // Worker
@@ -399,4 +411,5 @@ export const env = createEnv({
   // Skip validation in Docker builds
   // DOCKER_BUILD is set in Dockerfile
   skipValidation: process.env.DOCKER_BUILD === "1",
+  emptyStringAsUndefined: true, // https://env.t3.gg/docs/customization#treat-empty-strings-as-undefined
 });

web/src/features/audit-logs/auditLog.ts

@@ -1,6 +1,8 @@
 import { prisma as _prisma, type Role } from "@langfuse/shared/src/db";
 
 export type AuditableResource =
+  | "annotationQueue"
+  | "annotationQueueItem"
   | "organization"
   | "orgMembership"
   | "projectMembership"

web/src/features/auth-credentials/components/ResetPasswordPage.tsx

@@ -240,7 +240,7 @@ export function ResetPasswordPage({
           <div className="mx-auto mt-10 max-w-lg text-center text-xs text-muted-foreground">
             You will only receive an email if an account with this email exists
             and you have signed up with email and password. If you used an
-            authentication provider like Google, Okta, or GitHub, please{" "}
+            authentication provider like Google, Gitlab, Okta, or GitHub, please{" "}
             <Link href="/auth/sign-in" className="underline">
               sign in
             </Link>

web/src/features/datasets/components/DatasetItemsTable.tsx

@@ -112,13 +112,13 @@ export function DatasetItemsTable({
         if (!source) return null;
         return source.observationId ? (
           <TableLink
-            path={`/project/${projectId}/traces/${source.traceId}?observation=${source.observationId}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(source.traceId)}?observation=${encodeURIComponent(source.observationId)}`}
             value={source.observationId}
             icon={<ListTree className="h-4 w-4" />}
           />
         ) : (
           <TableLink
-            path={`/project/${projectId}/traces/${source.traceId}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(source.traceId)}`}
             value={source.traceId}
             icon={<ListTree className="h-4 w-4" />}
           />

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -120,13 +120,13 @@ export function DatasetRunItemsTable(
         if (!trace) return null;
         return trace.observationId ? (
           <TableLink
-            path={`/project/${props.projectId}/traces/${trace.traceId}?observation=${trace.observationId}`}
+            path={`/project/${props.projectId}/traces/${encodeURIComponent(trace.traceId)}?observation=${encodeURIComponent(trace.observationId)}`}
             value={`Trace: ${trace.traceId}, Observation: ${trace.observationId}`}
             icon={<ListTree className="h-4 w-4" />}
           />
         ) : (
           <TableLink
-            path={`/project/${props.projectId}/traces/${trace.traceId}`}
+            path={`/project/${props.projectId}/traces/${encodeURIComponent(trace.traceId)}`}
             value={`Trace: ${trace.traceId}`}
             icon={<ListTree className="h-4 w-4" />}
           />

web/src/features/datasets/components/NewDatasetItemFromObservationButton.tsx

@@ -1,4 +1,3 @@
-import { Button } from "@/src/components/ui/button";
 import { ChevronDown, LockIcon, PlusIcon } from "lucide-react";
 import {
   Dialog,
@@ -10,17 +9,18 @@ import { api } from "@/src/utils/api";
 import { cn } from "@/src/utils/tailwind";
 import { useState } from "react";
 import {
-  DropdownMenu,
-  DropdownMenuContent,
   DropdownMenuItem,
   DropdownMenuSeparator,
+  DropdownMenu,
+  DropdownMenuContent,
   DropdownMenuTrigger,
 } from "@/src/components/ui/dropdown-menu";
 import Link from "next/link";
 import { NewDatasetItemForm } from "@/src/features/datasets/components/NewDatasetItemForm";
-import { type Prisma } from "@langfuse/shared";
-import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { type Prisma } from "@langfuse/shared/src/db";
 import { useSession } from "next-auth/react";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { Button } from "@/src/components/ui/button";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const NewDatasetItemFromTrace = (props: {
@@ -31,7 +31,7 @@ export const NewDatasetItemFromTrace = (props: {
   output: Prisma.JsonValue;
   metadata: Prisma.JsonValue;
 }) => {
-  const [open, setOpen] = useState(false);
+  const [isFormOpen, setIsFormOpen] = useState(false);
   const session = useSession();
   const observationInDatasets =
     api.datasets.datasetItemsBasedOnTraceOrObservation.useQuery(
@@ -58,7 +58,7 @@ export const NewDatasetItemFromTrace = (props: {
             <DropdownMenuTrigger asChild>
               <Button variant="secondary" disabled={!hasAccess}>
                 <span>{`In ${observationInDatasets.data.length} dataset(s)`}</span>
-                <ChevronDown className="ml-2" />
+                <ChevronDown className="ml-2 h-3 w-3" />
               </Button>
             </DropdownMenuTrigger>
             <DropdownMenuContent align="end">
@@ -80,7 +80,9 @@ export const NewDatasetItemFromTrace = (props: {
               <DropdownMenuSeparator />
               <DropdownMenuItem
                 className="capitalize"
-                onClick={() => setOpen(true)}
+                onClick={() => {
+                  setIsFormOpen(true);
+                }}
               >
                 <PlusIcon size={16} className={cn("mr-2")} aria-hidden="true" />
                 Add new
@@ -91,7 +93,7 @@ export const NewDatasetItemFromTrace = (props: {
       ) : (
         <Button
           onClick={() => {
-            setOpen(true);
+            setIsFormOpen(true);
             capture("dataset_item:new_from_trace_form_open", {
               object: props.observationId ? "observation" : "trace",
             });
@@ -100,7 +102,10 @@ export const NewDatasetItemFromTrace = (props: {
           disabled={!hasAccess}
         >
           {hasAccess ? (
-            <PlusIcon className={cn("-ml-0.5 mr-1.5")} aria-hidden="true" />
+            <PlusIcon
+              className={cn("-ml-0.5 mr-1.5 h-4 w-4")}
+              aria-hidden="true"
+            />
           ) : null}
           Add to dataset
           {!hasAccess ? (
@@ -108,7 +113,7 @@ export const NewDatasetItemFromTrace = (props: {
           ) : null}
         </Button>
       )}
-      <Dialog open={hasAccess && open} onOpenChange={setOpen}>
+      <Dialog open={hasAccess && isFormOpen} onOpenChange={setIsFormOpen}>
         <DialogContent className="h-[calc(100vh-5rem)] max-h-none w-[calc(100vw-5rem)] max-w-none">
           <DialogHeader>
             <DialogTitle>Add to dataset</DialogTitle>
@@ -120,7 +125,7 @@ export const NewDatasetItemFromTrace = (props: {
             input={props.input}
             output={props.output}
             metadata={props.metadata}
-            onFormSuccess={() => setOpen(false)}
+            onFormSuccess={() => setIsFormOpen(false)}
             className="h-full overflow-y-auto"
           />
         </DialogContent>

web/src/features/entitlements/constants/entitlements.ts

@@ -8,6 +8,7 @@ const entitlements = [
   "cloud-billing",
   "integration-posthog",
   "batch-export",
+  "annotation-queues",
 ] as const;
 
 export type Entitlement = (typeof entitlements)[number];
@@ -18,6 +19,7 @@ const cloudAllPlansEntitlements: Entitlement[] = [
   "cloud-billing",
   "integration-posthog",
   "batch-export",
+  "annotation-queues",
 ];
 
 export const entitlementAccess: Record<Plan, Entitlement[]> = {
@@ -26,6 +28,7 @@ export const entitlementAccess: Record<Plan, Entitlement[]> = {
   "cloud:pro": [...cloudAllPlansEntitlements],
   "cloud:team": [...cloudAllPlansEntitlements, "rbac-project-roles"],
   "self-hosted:enterprise": [
+    "annotation-queues",
     "playground",
     "rbac-project-roles",
     // `LANGFUSE_ALLOWED_ORGANIZATION_CREATORS` -> directly checked on instance level in auth.ts

web/src/features/feature-flags/README.md

@@ -0,0 +1,15 @@
+# Feature Flags
+
+Configure feature flags in the `available-flags.ts` file.
+
+Use the `useIsFeatureEnabled` hook to check if a feature flag is enabled.
+
+```tsx
+const isFeatureEnabled = useIsFeatureEnabled("feature-flag-name");
+```
+
+When is a feature flag enabled?
+
+1. flag is in user.feature_flags
+2. LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES is set
+3. user.admin is true

web/src/features/feature-flags/components/FeatureFlagToggle.tsx

@@ -1,5 +1,23 @@
 import { type Flag } from "@/src/features/feature-flags/types";
-import { useSession } from "next-auth/react";
+import { type SessionContextValue, useSession } from "next-auth/react";
+
+const isAdminOrExperimentalFeatures = (
+  session: SessionContextValue,
+): boolean => {
+  const enableExperimentalFeatures =
+    session.data?.environment?.enableExperimentalFeatures ?? false;
+  const isAdmin = session.data?.user?.admin ?? false;
+
+  return enableExperimentalFeatures || isAdmin;
+};
+
+const isWhitelistedForFeature = (
+  session: SessionContextValue,
+  featureFlag: Flag,
+): boolean => {
+  const flags = session.data?.user?.featureFlags;
+  return flags !== undefined && flags[featureFlag];
+};
 
 export const FeatureFlagToggle = (props: {
   featureFlag: Flag;
@@ -9,18 +27,15 @@ export const FeatureFlagToggle = (props: {
 }) => {
   const session = useSession();
 
-  const enableExperimentalFeatures =
-    session.data?.environment.enableExperimentalFeatures ?? false;
-  const isAdmin = session.data?.user?.admin ?? false;
+  if (isAdminOrExperimentalFeatures(session)) return props.whenEnabled ?? <></>;
 
-  if (enableExperimentalFeatures || isAdmin) return props.whenEnabled ?? <></>;
-
-  const flags = session.data?.user?.featureFlags;
-  const isEnabled = flags !== undefined && flags[props.featureFlag];
+  const isEnabled = isWhitelistedForFeature(session, props.featureFlag);
 
   if (session.status === "loading") {
     return props.whenLoading ?? <div>Loading ...</div>;
   }
 
-  return isEnabled ? props.whenEnabled ?? <></> : props.whenDisabled ?? <></>;
+  return isEnabled
+    ? (props.whenEnabled ?? <></>)
+    : (props.whenDisabled ?? <></>);
 };

web/src/features/llm-api-key/server/router.ts

@@ -9,15 +9,16 @@ import {
 import {
   type ChatMessage,
   LLMApiKeySchema,
-  fetchLLMCompletion,
   ChatMessageRole,
   supportedModels,
 } from "@langfuse/shared";
 import { encrypt } from "@langfuse/shared/encryption";
-import { logger } from "@langfuse/shared/src/server";
+import { fetchLLMCompletion, logger } from "@langfuse/shared/src/server";
 
 export function getDisplaySecretKey(secretKey: string) {
-  return "..." + secretKey.slice(-4);
+  return secretKey.endsWith('"}')
+    ? "..." + secretKey.slice(-6, -2)
+    : "..." + secretKey.slice(-4);
 }
 
 export const llmApiKeyRouter = createTRPCRouter({
@@ -41,6 +42,7 @@ export const llmApiKeyRouter = createTRPCRouter({
             baseURL: input.baseURL,
             withDefaultModels: input.withDefaultModels,
             customModels: input.customModels,
+            config: input.config,
           },
         });
 
@@ -157,6 +159,7 @@ export const llmApiKeyRouter = createTRPCRouter({
           messages: testMessages,
           streaming: false,
           maxRetries: 1,
+          config: input.config,
         });
 
         return { success: true };

web/src/features/llm-api-key/types.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { LLMAdapter } from "@langfuse/shared";
+import { LLMAdapter, BedrockConfigSchema } from "@langfuse/shared";
 
 export const CreateLlmApiKey = z.object({
   projectId: z.string(),
@@ -9,4 +9,5 @@ export const CreateLlmApiKey = z.object({
   baseURL: z.string().url().optional(),
   withDefaultModels: z.boolean().optional(),
   customModels: z.array(z.string().min(1)).optional(),
+  config: BedrockConfigSchema.optional(),
 });

web/src/features/notifications/SuccessNotification.tsx

@@ -1,15 +1,21 @@
 import { BadgeCheck, X } from "lucide-react";
+import Link from "next/link";
 
 export type SuccessNotificationProps = {
   title: string;
   description: string;
   onDismiss: () => void;
+  link?: {
+    href: string;
+    text: string;
+  };
 };
 
 export const SuccessNotification: React.FC<SuccessNotificationProps> = ({
   title,
   description,
   onDismiss,
+  link,
 }) => {
   return (
     <div className="flex justify-between">
@@ -22,7 +28,12 @@ export const SuccessNotification: React.FC<SuccessNotificationProps> = ({
         </div>
         {description && (
           <div className="text-sm leading-tight text-primary-foreground">
-            {description}
+            {description}{" "}
+            {!!link && (
+              <Link href={link.href}>
+                <span className="hover:underline">{link.text}</span>
+              </Link>
+            )}
           </div>
         )}
       </div>

web/src/features/posthog-analytics/usePostHogClientCapture.ts

@@ -103,7 +103,7 @@ const events = {
     "modal_open",
     "create_new_button_click",
   ],
-  onboarding: ["code_example_tab_switch"],
+  onboarding: ["code_example_tab_switch", "tracing_check_active"],
   user_settings: ["theme_changed"],
   project_settings: [
     "project_delete",
@@ -128,6 +128,7 @@ const events = {
   ],
   help_popup: ["opened", "href_clicked"],
   navigate_detail_pages: ["button_click_prev_or_next"],
+  support_chat: ["initiated", "opened", "message_sent"], // also used on landing page for consistency
 } as const;
 
 // type that represents all possible event names, e.g. "traces:bookmark"

web/src/features/prompts/components/SetPromptVersionLabels/index.tsx

@@ -107,92 +107,98 @@ export function SetPromptVersionLabels({
           aria-label="Set prompt labels"
           title="Set prompt labels"
           disabled={!hasAccess}
+          onClick={(event) => {
+            event.stopPropagation();
+          }}
         >
           <TagIcon className="h-4 w-4" />
         </Button>
       </PopoverTrigger>
       <PopoverContent>
-        <h2 className="text-md mb-3 font-semibold">Prompt version labels</h2>
-        <h2 className="mb-3 text-xs">
-          Use labels to fetch prompts via SDKs. The <strong>production</strong>{" "}
-          labeled prompt will be served by default.
-        </h2>
-        <Command className="mx-0 my-3 px-0">
-          <CommandList className="max-h-full overflow-hidden">
-            <CommandSeparator />
-            <CommandGroup heading="Promote to production?">
-              <LabelCommandItem
-                {...{
-                  selectedLabels,
-                  setSelectedLabels,
-                  label: PRODUCTION_LABEL,
-                }}
-              />
-            </CommandGroup>
-            <CommandSeparator />
-            <CommandGroup heading="Custom labels">
-              <div
-                className="max-h-[300px] overflow-y-auto overflow-x-hidden"
-                ref={customLabelScrollRef}
-              >
-                {labels
-                  .filter((l) => !isReservedPromptLabel(l))
-                  .map((label) => (
-                    <LabelCommandItem
-                      key={label}
-                      {...{ selectedLabels, setSelectedLabels, label }}
-                    />
-                  ))}
-              </div>
-            </CommandGroup>
-          </CommandList>
-          <div className="px-1">
-            {isAddingLabel ? (
-              <AddLabelForm
-                {...{
-                  setLabels,
-                  setSelectedLabels,
-                  onAddLabel: () => {
-                    setTimeout(
-                      () =>
-                        customLabelScrollRef.current?.scrollTo({
-                          top: customLabelScrollRef.current?.scrollHeight,
-                          behavior: "smooth",
-                        }),
-                      0,
-                    );
-                  },
-                }}
-              />
-            ) : (
-              <Button
-                variant="ghost"
-                className="mt-2 w-full justify-start px-2 py-1 text-sm font-normal"
-                onClick={() => setIsAddingLabel(true)}
-              >
-                <PlusIcon className="mr-2 h-4 w-4" />
-                Add custom label
-              </Button>
-            )}
-          </div>
-        </Command>
-        <Button
-          type="button"
-          variant={
-            isPromotingToProduction || isDemotingFromProduction
-              ? "destructive"
-              : "default"
-          }
-          loading={mutatePromptVersionLabels.isLoading}
-          className="w-full"
-          onClick={handleSubmitLabels}
-        >
-          {isPromotingToProduction
-            ? "Save and promote to production"
-            : isDemotingFromProduction
-              ? "Save and remove from production"
-              : "Save"}
-        </Button>
+        <div onClick={(event) => event.stopPropagation()}>
+          <h2 className="text-md mb-3 font-semibold">Prompt version labels</h2>
+          <h2 className="mb-3 text-xs">
+            Use labels to fetch prompts via SDKs. The{" "}
+            <strong>production</strong> labeled prompt will be served by
+            default.
+          </h2>
+          <Command className="mx-0 my-3 px-0">
+            <CommandList className="max-h-full overflow-hidden">
+              <CommandSeparator />
+              <CommandGroup heading="Promote to production?">
+                <LabelCommandItem
+                  {...{
+                    selectedLabels,
+                    setSelectedLabels,
+                    label: PRODUCTION_LABEL,
+                  }}
+                />
+              </CommandGroup>
+              <CommandSeparator />
+              <CommandGroup heading="Custom labels">
+                <div
+                  className="max-h-[300px] overflow-y-auto overflow-x-hidden"
+                  ref={customLabelScrollRef}
+                >
+                  {labels
+                    .filter((l) => !isReservedPromptLabel(l))
+                    .map((label) => (
+                      <LabelCommandItem
+                        key={label}
+                        {...{ selectedLabels, setSelectedLabels, label }}
+                      />
+                    ))}
+                </div>
+              </CommandGroup>
+            </CommandList>
+            <div className="px-1">
+              {isAddingLabel ? (
+                <AddLabelForm
+                  {...{
+                    setLabels,
+                    setSelectedLabels,
+                    onAddLabel: () => {
+                      setTimeout(
+                        () =>
+                          customLabelScrollRef.current?.scrollTo({
+                            top: customLabelScrollRef.current?.scrollHeight,
+                            behavior: "smooth",
+                          }),
+                        0,
+                      );
+                    },
+                  }}
+                />
+              ) : (
+                <Button
+                  variant="ghost"
+                  className="mt-2 w-full justify-start px-2 py-1 text-sm font-normal"
+                  onClick={() => setIsAddingLabel(true)}
+                >
+                  <PlusIcon className="mr-2 h-4 w-4" />
+                  Add custom label
+                </Button>
+              )}
+            </div>
+          </Command>
+          <Button
+            type="button"
+            variant={
+              isPromotingToProduction || isDemotingFromProduction
+                ? "destructive"
+                : "default"
+            }
+            loading={mutatePromptVersionLabels.isLoading}
+            className="w-full"
+            onClick={handleSubmitLabels}
+          >
+            {isPromotingToProduction
+              ? "Save and promote to production"
+              : isDemotingFromProduction
+                ? "Save and remove from production"
+                : "Save"}
+          </Button>
+        </div>
       </PopoverContent>
     </Popover>
   );

web/src/features/prompts/components/delete-prompt-version.tsx

@@ -64,6 +64,9 @@ export function DeletePromptVersion({
           size="icon"
           className="h-7 w-7 px-0"
           disabled={!hasAccess}
+          onClick={(event) => {
+            event.stopPropagation();
+          }}
         >
           <Trash className="h-4 w-4" />
         </Button>

web/src/features/prompts/components/prompt-detail.tsx

@@ -25,14 +25,23 @@ import {
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { ChatMlArraySchema } from "@/src/components/schemas/ChatMlSchema";
 import { CommentList } from "@/src/features/comments/CommentList";
+import { Lock, Plus } from "lucide-react";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { Button } from "@/src/components/ui/button";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const PromptDetail = () => {
   const projectId = useProjectIdFromURL();
+  const capture = usePostHogClientCapture();
   const promptName = decodeURIComponent(useRouter().query.promptName as string);
   const [currentPromptVersion, setCurrentPromptVersion] = useQueryParam(
     "version",
     NumberParam,
   );
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "prompts:CUD",
+  });
   const promptHistory = api.prompts.allVersions.useQuery(
     {
       name: promptName,
@@ -49,7 +58,7 @@ export const PromptDetail = () => {
   const extractedVariables = prompt
     ? extractVariables(
         prompt?.type === PromptType.Text
-          ? prompt.prompt?.toString() ?? ""
+          ? (prompt.prompt?.toString() ?? "")
           : JSON.stringify(prompt.prompt),
       )
     : [];
@@ -105,6 +114,30 @@ export const PromptDetail = () => {
             ]}
             actionButtons={
               <>
+                {hasAccess ? (
+                  <Button
+                    variant="secondary"
+                    onClick={() => {
+                      capture("prompts:update_form_open");
+                    }}
+                  >
+                    <Link
+                      href={`/project/${projectId}/prompts/new?promptId=${encodeURIComponent(prompt.id)}`}
+                    >
+                      <div className="flex flex-row items-center">
+                        <Plus className="h-4 w-4" />
+                        <span className="ml-2">New version</span>
+                      </div>
+                    </Link>
+                  </Button>
+                ) : (
+                  <Button variant="secondary" disabled>
+                    <div className="flex flex-row items-center">
+                      <Lock className="h-3 w-3" />
+                      <span className="ml-2">New version</span>
+                    </div>
+                  </Button>
+                )}
                 <JumpToPlaygroundButton
                   source="prompt"
                   prompt={prompt}

web/src/features/prompts/components/prompt-history.tsx

@@ -1,13 +1,8 @@
 import { StatusBadge } from "@/src/components/layouts/status-badge";
-import { Button } from "@/src/components/ui/button";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
 import { SetPromptVersionLabels } from "@/src/features/prompts/components/SetPromptVersionLabels";
 import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
-import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { type RouterOutputs } from "@/src/utils/api";
-import { Pencil, PencilOff } from "lucide-react";
-import Link from "next/link";
 import { type NextRouter, useRouter } from "next/router";
 import { useState } from "react";
 
@@ -20,13 +15,8 @@ const PromptHistoryTraceNode = (props: {
   projectId: string;
   totalCount: number;
 }) => {
-  const capture = usePostHogClientCapture();
   const [isHovered, setIsHovered] = useState(false);
   const [isLabelPopoverOpen, setIsLabelPopoverOpen] = useState(false);
-  const hasAccess = useHasProjectAccess({
-    projectId: props.projectId,
-    scope: "prompts:CUD",
-  });
   const { prompt } = props;
   let badges: JSX.Element[] = prompt.labels
     .sort((a, b) =>
@@ -80,7 +70,7 @@ const PromptHistoryTraceNode = (props: {
             </span>
           </div>
         </div>
-        {isHovered && (
+        {(isHovered || props.currentPromptVersion === prompt.version) && (
           <div className="flex flex-row justify-end space-x-1">
             <SetPromptVersionLabels
               prompt={prompt}
@@ -90,31 +80,6 @@ const PromptHistoryTraceNode = (props: {
                 if (!open) setIsHovered(false);
               }}
             />
-            {hasAccess ? (
-              <Button
-                variant="outline"
-                size="icon"
-                className="h-7 w-7 px-0"
-                onClick={() => {
-                  capture("prompts:update_form_open");
-                }}
-              >
-                <Link
-                  href={`/project/${props.projectId}/prompts/new?promptId=${encodeURIComponent(prompt.id)}`}
-                >
-                  <Pencil className="h-4 w-4" />
-                </Link>
-              </Button>
-            ) : (
-              <Button
-                variant="outline"
-                size="icon"
-                className="h-7 w-7 px-0"
-                disabled
-              >
-                <PencilOff className="h-4 w-4" />
-              </Button>
-            )}
             <DeletePromptVersion
               promptVersionId={prompt.id}
               version={prompt.version}

web/src/features/prompts/components/prompts-table.tsx

@@ -256,7 +256,10 @@ export function PromptTable() {
               }}
             >
               {hasCUDAccess ? (
-                <PlusIcon className="-ml-0.5 mr-1.5" aria-hidden="true" />
+                <PlusIcon
+                  className="-ml-0.5 mr-1.5 h-4 w-4"
+                  aria-hidden="true"
+                />
               ) : (
                 <LockIcon
                   className="-ml-0.5 mr-1.5 h-3 w-3"

web/src/features/public-api/components/CreateLLMApiKeyDialog.tsx

@@ -26,7 +26,6 @@ export function CreateLLMApiKeyDialog({
     scope: "llmApiKeys:create",
   });
   const uiCustomization = useUiCustomization();
-  console.log("uiCustomization", uiCustomization);
 
   if (!hasAccess) return null;
 

web/src/features/public-api/components/CreateLLMApiKeyForm.tsx

@@ -1,6 +1,10 @@
 import { useFieldArray, useForm } from "react-hook-form";
 import { zodResolver } from "@hookform/resolvers/zod";
-import { LLMAdapter } from "@langfuse/shared";
+import {
+  type BedrockConfig,
+  type BedrockCredential,
+  LLMAdapter,
+} from "@langfuse/shared";
 import { PlusIcon, TrashIcon } from "lucide-react";
 import { z } from "zod";
 import { Button } from "@/src/components/ui/button";
@@ -29,7 +33,7 @@ import { type useUiCustomization } from "@/src/ee/features/ui-customization/useU
 
 const formSchema = z
   .object({
-    secretKey: z.string().min(1),
+    secretKey: z.string().optional(),
     provider: z
       .string()
       .min(1, "Please add a provider name that identifies this connection."),
@@ -37,11 +41,27 @@ const formSchema = z
     baseURL: z.union([z.literal(""), z.string().url()]),
     withDefaultModels: z.boolean(),
     customModels: z.array(z.object({ value: z.string().min(1) })),
+    awsAccessKeyId: z.string().optional(),
+    awsSecretAccessKey: z.string().optional(),
+    awsRegion: z.string().optional(),
   })
   .refine((data) => data.withDefaultModels || data.customModels.length > 0, {
     message:
       "At least one custom model name is required when default models are disabled.",
     path: ["withDefaultModels"],
+  })
+  .refine(
+    (data) =>
+      data.adapter !== LLMAdapter.Bedrock ||
+      (data.awsAccessKeyId && data.awsSecretAccessKey && data.awsRegion),
+    {
+      message: "AWS credentials are required when using Bedrock adapter.",
+      path: ["adapter"],
+    },
+  )
+  .refine((data) => data.adapter === LLMAdapter.Bedrock || data.secretKey, {
+    message: "Secret key is required.",
+    path: ["secretKey"],
   });
 
 export function CreateLLMApiKeyForm({
@@ -122,13 +142,29 @@ export function CreateLLMApiKeyForm({
       provider: values.provider,
     });
 
+    let secretKey = values.secretKey;
+    let config: BedrockConfig | undefined;
+
+    if (currentAdapter === LLMAdapter.Bedrock) {
+      const credentials: BedrockCredential = {
+        accessKeyId: values.awsAccessKeyId ?? "",
+        secretAccessKey: values.awsSecretAccessKey ?? "",
+      };
+      secretKey = JSON.stringify(credentials);
+
+      config = {
+        region: values.awsRegion ?? "",
+      };
+    }
+
     const newKey = {
       projectId,
-      secretKey: values.secretKey,
+      secretKey: secretKey ?? "",
       provider: values.provider,
       adapter: values.adapter,
       baseURL: values.baseURL || undefined,
       withDefaultModels: values.withDefaultModels,
+      config,
       customModels: values.customModels
         .map((m) => m.value.trim())
         .filter(Boolean),
@@ -232,59 +268,108 @@ export function CreateLLMApiKeyForm({
         />
 
         {/* baseURL */}
-        <FormField
-          control={form.control}
-          name="baseURL"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>API Base URL</FormLabel>
-              <FormDescription>
-                Leave blank to use the default base URL for the given LLM
-                adapter.{" "}
-                {currentAdapter === LLMAdapter.OpenAI && (
-                  <span>OpenAI default: https://api.openai.com/v1</span>
-                )}
-                {currentAdapter === LLMAdapter.Azure && (
-                  <span>
-                    Please add the base URL in the following format (or
-                    compatible API):
-                    https://&#123;instanceName&#125;.openai.azure.com/openai/deployments
-                  </span>
-                )}
-                {currentAdapter === LLMAdapter.Anthropic && (
-                  <span>
-                    Anthropic default: https://api.anthropic.com (excluding
-                    /v1/messages)
-                  </span>
-                )}
-              </FormDescription>
+        {currentAdapter !== LLMAdapter.Bedrock && (
+          <FormField
+            control={form.control}
+            name="baseURL"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>API Base URL</FormLabel>
+                <FormDescription>
+                  Leave blank to use the default base URL for the given LLM
+                  adapter.{" "}
+                  {currentAdapter === LLMAdapter.OpenAI && (
+                    <span>OpenAI default: https://api.openai.com/v1</span>
+                  )}
+                  {currentAdapter === LLMAdapter.Azure && (
+                    <span>
+                      Please add the base URL in the following format (or
+                      compatible API):
+                      https://&#123;instanceName&#125;.openai.azure.com/openai/deployments
+                    </span>
+                  )}
+                  {currentAdapter === LLMAdapter.Anthropic && (
+                    <span>
+                      Anthropic default: https://api.anthropic.com (excluding
+                      /v1/messages)
+                    </span>
+                  )}
+                </FormDescription>
 
-              <FormControl>
-                <Input {...field} placeholder="default" />
-              </FormControl>
+                <FormControl>
+                  <Input {...field} placeholder="default" />
+                </FormControl>
 
-              <FormMessage />
-            </FormItem>
-          )}
-        />
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+        )}
 
-        {/* API key */}
-        <FormField
-          control={form.control}
-          name="secretKey"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>API Key</FormLabel>
-              <FormDescription>
-                Your API keys are stored encrypted on our servers.
-              </FormDescription>
-              <FormControl>
-                <Input {...field} />
-              </FormControl>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
+        {currentAdapter === LLMAdapter.Bedrock ? (
+          <>
+            <FormField
+              control={form.control}
+              name="awsRegion"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>AWS Region</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="awsAccessKeyId"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>AWS Access Key ID</FormLabel>
+                  <FormDescription>
+                    These should be long-lived credentials for an AWS user with
+                    the appropriate Bedrock permissions.
+                  </FormDescription>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="awsSecretAccessKey"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>AWS Secret Access Key</FormLabel>
+                  <FormControl>
+                    <Input {...field} type="password" />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </>
+        ) : (
+          <FormField
+            control={form.control}
+            name="secretKey"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>API Key</FormLabel>
+                <FormDescription>
+                  Your API keys are stored encrypted on our servers.
+                </FormDescription>
+                <FormControl>
+                  <Input {...field} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+        )}
 
         {/* With default models */}
         <FormField
@@ -305,13 +390,25 @@ export function CreateLLMApiKeyForm({
                       add a custom model with your deployment name.
                     </FormDescription>
                   )}
+                  {currentAdapter === LLMAdapter.Bedrock && (
+                    <FormDescription className="text-dark-yellow">
+                      Bedrock LLM adapter does not support default models.
+                      Please add your enabled Bedrock model IDs.
+                    </FormDescription>
+                  )}
                 </span>
 
                 <FormControl>
                   <Switch
-                    disabled={currentAdapter === LLMAdapter.Azure}
+                    disabled={
+                      currentAdapter === LLMAdapter.Azure ||
+                      currentAdapter === LLMAdapter.Bedrock
+                    }
                     checked={
-                      currentAdapter === LLMAdapter.Azure ? false : field.value
+                      currentAdapter === LLMAdapter.Azure ||
+                      currentAdapter === LLMAdapter.Bedrock
+                        ? false
+                        : field.value
                     }
                     onCheckedChange={field.onChange}
                   />
@@ -341,6 +438,14 @@ export function CreateLLMApiKeyForm({
                 </FormDescription>
               )}
 
+              {currentAdapter === LLMAdapter.Bedrock && (
+                <FormDescription className="text-dark-yellow">
+                  {
+                    "For Bedrock, the model name is the Bedrock model ID, e.g. 'eu.anthropic.claude-3-5-sonnet-20240620-v1:0'"
+                  }
+                </FormDescription>
+              )}
+
               {fields.map((customModel, index) => (
                 <span key={index} className="flex flex-row space-x-2">
                   <Input

web/src/features/rbac/constants/projectAccessRights.ts

@@ -18,6 +18,9 @@ const projectScopes = [
   "scoreConfigs:CUD",
   "scoreConfigs:read",
 
+  "annotationQueues:read",
+  "annotationQueues:CUD",
+
   "project:read",
   "project:update",
   "project:delete",
@@ -83,6 +86,8 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
     "batchExport:create",
     "comments:CUD",
     "comments:read",
+    "annotationQueues:read",
+    "annotationQueues:CUD",
   ],
   ADMIN: [
     "project:read",
@@ -115,6 +120,8 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
     "batchExport:create",
     "comments:CUD",
     "comments:read",
+    "annotationQueues:read",
+    "annotationQueues:CUD",
   ],
   MEMBER: [
     "project:read",
@@ -138,6 +145,8 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
     "batchExport:create",
     "comments:CUD",
     "comments:read",
+    "annotationQueues:read",
+    "annotationQueues:CUD",
   ],
   VIEWER: [
     "project:read",
@@ -148,6 +157,7 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
     "evalJobExecution:read",
     "llmApiKeys:read",
     "comments:read",
+    "annotationQueues:read",
   ],
   NONE: [],
 };

web/src/features/scores/components/AnnotateDrawer.tsx

@@ -24,6 +24,7 @@ export function AnnotateDrawer({
   variant = "button",
   type = "trace",
   source = "TraceDetail",
+  hasGroupedButton = false,
 }: {
   traceId: string;
   scores: APIScore[];
@@ -34,6 +35,7 @@ export function AnnotateDrawer({
   variant?: "button" | "badge";
   type?: "trace" | "observation" | "session";
   source?: "TraceDetail" | "SessionDetail";
+  hasGroupedButton?: boolean;
 }) {
   const [isDrawerOpen, setIsDrawerOpen] = useState(false);
   const capture = usePostHogClientCapture();
@@ -62,6 +64,7 @@ export function AnnotateDrawer({
           <Button
             variant="secondary"
             disabled={!hasAccess}
+            className={hasGroupedButton ? "rounded-r-none" : ""}
             onClick={() => {
               setIsDrawerOpen(true);
               capture(

web/src/features/scores/components/AnnotateDrawerContent.tsx

@@ -1,6 +1,13 @@
-import React, { useEffect, useRef } from "react";
+import React, { useEffect, useRef, useState } from "react";
 import { Button } from "@/src/components/ui/button";
-import { MessageCircleMore, MessageCircle, X, Archive } from "lucide-react";
+import {
+  MessageCircleMore,
+  MessageCircle,
+  X,
+  Archive,
+  Loader2,
+  Check,
+} from "lucide-react";
 import {
   type ControllerRenderProps,
   useFieldArray,
@@ -141,6 +148,9 @@ export function AnnotateDrawerContent({
   projectId,
   type = "trace",
   source = "TraceDetail",
+  isSelectHidden = false,
+  queueId,
+  actionButtons,
 }: {
   traceId: string;
   scores: APIScore[];
@@ -151,9 +161,13 @@ export function AnnotateDrawerContent({
   projectId: string;
   type?: "trace" | "observation" | "session";
   source?: "TraceDetail" | "SessionDetail";
+  isSelectHidden?: boolean;
+  queueId?: string;
+  actionButtons?: React.ReactNode;
 }) {
   const capture = usePostHogClientCapture();
   const router = useRouter();
+  const [showSaving, setShowSaving] = useState(false);
 
   const form = useForm<AnnotateFormSchemaType>({
     resolver: zodResolver(AnnotateFormSchema),
@@ -176,8 +190,14 @@ export function AnnotateDrawerContent({
   const prevEmptySelectedConfigIdsRef = useRef(emptySelectedConfigIds);
 
   useEffect(() => {
-    // Only reset the form if emptySelectedConfigIds has changed
-    if (prevEmptySelectedConfigIdsRef.current !== emptySelectedConfigIds) {
+    // Only reset the form if emptySelectedConfigIds has changed, compare by value not reference
+    if (
+      prevEmptySelectedConfigIdsRef.current.length !==
+        emptySelectedConfigIds.length ||
+      !prevEmptySelectedConfigIdsRef.current.every(
+        (id, index) => id === emptySelectedConfigIds[index],
+      )
+    ) {
       form.reset({
         scoreData: getDefaultScoreData({
           scores,
@@ -259,6 +279,26 @@ export function AnnotateDrawerContent({
     onSettled: onSettledUpsert,
   });
 
+  useEffect(() => {
+    if (
+      mutUpdateScores.isLoading ||
+      mutCreateScores.isLoading ||
+      mutDeleteScore.isLoading
+    ) {
+      setShowSaving(true);
+    } else {
+      const timer = setTimeout(() => {
+        setShowSaving(false);
+      }, 300); // Keep saving message for 1 second after loading
+
+      return () => clearTimeout(timer); // Cleanup timer on unmount or when loading state changes
+    }
+  }, [
+    mutUpdateScores.isLoading,
+    mutCreateScores.isLoading,
+    mutDeleteScore.isLoading,
+  ]);
+
   function handleOnCheckedChange(
     values: Record<string, string>[],
     changedValueId?: string,
@@ -329,6 +369,7 @@ export function AnnotateDrawerContent({
               observationId,
               value: newValue,
               stringValue,
+              queueId,
             });
 
             await mutUpdateScores.mutateAsync({
@@ -350,6 +391,7 @@ export function AnnotateDrawerContent({
               observationId,
               value: newValue,
               stringValue,
+              queueId,
             });
 
             await mutCreateScores.mutateAsync({
@@ -392,6 +434,7 @@ export function AnnotateDrawerContent({
           observationId,
           value,
           comment,
+          queueId,
         });
 
         await mutUpdateScores.mutateAsync({
@@ -450,6 +493,7 @@ export function AnnotateDrawerContent({
             comment: score.comment,
             observationId,
             value: Number(field.value),
+            queueId,
           });
 
           await mutUpdateScores.mutateAsync({
@@ -472,6 +516,7 @@ export function AnnotateDrawerContent({
             comment: score.comment,
             observationId,
             value: Number(field.value),
+            queueId,
           });
 
           await mutCreateScores.mutateAsync({
@@ -498,49 +543,67 @@ export function AnnotateDrawerContent({
             description: `Annotate ${observationId ? "observation" : "trace"} with scores to capture human evaluation across different dimensions.`,
             href: "https://langfuse.com/docs/scores/manually",
           }}
+          actionButtons={[
+            <div className="flex items-center justify-end" key="saving-spinner">
+              <div className="mr-1 items-center justify-center">
+                {showSaving ? (
+                  <Loader2 className="h-3 w-3 animate-spin" />
+                ) : (
+                  <Check className="h-3 w-3" />
+                )}
+              </div>
+              <span className="text-xs text-muted-foreground">
+                {showSaving ? "Saving score data" : "Score data saved"}
+              </span>
+            </div>,
+            actionButtons,
+          ]}
         ></Header>
-        <div className="grid grid-flow-col items-center">
-          <MultiSelectKeyValues
-            title="Value"
-            align="end"
-            items="empty scores"
-            className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
-            onValueChange={handleOnCheckedChange}
-            options={configs
-              .filter(
-                (config) =>
-                  !config.isArchived ||
-                  fields.find((field) => field.configId === config.id),
-              )
-              .map((config) => ({
-                key: config.id,
-                value: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
-                disabled: fields.some(
-                  (field) => !!field.scoreId && field.configId === config.id,
-                ),
-                isArchived: config.isArchived,
-              }))}
-            values={fields
-              .filter((field) => !!field.configId)
-              .map((field) => ({
-                value: `${getScoreDataTypeIcon(field.dataType)} ${field.name}`,
-                key: field.configId as string,
-              }))}
-            controlButtons={
-              <CommandItem
-                onSelect={() => {
-                  capture("score_configs:manage_configs_item_click", {
-                    type: type,
-                    source: source,
-                  });
-                  router.push(`/project/${projectId}/settings/scores`);
-                }}
-              >
-                Manage score configs
-              </CommandItem>
-            }
-          />
-        </div>
+
+        {!isSelectHidden && (
+          <div className="grid grid-flow-col items-center">
+            <MultiSelectKeyValues
+              title="Value"
+              align="end"
+              items="empty scores"
+              className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
+              onValueChange={handleOnCheckedChange}
+              options={configs
+                .filter(
+                  (config) =>
+                    !config.isArchived ||
+                    fields.find((field) => field.configId === config.id),
+                )
+                .map((config) => ({
+                  key: config.id,
+                  value: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
+                  disabled: fields.some(
+                    (field) => !!field.scoreId && field.configId === config.id,
+                  ),
+                  isArchived: config.isArchived,
+                }))}
+              values={fields
+                .filter((field) => !!field.configId)
+                .map((field) => ({
+                  value: `${getScoreDataTypeIcon(field.dataType)} ${field.name}`,
+                  key: field.configId as string,
+                }))}
+              controlButtons={
+                <CommandItem
+                  onSelect={() => {
+                    capture("score_configs:manage_configs_item_click", {
+                      type: type,
+                      source: source,
+                    });
+                    router.push(`/project/${projectId}/settings/scores`);
+                  }}
+                >
+                  Manage score configs
+                </CommandItem>
+              }
+            />
+          </div>
+        )}
       </DrawerHeader>
       <Form {...form}>
         <form className="flex flex-col gap-4">
@@ -764,6 +827,7 @@ export function AnnotateDrawerContent({
                                   ) : config.categories &&
                                     renderSelect(categories) ? (
                                     <Select
+                                      name={field.name}
                                       defaultValue={score.stringValue}
                                       disabled={config.isArchived}
                                       onValueChange={handleOnValueChange(

web/src/features/setup/components/SetupPage.tsx

@@ -9,6 +9,7 @@ import {
 import { Button } from "@/src/components/ui/button";
 import { Card } from "@/src/components/ui/card";
 import { NewOrganizationForm } from "@/src/features/organizations/components/NewOrganizationForm";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { NewProjectForm } from "@/src/features/projects/components/NewProjectForm";
 import { useQueryProjectOrOrganization } from "@/src/features/projects/hooks";
 import { ApiKeyRender } from "@/src/features/public-api/components/CreateApiKeyButton";
@@ -48,19 +49,25 @@ export function SetupPage() {
         ? 3
         : 2;
 
-  const hasAnyTrace =
-    api.traces.hasAny.useQuery(
-      { projectId: queryProjectId as string },
-      {
-        enabled: queryProjectId !== undefined && stepInt === 4,
-        refetchInterval: 5000,
-        trpc: {
-          context: {
-            skipBatch: true,
-          },
+  const hasAnyTrace = api.traces.hasAny.useQuery(
+    { projectId: queryProjectId as string },
+    {
+      enabled: queryProjectId !== undefined && stepInt === 4,
+      refetchInterval: 5000,
+      trpc: {
+        context: {
+          skipBatch: true,
         },
       },
-    ).data ?? false;
+    },
+  ).data;
+
+  const capture = usePostHogClientCapture();
+  useEffect(() => {
+    if (hasAnyTrace !== undefined) {
+      capture("onboarding:tracing_check_active", { active: hasAnyTrace });
+    }
+  }, [hasAnyTrace, capture]);
 
   return (
     <div className="mb-12 md:container">
@@ -170,7 +177,7 @@ export function SetupPage() {
                 <Header title="API Keys" level="h3" />
                 <TracingSetup
                   projectId={project.id}
-                  hasAnyTrace={hasAnyTrace}
+                  hasAnyTrace={hasAnyTrace ?? false}
                 />
               </div>
             </div>

web/src/features/setup/components/SetupTracingButton.tsx

@@ -6,6 +6,8 @@ import { setupTracingRoute } from "@/src/features/setup/setupRoutes";
 import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { LockIcon } from "lucide-react";
 import { useRouter } from "next/router";
+import { useEffect } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 const SetupTracingButton = () => {
   const { project } = useQueryProjectOrOrganization();
@@ -25,6 +27,13 @@ const SetupTracingButton = () => {
     },
   );
 
+  const capture = usePostHogClientCapture();
+  useEffect(() => {
+    if (hasAnyTrace !== undefined) {
+      capture("onboarding:tracing_check_active", { active: hasAnyTrace });
+    }
+  }, [hasAnyTrace, capture]);
+
   const hasAccess = useHasProjectAccess({
     projectId: project?.id,
     scope: "apiKeys:create",