chore: release v2.52.0

upgrade

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+{
+	"name": "langfuse dev",
+    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+        "version": "latest",
+        "moby": true
+        }
+    },
+    "forwardPorts": [3000],
+
+    "onCreateCommand": "npm install -g pnpm@8.15.5",
+    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+}

.env.dev.example

@@ -34,10 +34,6 @@ S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 S3_REGION=
 
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Set during docker build of application
 # Used to disable environment verification at build time
 # DOCKER_BUILD=1

.env.local.example

@@ -13,11 +13,6 @@ NEXTAUTH_URL="http://localhost:3000"
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"
 
-
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379

.env.prod.example

@@ -62,7 +62,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
-
+# AUTH_CUSTOM_CLIENT_ID=
+# AUTH_CUSTOM_CLIENT_SECRET=
+# AUTH_CUSTOM_ISSUER=
+# AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -123,10 +127,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 

.github/workflows/pipeline.yml

@@ -40,19 +40,27 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build Web Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./web/Dockerfile
-          push: false
+      - name: Build and run both images from compose
+        run: |
+          docker compose -f docker-compose.build.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Build Worker Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: false
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Ensure no unhealthy status
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
 
   tests-web:
     runs-on: ubuntu-latest

.github/workflows/snyk.yml

@@ -0,0 +1,39 @@
+name: Snyk Container
+on: push
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker compose -f docker-compose.build.yml up -d
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-server
+          args: --file=web/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: web
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-worker
+          args: --file=worker/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: worker

.husky/pre-commit

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Commit will proceed
+    else
+        echo "Commit to $protected_branch branch has been canceled."
+        exit 1 # Commit will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the commit
+exit 0

.husky/pre-push

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to push to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Push will proceed
+    else
+        echo "Push to $protected_branch branch has been canceled."
+        exit 1 # Push will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the push
+exit 0

CONTRIBUTING.md

@@ -151,6 +151,8 @@ Requirements
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
 - Docker to run the database locally
 
+**Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
+
 **Steps**
 
 1. Fork the repository and clone it locally
@@ -191,6 +193,16 @@ Requirements
     pnpm run dev
    ```
 
+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -358,7 +370,9 @@ The background color of the following component will be `hsl(var(--primary))` an
 
 By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
 
-## Using secrets stored in 1Password
+## Maintainers
+
+### Using secrets stored in 1Password
 
 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
 

docker-compose.build.yml

@@ -20,6 +20,11 @@ services:
       - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     build:
@@ -41,6 +46,11 @@ services:
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:7.2.4

ee/package.json

@@ -33,7 +33,7 @@
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

ee/src/sso/index.ts

@@ -11,6 +11,7 @@ import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { env } from "../env";
+import { CustomSSOProvider } from "@langfuse/shared/src/server/auth";
 
 // Local cache for SSO configurations
 let cachedSsoConfigs: {
@@ -173,6 +174,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "custom")
+    return CustomSSOProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -76,12 +76,26 @@ export const CognitoProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CustomProviderSchema = base.extend({
+  authProvider: z.literal("custom"),
+  authConfig: z
+    .object({
+      name: z.string(),
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
 export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -90,6 +104,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   OktaProviderSchema,
   AzureAdProviderSchema,
   CognitoProviderSchema,
+  CustomProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/metrics.yml

@@ -48,12 +48,25 @@ types:
       date: date
       countTraces: integer
       countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
       model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD

fern/apis/server/definition/prompts.yml

@@ -70,14 +70,24 @@ types:
       name: string
       prompt: list<ChatMessage>
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   CreateTextPromptRequest:
     properties:
       name: string
       prompt: string
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   Prompt:
     union:
@@ -89,7 +99,12 @@ types:
       name: string
       version: integer
       config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -31,7 +31,7 @@ service:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
-            type: commons.ScoreSource
+            type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
           operator:
             type: optional<string>
@@ -39,6 +39,9 @@ service:
           value:
             type: optional<double>
             docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
       response: Scores
     get-by-id:
       docs: Get a score

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
       path: /sessions/{sessionId}
       path-parameters:

fern/apis/server/definition/trace.yml

@@ -30,6 +30,7 @@ service:
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only traces newer than this datetime (ISO 8601).

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.47.2",
+  "version": "2.52.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -17,26 +17,36 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "models:migrate": "turbo run models:migrate",
-    "release": "dotenv -e ../.env -- release-it"
+    "release": "dotenv -e ../.env -- release-it",
+    "prepare": "husky"
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
+    "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
+    "husky": "^9.0.11",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.1",
+    "release-it": "^17.3.0",
     "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {
       "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
+      "tagName": "v${version}",
+      "commitArgs": [
+        "--no-verify"
+      ],
+      "pushArgs": [
+        "--no-verify"
+      ]
     },
     "plugins": {
       "@release-it/bumper": {

packages/config-eslint/package.json

@@ -9,7 +9,7 @@
   ],
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",

packages/shared/package.json

@@ -52,6 +52,8 @@
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
     "langchain": "^0.1.36",
+    "lodash": "^4.17.21",
+    "next-auth": "^4.24.7",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -59,17 +61,17 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
     "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
     "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.0.3",
+    "nodemon": "^3.1.3",
     "prettier": "^3.2.5",
     "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",

packages/shared/prisma/generated/types.ts

@@ -32,20 +32,10 @@ export const ScoreSource = {
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const ScoreDataType = {
     CATEGORICAL: "CATEGORICAL",
-    NUMERIC: "NUMERIC"
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
-} as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -162,6 +152,7 @@ export type EvalTemplate = {
     version: number;
     prompt: string;
     model: string;
+    provider: string;
     model_params: unknown;
     vars: Generated<string[]>;
     output_schema: unknown;
@@ -209,8 +200,12 @@ export type LlmApiKeys = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     provider: string;
+    adapter: string;
     display_secret_key: string;
     secret_key: string;
+    base_url: string | null;
+    custom_models: string[];
+    with_default_models: Generated<boolean>;
     project_id: string;
 };
 export type MembershipInvitation = {
@@ -309,14 +304,6 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -470,7 +457,6 @@ export type DB = {
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;

packages/shared/prisma/migrations/20240603212024_dataset_items_add_index_source_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");

packages/shared/prisma/migrations/20240604133338_scores_add_index_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");

packages/shared/prisma/migrations/20240604133339_score_data_type_add_boolean/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';

packages/shared/prisma/migrations/20240604133340_backfill_manual_scores/migration.sql

@@ -0,0 +1,50 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	gen_random_uuid () AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,5 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240606093356_drop_unused_pricings_table/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";

packages/shared/prisma/migrations/20240606133011_remove_trace_fkey_datasetrunitems/migration.sql

@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";

packages/shared/prisma/migrations/20240611105521_llm_api_keys_custom_endpoints/migration.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+  ALTER TABLE "llm_api_keys" 
+    ADD COLUMN "base_url" TEXT,
+    ADD COLUMN "adapter" TEXT,
+    ADD COLUMN "custom_models" TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+    ADD COLUMN "with_default_models" BOOLEAN NOT NULL DEFAULT true;
+
+  UPDATE "llm_api_keys"
+    SET "adapter" = "provider";
+
+  ALTER TABLE "llm_api_keys" 
+    ALTER COLUMN "adapter" SET NOT NULL;
+
+  ALTER TABLE "eval_templates"
+    ADD COLUMN "provider" TEXT;
+  
+  UPDATE "eval_templates"
+    SET "provider" = 'openai';
+  
+  ALTER TABLE "eval_templates"
+    ALTER COLUMN "provider" SET NOT NULL;
+
+COMMIT;

packages/shared/prisma/schema.prisma

@@ -144,9 +144,13 @@ model LlmApiKeys {
     createdAt DateTime @default(now()) @map("created_at")
     updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
 
-    provider         String
-    displaySecretKey String @map("display_secret_key")
-    secretKey        String @map("secret_key")
+    provider          String
+    adapter           String // This controls the interface that is used to connect with the LLM, e.g. 'openai' or 'anthropic'
+    displaySecretKey  String   @map("display_secret_key")
+    secretKey         String   @map("secret_key")
+    baseURL           String?  @map("base_url")
+    customModels      String[] @map("custom_models")
+    withDefaultModels Boolean  @default(true) @map("with_default_models")
 
     projectId String  @map("project_id")
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -233,9 +237,8 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetRunItems DatasetRunItems[]
-    DatasetItem     DatasetItem[]
-    JobExecution    JobExecution[]
+    DatasetItem  DatasetItem[]
+    JobExecution JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -298,21 +301,20 @@ model Observation {
 
     // GENERATION ONLY
     model               String?
-    internalModel       String?           @map("internal_model")
+    internalModel       String?       @map("internal_model")
     modelParameters     Json?
     input               Json?
     output              Json?
-    promptTokens        Int               @default(0) @map("prompt_tokens")
-    completionTokens    Int               @default(0) @map("completion_tokens")
-    totalTokens         Int               @default(0) @map("total_tokens")
+    promptTokens        Int           @default(0) @map("prompt_tokens")
+    completionTokens    Int           @default(0) @map("completion_tokens")
+    totalTokens         Int           @default(0) @map("total_tokens")
     unit                String?
-    inputCost           Decimal?          @map("input_cost")
-    outputCost          Decimal?          @map("output_cost")
-    totalCost           Decimal?          @map("total_cost")
-    completionStartTime DateTime?         @map("completion_start_time")
-    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    inputCost           Decimal?      @map("input_cost")
+    outputCost          Decimal?      @map("output_cost")
+    totalCost           Decimal?      @map("total_cost")
+    completionStartTime DateTime?     @map("completion_start_time")
+    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
-    datasetRunItems     DatasetRunItems[]
 
     promptId String? @map("prompt_id")
     prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@ -419,6 +421,7 @@ model Score {
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
+    @@index([projectId, name])
     @@index([authorUserId])
     @@index([configId])
     @@index([traceId], type: Hash)
@@ -463,29 +466,7 @@ model ScoreConfig {
 enum ScoreDataType {
     CATEGORICAL
     NUMERIC
-}
-
-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
-}
-
-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
+    BOOLEAN
 }
 
 model CronJobs {
@@ -532,6 +513,7 @@ model DatasetItem {
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
     @@index([createdAt])
@@ -563,17 +545,15 @@ model DatasetRuns {
 }
 
 model DatasetRunItems {
-    id            String       @id @default(cuid())
-    datasetRunId  String       @map("dataset_run_id")
-    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String       @map("dataset_item_id")
-    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime     @default(now()) @map("created_at")
-    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String      @map("trace_id")
+    observationId String?     @map("observation_id")
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)
@@ -688,6 +668,7 @@ model EvalTemplate {
     version          Int
     prompt           String
     model            String
+    provider         String
     modelParams      Json               @map("model_params")
     vars             String[]           @default([])
     outputSchema     Json               @map("output_schema")

packages/shared/prisma/seed.ts

@@ -4,6 +4,7 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
+  ScoreDataType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -39,6 +40,7 @@ async function main() {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
 
@@ -144,6 +146,11 @@ async function main() {
       });
     }
 
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -157,7 +164,8 @@ async function main() {
       colorTags,
       project1,
       project2,
-      promptIds
+      promptIds,
+      configIdsAndNames
     );
 
     console.log(
@@ -176,6 +184,7 @@ async function main() {
           secretKey: encrypt(OPENAI_API_KEY),
           displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
           provider: "openai",
+          adapter: "openai",
         },
       });
     } else {
@@ -201,6 +210,7 @@ async function main() {
           "Please evaluate the toxicity of the following text {{input}} {{output}}",
         model: "gpt-3.5-turbo",
         vars: ["input", "output"],
+        provider: "openai",
         outputSchema: {
           score: "provide a score between 0 and 1",
           reasoning: "one sentence reasoning for the score",
@@ -453,13 +463,15 @@ function createObjects(
   colorTags: (string | null)[],
   project1: Project,
   project2: Project,
-  promptIds: Map<string, string[]>
+  promptIds: Map<string, string[]>,
+  configIdsAndNames: Map<string, { name: string; id: string }[]>
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
   const scores: Prisma.ScoreCreateManyInput[] = [];
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
+  const configs: Prisma.ScoreConfigCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -511,17 +523,28 @@ function createObjects(
 
     traces.push(trace);
 
+    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const randomIndex = Math.floor(Math.random() * 3);
+    const config =
+      configArray.length >= randomIndex - 1 && configArray[randomIndex];
+    const { name: annotationScoreName, id: configId } = config || {
+      name: "manual-score",
+      id: undefined,
+    };
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
             {
               traceId: trace.id,
-              name: "manual-score",
+              name: annotationScoreName,
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
+              dataType: ScoreDataType.NUMERIC,
+              ...(configId ? { configId } : {}),
             },
           ]
         : []),
@@ -534,6 +557,7 @@ function createObjects(
               timestamp: traceTs,
               source: ScoreSource.API,
               projectId,
+              dataType: ScoreDataType.NUMERIC,
             },
           ]
         : []),
@@ -772,6 +796,7 @@ function createObjects(
     traces,
     observations,
     scores,
+    configs,
     sessions: uniqueSessions,
     events,
   };
@@ -954,3 +979,80 @@ async function generatePrompts(project: Project) {
   }
   return promptIds;
 }
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
+    new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    })
+  );
+  return projectIdsToConfigs;
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: { name: string; id: string }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({ name: config.name, id: config.id });
+  }
+
+  return configNameAndId;
+}

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, UIModelParams } from "../..";
+import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,38 +77,15 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels: UIModelParams[] = [
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-3.5-turbo", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4-turbo-preview", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4o", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-] as const;
-
-export const EvalModelNames = z.enum([
-  "gpt-3.5-turbo",
-  "gpt-4-turbo-preview",
-  "gpt-4o",
-]);
+export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
+  [LLMAdapter.OpenAI]: [
+    "gpt-4o",
+    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+  ] as const,
+  [LLMAdapter.Anthropic]: [] as const,
+  [LLMAdapter.Azure]: [] as const,
+};
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/features/ingestion/types.ts

@@ -1,9 +1,9 @@
 import lodash from "lodash";
 import { z } from "zod";
 
-import { ModelUsageUnit } from "@langfuse/shared";
-import { NonEmptyString, jsonSchema } from "@/src/utils/zod";
-import { ObservationLevel } from "@langfuse/shared";
+import { NonEmptyString, jsonSchema } from "../../utils/zod";
+import { ModelUsageUnit } from "../../constants";
+import { ObservationLevel } from "@prisma/client";
 
 export const Usage = z.object({
   input: z.number().int().nullish(),
@@ -109,7 +109,7 @@ export const CreateGenerationBody = CreateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -131,7 +131,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -195,7 +195,7 @@ export const LegacyGenerationsCreateSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -219,7 +219,7 @@ export const LegacyGenerationPatchSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -243,7 +243,7 @@ export const LegacyObservationBody = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   input: jsonSchema.nullish(),

packages/shared/src/filterToPrisma.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import {
   ColumnDefinition,
-  type TableNames as TableName,
+  type TableNames,
 } from "./interfaces/tableDefinition";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,7 +43,7 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName
+  table: TableNames
 ) => {
   return column.name === "type" &&
     (table === "observations" ||

packages/shared/src/index.ts

@@ -9,6 +9,9 @@ export * from "./filterToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
 export * from "./observationsTable";
+export * from "./features/ingestion/types";
+export * from "./utils/zod";
+export * from "./utils/json";
 export { env } from "./env";
 
 // llm api

packages/shared/src/interfaces/tableDefinition.ts

@@ -42,6 +42,7 @@ export const tableNames = [
   "traces_parent_observation_scores",
   "sessions",
   "prompts",
+  "users",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/server/auth.ts

@@ -1,6 +1,7 @@
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
 
 export function generateSecretKey() {
   return `sk-lf-${randomUUID()}`;
@@ -48,3 +49,33 @@ export function createShaHash(privateKey: string, salt: string): string {
 
   return hash;
 }
+
+export interface CustomSSOUser extends Record<string, any> {
+  email: string;
+  id: string;
+  name: string;
+  verified: boolean;
+}
+
+export function CustomSSOProvider<P extends CustomSSOUser>(
+  options: OAuthUserConfig<P>
+): OAuthConfig<P> {
+  return {
+    id: "custom",
+    name: "CustomSSOProvider",
+    type: "oauth",
+    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
+    authorization: { params: { scope: "openid email profile" } },
+    checks: ["pkce", "state"],
+    idToken: true,
+    profile(profile) {
+      return {
+        id: profile.sub,
+        name: profile.name,
+        email: profile.email,
+        image: null,
+      };
+    },
+    options,
+  };
+}

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -17,7 +17,7 @@ import {
   ChatMessageRole,
   LLMFunctionCall,
   ModelParams,
-  ModelProvider,
+  LLMAdapter,
 } from "./types";
 import zodToJsonSchema from "zod-to-json-schema";
 import { JsonOutputFunctionsParser } from "langchain/output_parsers";
@@ -27,7 +27,9 @@ type LLMCompletionParams = {
   modelParams: ModelParams;
   functionCall?: LLMFunctionCall;
   callbacks?: BaseCallbackHandler[];
+  baseURL?: string;
   apiKey?: string;
+  maxRetries?: number;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -57,7 +59,16 @@ export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   // the apiKey must never be printed to the console
-  const { messages, modelParams, streaming, callbacks, apiKey } = params;
+  const {
+    messages,
+    modelParams,
+    streaming,
+    callbacks,
+    apiKey,
+    baseURL,
+    maxRetries,
+  } = params;
+
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -68,16 +79,18 @@ export async function fetchLLMCompletion(
   });
 
   let chatModel: ChatOpenAI | ChatAnthropic;
-  if (modelParams.provider === ModelProvider.Anthropic) {
+  if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
+      anthropicApiUrl: baseURL,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      clientOptions: { maxRetries },
     });
-  } else if (modelParams.provider === ModelProvider.OpenAI) {
+  } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
       openAIApiKey: apiKey,
       modelName: modelParams.model,
@@ -85,15 +98,29 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      maxRetries,
+      configuration: {
+        baseURL,
+      },
+    });
+  } else if (modelParams.adapter === LLMAdapter.Azure) {
+    chatModel = new ChatOpenAI({
+      azureOpenAIApiKey: apiKey,
+      azureOpenAIBasePath: baseURL,
+      azureOpenAIApiDeploymentName: modelParams.model,
+      azureOpenAIApiVersion: "2024-02-01",
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+      maxRetries,
     });
   } else {
     // eslint-disable-next-line no-unused-vars
-    const _exhaustiveCheck: never = modelParams;
+    const _exhaustiveCheck: never = modelParams.adapter;
     throw new Error("This model provider is not supported.");
   }
 
-  console.log("Making LLM call with params: ", modelParams);
-
   if (params.functionCall) {
     const functionCallingModel = chatModel.bind({
       functions: [

packages/shared/src/server/llm/types.ts

@@ -1,3 +1,4 @@
+import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
@@ -9,9 +10,10 @@ export type ChatMessage = {
 
 export type ChatMessageWithId = ChatMessage & { id: string };
 
-export enum ModelProvider {
+export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
+  Azure = "azure",
 }
 
 export enum ChatMessageRole {
@@ -19,32 +21,30 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
 }
-export type ModelParams = AnthropicModelParams | OpenAIModelParams;
+
+export type ModelParams = {
+  provider: string;
+  adapter: LLMAdapter;
+  model: string;
+} & ModelConfig;
 
 type RecordWithEnabledFlag<T> = {
   [K in keyof T]: { value: T[K]; enabled: boolean };
 };
 export type UIModelParams = RecordWithEnabledFlag<
-  Required<AnthropicModelParams | OpenAIModelParams> & {
+  Required<ModelParams> & {
     maxTemperature: number;
   }
 >;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
-
 export const ZodModelConfig = z.object({
   max_tokens: z.coerce.number().optional(),
   temperature: z.coerce.number().optional(),
   top_p: z.coerce.number().optional(),
 });
 
-// OpenAI
-export type OpenAIModelParams = {
-  provider: ModelProvider.OpenAI;
-  model: OpenAIModel;
-} & ModelConfig;
-
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-05-13",
@@ -64,12 +64,6 @@ export const openAIModels = [
 
 export type OpenAIModel = (typeof openAIModels)[number];
 
-// Anthropic
-export type AnthropicModelParams = {
-  provider: ModelProvider.Anthropic;
-  model: AnthropicModel;
-} & ModelConfig;
-
 export const anthropicModels = [
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
@@ -81,8 +75,9 @@ export const anthropicModels = [
 
 export type AnthropicModel = (typeof anthropicModels)[number];
 export const supportedModels = {
-  [ModelProvider.Anthropic]: anthropicModels,
-  [ModelProvider.OpenAI]: openAIModels,
+  [LLMAdapter.Anthropic]: anthropicModels,
+  [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.Azure]: [],
 } as const;
 
 export type LLMFunctionCall = {
@@ -90,3 +85,26 @@ export type LLMFunctionCall = {
   description: string;
   parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
 };
+
+export const LLMApiKeySchema = z
+  .object({
+    id: z.string(),
+    projectId: z.string(),
+    createdAt: z.date(),
+    updatedAt: z.date(),
+    adapter: z.nativeEnum(LLMAdapter),
+    provider: z.string(),
+    displaySecretKey: z.string(),
+    secretKey: z.string(),
+    baseURL: z.string().nullable(),
+    customModels: z.array(z.string()),
+    withDefaultModels: z.boolean(),
+  })
+  // strict mode to prevent extra keys. Thorws error otherwise
+  // https://github.com/colinhacks/zod?tab=readme-ov-file#strict
+  .strict();
+
+export type LLMApiKey =
+  z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
+    ? z.infer<typeof LLMApiKeySchema>
+    : never;

packages/shared/src/types.ts

@@ -30,4 +30,5 @@ export type TableName =
   | "sessions"
   | "scores"
   | "prompts"
-  | "dashboard";
+  | "dashboard"
+  | "users";

packages/shared/src/utils/json.ts

@@ -1,6 +1,6 @@
-import { type jsonSchema } from "@/src/utils/zod";
 import { type z } from "zod";
 import lodash from "lodash";
+import { jsonSchema } from "./zod";
 
 export const parseJson = (input: string) => {
   try {
@@ -37,7 +37,7 @@ export function deepParseJson(json: unknown): unknown {
         // Ensure we only iterate over the object's own properties
         if (Object.prototype.hasOwnProperty.call(json, key)) {
           (json as Record<string, unknown>)[key] = deepParseJson(
-            (json as Record<string, unknown>)[key],
+            (json as Record<string, unknown>)[key]
           );
         }
       }
@@ -50,7 +50,7 @@ export function deepParseJson(json: unknown): unknown {
 
 export const mergeJson = (
   json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
+  json2?: z.infer<typeof jsonSchema>
 ) => {
   if (json1 === undefined) {
     return json2;

packages/shared/src/utils/zod.ts

@@ -26,7 +26,7 @@ const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
     nestedLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 // Root schema that does not allow nulls at the root level
@@ -35,17 +35,17 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
     rootLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().default(1)
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().lte(100).default(50)
   ),
 };
 

web/Dockerfile

@@ -1,11 +1,11 @@
-FROM node:20-alpine AS alpine
+FROM node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
+RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.47.2",
+  "version": "2.52.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -27,7 +27,7 @@
     "@codemirror/lang-json": "^6.0.1",
     "@codemirror/lint": "^6.8.0",
     "@headlessui/react": "1.7.18",
-    "@headlessui/tailwindcss": "0.2.0",
+    "@headlessui/tailwindcss": "0.2.1",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
     "@langchain/anthropic": "^0.1.21",
@@ -56,9 +56,10 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
+    "@radix-ui/react-toggle-group": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.18",
-    "@react-email/render": "^0.0.14",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
@@ -70,14 +71,13 @@
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
-    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
@@ -88,7 +88,7 @@
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.4",
+    "dompurify": "^3.1.5",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
@@ -116,6 +116,7 @@
     "tailwindcss-animate": "^1.0.7",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
@@ -123,21 +124,21 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
     "@tailwindcss/forms": "^0.5.7",
-    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/jest-dom": "^6.4.6",
     "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
@@ -148,7 +149,7 @@
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.14",
-    "tailwindcss": "^3.4.3",
+    "tailwindcss": "^3.4.4",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.2",

web/public/generated/api/openapi.yml

@@ -997,9 +997,10 @@ paths:
         - name: source
           in: query
           description: Retrieve only scores from a specific source.
-          required: true
+          required: false
           schema:
             $ref: '#/components/schemas/ScoreSource'
+            nullable: true
         - name: operator
           in: query
           description: Retrieve only scores with <operator> value.
@@ -1015,6 +1016,13 @@ paths:
             type: number
             format: double
             nullable: true
+        - name: scoreIds
+          in: query
+          description: Comma-separated list of score IDs to limit the results to.
+          required: false
+          schema:
+            type: string
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1137,7 +1145,10 @@ paths:
       security: *ref_0
   /api/public/sessions/{sessionId}:
     get:
-      description: Get a session
+      description: >-
+        Get a session. Please note that `traces` on this endpoint are not
+        paginated, if you plan to fetch large sessions, consider `GET
+        /api/public/traces?sessionId=<sessionId>`
       operationId: sessions_get
       tags:
         - Sessions
@@ -1262,6 +1273,12 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: sessionId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
         - name: fromTimestamp
           in: query
           description: Retrieve only traces newer than this datetime (ISO 8601).
@@ -2499,6 +2516,7 @@ components:
         totalCost:
           type: number
           format: double
+          description: Total model cost in USD
         usage:
           type: array
           items:
@@ -2521,14 +2539,28 @@ components:
           nullable: true
         inputUsage:
           type: integer
+          description: Total number of generation input units (e.g. tokens)
         outputUsage:
           type: integer
+          description: Total number of generation output units (e.g. tokens)
         totalUsage:
           type: integer
+          description: Total number of generation total units (e.g. tokens)
+        countTraces:
+          type: integer
+        countObservations:
+          type: integer
+        totalCost:
+          type: number
+          format: double
+          description: Total model cost in USD
       required:
         - inputUsage
         - outputUsage
         - totalUsage
+        - countTraces
+        - countObservations
+        - totalCost
     Observations:
       title: Observations
       type: object
@@ -2654,6 +2686,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2672,6 +2711,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2713,11 +2759,20 @@ components:
           type: array
           items:
             type: string
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          description: >-
+            List of tags. Used to filter via UI and API. The same across
+            versions of a prompt.
       required:
         - name
         - version
         - config
         - labels
+        - tags
     ChatMessage:
       title: ChatMessage
       type: object

web/public/generated/postman/collection.json

@@ -699,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -755,7 +755,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -804,6 +804,11 @@
                   "key": "value",
                   "value": "",
                   "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "scoreIds",
+                  "value": "",
+                  "description": "Comma-separated list of score IDs to limit the results to."
                 }
               ],
               "variable": []
@@ -890,7 +895,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a session",
+            "description": "Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`",
             "url": {
               "raw": "{{baseUrl}}/api/public/sessions/:sessionId",
               "host": [
@@ -963,7 +968,7 @@
           "request": {
             "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -993,6 +998,11 @@
                   "value": "",
                   "description": null
                 },
+                {
+                  "key": "sessionId",
+                  "value": "",
+                  "description": null
+                },
                 {
                   "key": "fromTimestamp",
                   "value": "",

web/sentry.server.config.ts

@@ -2,7 +2,7 @@ import { prisma } from "@langfuse/shared/src/db";
 import * as Sentry from "@sentry/nextjs";
 import { ProfilingIntegration } from "@sentry/profiling-node";
 import type { SamplingContext, TransactionEvent } from "@sentry/types";
-import { jsonSchema } from "@/src/utils/zod";
+import { jsonSchema } from "@langfuse/shared";
 
 if (process.env.NEXT_PUBLIC_SENTRY_DSN)
   Sentry.init({
@@ -18,7 +18,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.3;
+        return 0.1;
       }
       if (
         samplingContext.request &&
@@ -27,12 +27,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.transactionContext.status !== "ok" &&
         samplingContext.transactionContext.status !== "unauthenticated"
       ) {
-        return 1;
+        return 0.1;
       }
-      return 0.1;
+      return 0.01;
     },
 
-    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

web/src/__tests__/llm-api-key.servertest.ts

@@ -3,7 +3,7 @@
 
 import type { Session } from "next-auth";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { ModelProvider } from "@langfuse/shared";
+import { LLMAdapter } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
@@ -38,11 +38,20 @@ describe("llmApiKey.all RPC", () => {
 
   it("should create an llm api key", async () => {
     const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;
 
     await caller.llmApiKey.create({
       projectId,
       secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
     });
 
     const llmApiKeys = await prisma.llmApiKeys.findMany({
@@ -55,18 +64,31 @@ describe("llmApiKey.all RPC", () => {
     expect(llmApiKeys[0].projectId).toBe(projectId);
     expect(llmApiKeys[0].secretKey).not.toBeNull();
     expect(llmApiKeys[0].secretKey).not.toEqual(secret);
-    expect(llmApiKeys[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
     // this has to be 3 dots and the last 4 characters of the secret
     expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
   });
 
   it("should create and get an llm api key", async () => {
     const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;
 
     await caller.llmApiKey.create({
       projectId,
       secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
     });
 
     const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
@@ -77,16 +99,23 @@ describe("llmApiKey.all RPC", () => {
 
     expect(dbLlmApiKeys.length).toBe(1);
 
-    const llmApiKeys = await caller.llmApiKey.all({
+    const { data: llmApiKeys } = await caller.llmApiKey.all({
       projectId,
     });
 
-    expect(llmApiKeys.data.length).toBe(1);
-    expect(llmApiKeys.data[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys.length).toBe(1);
+    expect(llmApiKeys[0].projectId).toBe(projectId);
+    expect(llmApiKeys[0].secretKey).not.toBeNull();
+    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
     // this has to be 3 dots and the last 4 characters of the secret
-    expect(llmApiKeys.data[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
+    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
 
     // response must not contain the secret key itself
-    expect(llmApiKeys.data[0]).not.toHaveProperty("secretKey");
+    expect(llmApiKeys[0]).not.toHaveProperty("secretKey");
   });
 });

web/src/__tests__/metrics-api.servertest.ts

@@ -68,12 +68,18 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 333,
         outputUsage: 0,
         totalUsage: 333,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
       {
         model: "modelC",
         inputUsage: 666,
         outputUsage: 777,
         totalUsage: 1443,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 1024.22,
       },
     ]);
 
@@ -88,6 +94,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 100,
         outputUsage: 200,
         totalUsage: 300,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
     ]);
   });

web/src/__tests__/prompts.v2.servertest.ts

@@ -680,6 +680,71 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(fetchedPrompt.body.createdBy).toBe("API");
       expect(fetchedPrompt.body.config).toEqual({});
     });
+
+    it("should update tags across versions", async () => {
+      const promptName = "prompt-name" + nanoid();
+
+      const createPromptVersion = async (tags?: string[]) => {
+        await makeAPICall("POST", baseURI, {
+          name: promptName,
+          prompt: "This is a test prompt",
+          type: PromptType.Text,
+          ...(tags !== undefined && { tags: tags }),
+        });
+      };
+
+      const fetchPromptVersion = async (version: number) => {
+        const fetchedPrompt = await makeAPICall(
+          "GET",
+          `${baseURI}/${promptName}?version=${version}`,
+          undefined,
+        );
+        expect(fetchedPrompt.status).toBe(200);
+        if (!isPrompt(fetchedPrompt.body)) {
+          throw new Error("Expected body to be a prompt");
+        }
+        return fetchedPrompt.body;
+      };
+
+      // Create version 1 with ["tag"]
+      await createPromptVersion(["tag"]);
+      let fetchedPrompt1 = await fetchPromptVersion(1);
+      expect(fetchedPrompt1.tags).toEqual(["tag"]);
+      expect(fetchedPrompt1.version).toBe(1);
+
+      // Create version 2 with no tags provided (should use tags from version 1)
+      await createPromptVersion();
+      let fetchedPrompt2 = await fetchPromptVersion(2);
+      expect(fetchedPrompt2.tags).toEqual(["tag"]);
+      expect(fetchedPrompt2.version).toBe(2);
+
+      // Create version 3 with ["tag1", "tag2", "tag3"] (should update tags across versions)
+      await createPromptVersion(["tag1", "tag2", "tag3"]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      let fetchedPrompt3 = await fetchPromptVersion(3);
+      expect(fetchedPrompt1.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt3.version).toBe(3);
+
+      // remove tags
+      await createPromptVersion([]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      fetchedPrompt3 = await fetchPromptVersion(3);
+      let fetchedPrompt4 = await fetchPromptVersion(4);
+      expect(fetchedPrompt1.tags).toEqual([]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual([]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual([]);
+      expect(fetchedPrompt3.version).toBe(3);
+      expect(fetchedPrompt4.tags).toEqual([]);
+      expect(fetchedPrompt4.version).toBe(4);
+    });
   });
 
   describe("when fetching a prompt list", () => {

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
+          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id  WHERE  t."project_id" = `,
           ";",
         ],
       } as const,

web/src/__tests__/scores.servertest.ts

@@ -591,5 +591,37 @@ describe("/api/public/scores API Endpoint", () => {
         message: "Invalid request data",
       });
     });
+
+    it("should filter scores by score IDs", async () => {
+      const getScore = await makeAPICall<{
+        data: [
+          {
+            id: string;
+            name: string;
+            value: number;
+          },
+        ];
+        meta: object;
+      }>("GET", `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`);
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
   });
 });

web/src/__tests__/traces.servertest.ts

@@ -238,4 +238,62 @@ describe("/api/public/traces API Endpoint", () => {
     expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
     expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
   });
+
+  it("should filter traces by session ID", async () => {
+    const sessionId = "test-session-id";
+    const anotherSessionId = "another-session-id";
+
+    // Create traces with different session IDs
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-1",
+      name: "test-trace-1",
+      sessionId,
+      userId: "user-1",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-2",
+      name: "test-trace-2",
+      sessionId: anotherSessionId,
+      userId: "user-2",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    // Filter by session ID
+    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${sessionId}`,
+    );
+
+    expect(tracesBySessionId.status).toBe(200);
+    expect(tracesBySessionId.body.data).toHaveLength(1);
+    expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
+
+    // Filter by another session ID
+    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${anotherSessionId}`,
+    );
+
+    expect(tracesByAnotherSessionId.status).toBe(200);
+    expect(tracesByAnotherSessionId.body.data).toHaveLength(1);
+    expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
+
+    // Filter by non-existent session ID
+    const tracesByNonExistentSessionId =
+      await makeAPICall<GetTracesAPIResponse>(
+        "GET",
+        `/api/public/traces?sessionId=non-existent-session-id`,
+      );
+
+    expect(tracesByNonExistentSessionId.status).toBe(200);
+    expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);
+  });
 });

web/src/__tests__/zod.servertest.ts

@@ -1,4 +1,4 @@
-import { paginationZod } from "@/src/utils/zod";
+import { paginationZod } from "@langfuse/shared";
 import { ZodError } from "zod";
 
 // Create test cases

web/src/components/LangfuseLogo.tsx

@@ -1,8 +1,105 @@
-import { AlertTriangle, Check } from "lucide-react";
+import {
+  AlertTriangle,
+  Check,
+  Github,
+  HardDriveDownload,
+  Newspaper,
+} from "lucide-react";
 
 import { VERSION } from "@/src/constants";
 import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
+import {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+} from "@/src/components/ui/dropdown-menu";
+import { ArrowUp } from "lucide-react";
+import { api } from "@/src/utils/api";
+import { Button } from "@/src/components/ui/button";
+import Link from "next/link";
+
+const VersionLabel = ({ className }: { className?: string }) => {
+  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
+    refetchOnMount: false,
+    refetchOnWindowFocus: false,
+    refetchOnReconnect: false,
+    enabled: !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION, // do not check for updates on Langfuse Cloud
+    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
+  });
+
+  const hasUpdate =
+    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+    checkUpdate.data &&
+    checkUpdate.data.updateType;
+
+  const color =
+    checkUpdate.data?.updateType === "major"
+      ? "text-dark-red"
+      : checkUpdate.data?.updateType === "minor"
+        ? "text-dark-yellow"
+        : undefined;
+
+  return (
+    <DropdownMenu>
+      <DropdownMenuTrigger asChild>
+        <Button variant="ghost" size="xs" className={className}>
+          {VERSION}
+          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
+        </Button>
+      </DropdownMenuTrigger>
+      <DropdownMenuContent>
+        {hasUpdate ? (
+          <>
+            <DropdownMenuLabel>
+              New {checkUpdate.data?.updateType} version:{" "}
+              {checkUpdate.data?.latestRelease}
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ? (
+          <>
+            <DropdownMenuLabel>This is the latest release</DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : null}
+
+        <DropdownMenuItem asChild>
+          <Link
+            href="https://github.com/langfuse/langfuse/releases"
+            target="_blank"
+          >
+            <Github size={16} className="mr-2" />
+            Releases
+          </Link>
+        </DropdownMenuItem>
+        <DropdownMenuItem asChild>
+          <Link href="https://langfuse.com/changelog" target="_blank">
+            <Newspaper size={16} className="mr-2" />
+            Changelog
+          </Link>
+        </DropdownMenuItem>
+        {hasUpdate && (
+          <>
+            <DropdownMenuSeparator />
+            <DropdownMenuItem asChild>
+              <Link
+                href="https://langfuse.com/docs/deployment/self-host#update"
+                target="_blank"
+              >
+                <HardDriveDownload size={16} className="mr-2" />
+                Update
+              </Link>
+            </DropdownMenuItem>
+          </>
+        )}
+      </DropdownMenuContent>
+    </DropdownMenu>
+  );
+};
 
 export const LangfuseIcon = ({
   size = 32,
@@ -65,22 +162,12 @@ export const LangfuseLogo = ({
       <span
         className={cn(
           "ml-2 font-mono font-semibold",
-          size === "sm" ? "text-sm" : "text-xl",
+          size === "sm" ? "text-sm" : "text-lg",
         )}
       >
         Langfuse
       </span>
-      {version && (
-        <a
-          href="https://github.com/langfuse/langfuse/releases"
-          target="_blank"
-          rel="noopener"
-          title="View releases on GitHub"
-          className="ml-2 text-xs text-muted-foreground"
-        >
-          {VERSION}
-        </a>
-      )}
+      {version && <VersionLabel className="ml-2" />}
     </div>
   </div>
 );

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -3,7 +3,7 @@ import Link from "next/link";
 import { Label } from "@/src/components/ui/label";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
-import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { type UIModelParams } from "@langfuse/shared";
 import { ArrowTopRightIcon } from "@radix-ui/react-icons";
 
 export const LLMApiKeyComponent = (p: {
@@ -39,11 +39,7 @@ export const LLMApiKeyComponent = (p: {
     );
   }
 
-  const model = p.modelParams.model.value;
-  const modelProvider = Object.entries(supportedModels).find((providerData) =>
-    (providerData[1] as any as string[]).includes(model),
-  )?.[0];
-
+  const modelProvider = p.modelParams.provider.value;
   const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
 
   return (
@@ -51,9 +47,11 @@ export const LLMApiKeyComponent = (p: {
       <Label className="text-xs font-semibold">API key</Label>
       <div>
         {apiKey ? (
-          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
-            {apiKey.displaySecretKey}
-          </span>
+          <Link href={`/project/${p.projectId}/settings`}>
+            <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+              {apiKey.displaySecretKey}
+            </span>
+          </Link>
         ) : undefined}
       </div>
       {/* Custom form message to include a link to the already existing prompt */}

web/src/components/ModelParameters/index.tsx

@@ -8,11 +8,12 @@ import {
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
 import { Switch } from "@/src/components/ui/switch";
+import { CreateLLMApiKeyDialog } from "@/src/features/public-api/components/CreateLLMApiKeyDialog";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { cn } from "@/src/utils/tailwind";
 import {
-  ModelProvider,
-  supportedModels,
+  type LLMAdapter,
+  type supportedModels,
   type UIModelParams,
 } from "@langfuse/shared";
 
@@ -20,7 +21,8 @@ import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
 
 export type ModelParamsContext = {
   modelParams: UIModelParams;
-  availableModels?: UIModelParams[];
+  availableProviders: string[];
+  availableModels: string[];
   updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
     value: UIModelParams[Key]["value"],
@@ -31,6 +33,7 @@ export type ModelParamsContext = {
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
+  availableProviders,
   availableModels,
   updateModelParamValue,
   setModelParamEnabled,
@@ -43,76 +46,71 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-4">
-        <ModelParamsSelect
-          title="Provider"
-          modelParamsKey="provider"
-          disabled={formDisabled}
-          value={modelParams.provider.value}
-          options={
-            availableModels
-              ? [...new Set(availableModels.map((m) => m.provider.value))]
-              : Object.values(ModelProvider)
-          }
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSelect
-          title="Model name"
-          modelParamsKey="model"
-          disabled={formDisabled}
-          value={modelParams.model.value}
-          options={Object.values(
-            availableModels
-              ? availableModels
-                  .filter(
-                    (m) => m.provider.value === modelParams.provider.value,
-                  )
-                  .map((m) => m.model.value)
-              : supportedModels[modelParams.provider.value],
-          )}
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Temperature"
-          modelParamsKey="temperature"
-          formDisabled={formDisabled}
-          enabled={modelParams.temperature.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.temperature.value}
-          min={0}
-          max={modelParams.maxTemperature.value}
-          step={0.01}
-          tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Output token limit"
-          modelParamsKey="max_tokens"
-          formDisabled={formDisabled}
-          enabled={modelParams.max_tokens.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.max_tokens.value}
-          min={1}
-          max={4096}
-          step={1}
-          tooltip="The maximum number of tokens that can be generated in the chat completion."
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Top P"
-          modelParamsKey="top_p"
-          formDisabled={formDisabled}
-          enabled={modelParams.top_p.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.top_p.value}
-          min={0}
-          max={1}
-          step={0.01}
-          tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
-          updateModelParam={updateModelParamValue}
-        />
-        <LLMApiKeyComponent {...{ projectId, modelParams }} />
-      </div>
+      {availableProviders.length === 0 ? (
+        <>
+          <p className="text-sm">No LLM API key set in project.</p>
+          <CreateLLMApiKeyDialog />
+        </>
+      ) : (
+        <div className="space-y-4">
+          <ModelParamsSelect
+            title="Provider"
+            modelParamsKey="provider"
+            disabled={formDisabled}
+            value={modelParams.provider.value}
+            options={availableProviders}
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSelect
+            title="Model name"
+            modelParamsKey="model"
+            disabled={formDisabled}
+            value={modelParams.model.value}
+            options={availableModels}
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Temperature"
+            modelParamsKey="temperature"
+            formDisabled={formDisabled}
+            enabled={modelParams.temperature.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.temperature.value}
+            min={0}
+            max={modelParams.maxTemperature.value}
+            step={0.01}
+            tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Output token limit"
+            modelParamsKey="max_tokens"
+            formDisabled={formDisabled}
+            enabled={modelParams.max_tokens.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.max_tokens.value}
+            min={1}
+            max={4096}
+            step={1}
+            tooltip="The maximum number of tokens that can be generated in the chat completion."
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Top P"
+            modelParamsKey="top_p"
+            formDisabled={formDisabled}
+            enabled={modelParams.top_p.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.top_p.value}
+            min={0}
+            max={1}
+            step={0.01}
+            tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
+            updateModelParam={updateModelParamValue}
+          />
+          <LLMApiKeyComponent {...{ projectId, modelParams }} />
+        </div>
+      )}
     </div>
   );
 };
@@ -148,7 +146,7 @@ const ModelParamsSelect = ({
         onValueChange={(value) =>
           updateModelParam(
             modelParamsKey,
-            value as (typeof supportedModels)[ModelProvider][number],
+            value as (typeof supportedModels)[LLMAdapter][number],
           )
         }
         value={value}

web/src/components/Support.tsx

@@ -67,7 +67,7 @@ const supportChannels = [
     title: "Slack Connect",
     description: "Get a dedicated support channel for you and your team.",
     href: "mailto:help@langfuse.com?subject=Slack%20Connect%20Request&body=I'd%20like%20to%20request%20a%20dedicated%20Slack%20Connect%20channel%20for%20me%20and%20my%20team.%0D%0A%0D%0AUsers%20(emails)%20to%20include%20besides%20mine%3A%0D%0A%0D%0A",
-    buttonText: "Request Slack Connect (via Email)",
+    buttonText: "Request via Email",
   },
 ];
 

web/src/components/deleteButton.tsx

@@ -53,16 +53,13 @@ export function DeleteButton({
     },
   });
 
-  if (!hasAccess) {
-    return null;
-  }
-
   return (
     <Popover key={itemId}>
       <PopoverTrigger asChild>
         <Button
           variant={isTableAction ? "ghost" : "outline"}
           size={isTableAction ? "xs" : "icon"}
+          disabled={!hasAccess}
           onClick={() =>
             type === "trace"
               ? capture("trace:delete_form_open", {

web/src/components/grouped-score-badge.tsx

@@ -4,9 +4,13 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import {
+  isBooleanDataType,
+  isCategoricalDataType,
+} from "@/src/features/manual-scoring/lib/helpers";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
 import { cn } from "@/src/utils/tailwind";
-import { MessageCircle } from "lucide-react";
+import { MessageCircleMore } from "lucide-react";
 
 export const GroupedScoreBadges = ({
   scores,
@@ -34,11 +38,13 @@ export const GroupedScoreBadges = ({
     <div className={cn("text-xs", props.className)}>
       {props.scores.map((s, i) => (
         <span key={i} className="group/score ml-1 first:ml-0">
-          {s.value.toFixed(2)}
+          {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
+            ? s.stringValue
+            : s.value.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
-                <MessageCircle size={12} />
+                <MessageCircleMore size={12} />
               </HoverCardTrigger>
               <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                 <p>{s.comment}</p>
@@ -73,9 +79,12 @@ export const GroupedScoreBadges = ({
             <Badge
               variant="outline"
               key={name}
-              className="break-all font-normal"
+              className="grid grid-cols-[1fr,auto] gap-1 font-normal"
             >
-              {name}: <ScoresOfGroup scores={scores} className="ml-2" />
+              <p className="truncate" title={name}>
+                {name}:
+              </p>
+              <ScoresOfGroup scores={scores} />
             </Badge>
           ))}
       </>

web/src/components/layouts/layout.tsx

@@ -521,9 +521,9 @@ const MainNavigation: React.FC<{
   onNavitemClick?: () => void;
   className?: string;
 }> = ({ nav, onNavitemClick, className }) => {
-  const [isOpen, setIsOpen] = useLocalStorage(
-    "sidebar-tracing-default-open",
-    false,
+  const [isOpen, setIsOpen] = useLocalStorage<Record<string, boolean>>(
+    "sidebar-item-default-open",
+    {},
   );
 
   return (
@@ -572,14 +572,20 @@ const MainNavigation: React.FC<{
               <Disclosure
                 as="div"
                 defaultOpen={
-                  item.children.some((child) => child.current) || isOpen
+                  item.children.some((child) => child.current) ||
+                  isOpen[item.name]
                 }
               >
                 {({ open }) => (
                   <>
                     <Disclosure.Button
                       className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold hover:bg-primary-foreground hover:text-primary-accent"
-                      onClick={() => setIsOpen(!isOpen)}
+                      onClick={() =>
+                        setIsOpen((prev) => ({
+                          ...prev,
+                          [item.name]: !prev[item.name],
+                        }))
+                      }
                     >
                       {item.icon && (
                         <item.icon

web/src/components/session/index.tsx

@@ -7,38 +7,24 @@ import { IOPreview } from "@/src/components/trace/IOPreview";
 import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 import { Badge } from "@/src/components/ui/badge";
 import { Card } from "@/src/components/ui/card";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect, useRef } from "react";
-import { useWindowVirtualizer } from "@tanstack/react-virtual";
-import { type RouterOutput } from "@/src/utils/types";
-import {
-  DataTableRowHeightSwitch,
-  type RowHeight,
-  useRowHeightLocalStorage,
-} from "@/src/components/table/data-table-row-height-switch";
-import { ScrollArea } from "@/src/components/ui/scroll-area";
+import { useEffect, useState } from "react";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
+import { Button } from "@/src/components/ui/button";
 
-// do not use the usual table row heights here
-const rowHeightMapping: Record<RowHeight, number> = {
-  s: 200,
-  m: 350,
-  l: 700,
-};
+// some projects have thousands of traces in a sessions, paginate to avoid rendering all at once
+const PAGE_SIZE = 50;
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
-  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
-    "single-session",
-    "m",
-  );
+  const [visibleTraces, setVisibleTraces] = useState(PAGE_SIZE);
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -96,11 +82,6 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
-          <DataTableRowHeightSwitch
-            rowHeight={rowHeight}
-            setRowHeight={setRowHeight}
-            key="height"
-          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -121,97 +102,49 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      {session.data && (
-        <TraceCardList
-          session={session.data}
-          projectId={projectId}
-          rowHeight={rowHeightMapping[rowHeight]}
-        />
-      )}
-    </div>
-  );
-};
-
-const TraceCardList = ({
-  session,
-  projectId,
-  rowHeight,
-}: {
-  session: RouterOutput["sessions"]["byId"];
-  projectId: string;
-  rowHeight: number;
-}) => {
-  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
-
-  const virtualizer = useWindowVirtualizer({
-    count: session.traces.length,
-    estimateSize: () => rowHeight,
-    overscan: 5,
-    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
-    gap: 10,
-  });
-  useEffect(() => {
-    // re-measure when rowHeight changes to update the virtualizer
-    virtualizer.measure();
-  }, [rowHeight, virtualizer]);
-  return (
-    <div className="mt-5 border-t pt-5">
-      <div
-        ref={listVirtualizationRef}
-        style={{
-          height: `${virtualizer.getTotalSize()}px`,
-          position: "relative",
-        }}
-      >
-        {virtualizer
-          .getVirtualItems()
-          .map((virtualItem) => ({
-            virtualItem,
-            trace: session.traces[virtualItem.index],
-          }))
-          .map(({ virtualItem, trace }) => (
-            <Card
-              className="group grid w-full gap-3 overflow-hidden border-border p-2 shadow-none hover:border-ring   md:grid-cols-3"
-              key={virtualItem.key}
-              data-index={virtualItem.index}
-              style={{
-                position: "absolute",
-                top: 0,
-                left: 0,
-                height: `${rowHeight}px`,
-                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
-              }}
-            >
-              <ScrollArea className="col-span-2 pr-3">
-                <SessionIO traceId={trace.id} />
-              </ScrollArea>
-              <ScrollArea>
-                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
-                  <Link
-                    href={`/project/${projectId}/traces/${trace.id}`}
-                    className="text-xs hover:underline"
-                  >
-                    Trace: {trace.name} ({trace.id})&nbsp;↗
-                  </Link>
-                  <div className="text-xs text-muted-foreground">
-                    {trace.timestamp.toLocaleString()}
-                  </div>
-                  <div className="mb-1 mt-2 text-xs text-muted-foreground">
-                    Scores
-                  </div>
-                  <div className="flex flex-wrap content-start items-start gap-1">
-                    <GroupedScoreBadges scores={trace.scores} />
-                  </div>
-                  <ManualScoreButton
-                    projectId={projectId}
-                    traceId={trace.id}
-                    scores={trace.scores}
-                    variant="badge"
-                  />
-                </div>
-              </ScrollArea>
-            </Card>
-          ))}
+      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
+        {session.data?.traces.slice(0, visibleTraces).map((trace) => (
+          <Card
+            className="group grid gap-3 border-border p-2 shadow-none hover:border-ring md:grid-cols-3"
+            key={trace.id}
+          >
+            <SessionIO traceId={trace.id} />
+            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
+              <Link
+                href={`/project/${projectId}/traces/${trace.id}`}
+                className="text-xs hover:underline"
+              >
+                Trace: {trace.name} ({trace.id})&nbsp;↗
+              </Link>
+              <div className="text-xs text-muted-foreground">
+                {trace.timestamp.toLocaleString()}
+              </div>
+              <div className="mb-1 mt-2 text-xs text-muted-foreground">
+                Scores
+              </div>
+              <div className="mb-1 flex flex-wrap content-start items-start gap-1">
+                <GroupedScoreBadges scores={trace.scores} />
+              </div>
+              <AnnotateButton
+                projectId={projectId}
+                traceId={trace.id}
+                scores={trace.scores}
+                variant="badge"
+                type="session"
+                source="SessionDetail"
+              />
+            </div>
+          </Card>
+        ))}
+        {session.data?.traces && session.data.traces.length > visibleTraces && (
+          <Button
+            onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
+            variant="ghost"
+            className="self-center"
+          >
+            {`Load ${Math.min(session.data.traces.length - visibleTraces, PAGE_SIZE)} More`}
+          </Button>
+        )}
       </div>
     </div>
   );
@@ -230,9 +163,8 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
-
   return (
-    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
+    <div className="col-span-2 flex flex-col gap-2 p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"

web/src/components/table/data-table.tsx

@@ -50,6 +50,8 @@ interface DataTableProps<TData, TValue> {
   help?: { description: string; href: string };
   rowHeight?: RowHeight;
   className?: string;
+  paginationClassName?: string;
+  isBorderless?: boolean;
 }
 
 export interface AsyncTableData<T> {
@@ -71,6 +73,9 @@ export function DataTable<TData extends object, TValue>({
   orderBy,
   setOrderBy,
   rowHeight,
+  className,
+  paginationClassName,
+  isBorderless = false,
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight);
@@ -105,8 +110,18 @@ export function DataTable<TData extends object, TValue>({
 
   return (
     <>
-      <div className="flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto">
-        <div className="w-full overflow-auto rounded-md border">
+      <div
+        className={cn(
+          "flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto",
+          className,
+        )}
+      >
+        <div
+          className={cn(
+            "w-full overflow-auto",
+            isBorderless ? "" : "rounded-md border",
+          )}
+        >
           <Table>
             <TableHeader>
               {table.getHeaderGroups().map((headerGroup) => (
@@ -242,7 +257,12 @@ export function DataTable<TData extends object, TValue>({
         <div className="grow"></div>
       </div>
       {pagination !== undefined ? (
-        <div className="bg:background sticky bottom-0 z-10 flex w-full justify-end font-medium">
+        <div
+          className={cn(
+            "sticky bottom-0 z-10 flex w-full justify-end bg-background font-medium",
+            paginationClassName,
+          )}
+        >
           <DataTablePagination
             table={table}
             paginationOptions={pagination.options}

web/src/components/table/use-cases/models.tsx

@@ -1,6 +1,12 @@
 import { DataTable } from "@/src/components/table/data-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { Button } from "@/src/components/ui/button";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { useState } from "react";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
@@ -10,6 +16,7 @@ import { type Prisma, type Model } from "@langfuse/shared/src/db";
 import Decimal from "decimal.js";
 import { Trash } from "lucide-react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+import { cn } from "@/src/utils/tailwind";
 
 export type ModelTableRow = {
   modelId: string;
@@ -222,13 +229,12 @@ export default function ModelTable({ projectId }: { projectId: string }) {
       accessorKey: "actions",
       header: "Actions",
       cell: ({ row }) => {
-        return row.original.maintainer === "User" ? (
+        return (
           <DeleteModelButton
             projectId={projectId}
             modelId={row.original.modelId}
+            isBuiltIn={row.original.maintainer === "Langfuse"}
           />
-        ) : (
-          <div className="h-6" />
         );
       },
     },
@@ -287,10 +293,13 @@ export default function ModelTable({ projectId }: { projectId: string }) {
 const DeleteModelButton = ({
   modelId,
   projectId,
+  isBuiltIn,
 }: {
   modelId: string;
   projectId: string;
+  isBuiltIn?: boolean;
 }) => {
+  const [isOpen, setIsOpen] = useState(false);
   const utils = api.useUtils();
   const capture = usePostHogClientCapture();
   const mut = api.models.delete.useMutation({
@@ -304,32 +313,48 @@ const DeleteModelButton = ({
     scope: "models:CUD",
   });
 
-  if (!hasAccess) {
-    return null;
-  }
-
   return (
-    <Button
-      size="xs"
-      variant="ghost"
-      onClick={() => {
-        const confirmDelete = window.confirm(
-          "Are you sure you want to delete this model?",
-        );
-        if (confirmDelete) {
-          capture("models:delete_button_click");
-          mut
-            .mutateAsync({
-              projectId,
-              modelId,
-            })
-            .catch((error) => {
-              console.error(error);
-            });
-        }
-      }}
-    >
-      <Trash size={14} />
-    </Button>
+    <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
+      <PopoverTrigger asChild>
+        <Button
+          variant="ghost"
+          size="xs"
+          disabled={!hasAccess || isBuiltIn}
+          title={
+            isBuiltIn ? "Built-in models cannot be deleted" : "Delete model"
+          }
+          className={cn(
+            isBuiltIn &&
+              "disabled:pointer-events-auto disabled:cursor-not-allowed",
+          )}
+        >
+          <Trash className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action permanently deletes this model definition.
+        </p>
+        <div className="flex justify-end space-x-4">
+          <Button
+            type="button"
+            variant="destructive"
+            loading={mut.isLoading}
+            onClick={() => {
+              capture("models:delete_button_click");
+              mut.mutateAsync({
+                projectId,
+                modelId,
+              });
+
+              setIsOpen(false);
+            }}
+          >
+            Delete Model
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
   );
 };

web/src/components/table/use-cases/score-configs.tsx

@@ -0,0 +1,174 @@
+import React from "react";
+import { Card } from "@/src/components/ui/card";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import { DataTable } from "@/src/components/table/data-table";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { NumberParam, useQueryParams, withDefault } from "use-query-params";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
+
+type ScoreConfigTableRow = {
+  id: string;
+  name: string;
+  dataType: ScoreDataType;
+  createdAt: string;
+  updatedAt: string;
+  range: {
+    maxValue?: number | null;
+    minValue?: number | null;
+    categories?: Prisma.JsonValue | null;
+  };
+  description?: string | null;
+};
+
+function getConfigRange(
+  originalRow: ScoreConfigTableRow,
+): Prisma.JsonValue | undefined {
+  const { range, dataType } = originalRow;
+  if (isNumericDataType(dataType)) {
+    return [
+      { minValue: range.minValue ?? "-∞", maxValue: range.maxValue ?? "∞" },
+    ];
+  }
+  return range.categories;
+}
+
+export function ScoreConfigsTable({ projectId }: { projectId: string }) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "scoreConfigs",
+    "s",
+  );
+
+  const configs = api.scoreConfigs.all.useQuery({
+    projectId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  const totalCount = configs.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<ScoreConfigTableRow>[] = [
+    {
+      accessorKey: "name",
+      id: "name",
+      header: "Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "dataType",
+      id: "dataType",
+      header: "Data Type",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "range",
+      id: "range",
+      header: "Range",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const range = getConfigRange(row.original);
+
+        return !!range ? (
+          <IOTableCell data={range} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "description",
+      id: "description",
+      header: "Description",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const value = row.original.description;
+
+        return !!value ? (
+          <IOTableCell data={value} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "id",
+      id: "id",
+      header: "Config ID",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "createdAt",
+      id: "createdAt",
+      header: "Created At",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<ScoreConfigTableRow>(
+      "scoreConfigsColumnVisibility",
+      columns,
+    );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+      />
+      <Card className="mb-4 flex max-h-[calc(100dvh-40rem)] flex-col overflow-hidden">
+        <DataTable
+          columns={columns}
+          data={
+            configs.isLoading
+              ? { isLoading: true, isError: false }
+              : configs.isError
+                ? {
+                    isLoading: false,
+                    isError: true,
+                    error: configs.error.message,
+                  }
+                : {
+                    isLoading: false,
+                    isError: false,
+                    data: configs.data?.configs.map((config) => ({
+                      id: config.id,
+                      name: config.name,
+                      dataType: config.dataType,
+                      description: config.description,
+                      createdAt: config.createdAt.toLocaleString(),
+                      updatedAt: config.updatedAt.toLocaleString(),
+                      range: {
+                        maxValue: config.maxValue,
+                        minValue: config.minValue,
+                        categories: config.categories,
+                      },
+                    })),
+                  }
+          }
+          pagination={{
+            pageCount: Math.ceil(totalCount / paginationState.pageSize),
+            onChange: setPaginationState,
+            state: paginationState,
+          }}
+          columnVisibility={columnVisibility}
+          onColumnVisibilityChange={setColumnVisibility}
+          rowHeight={rowHeight}
+          className="gap-2"
+          paginationClassName="-mx-2 mb-2"
+          isBorderless
+        />
+      </Card>
+    </>
+  );
+}

web/src/components/table/use-cases/scores.tsx

@@ -4,8 +4,10 @@ import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { Avatar, AvatarImage } from "@/src/components/ui/avatar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
   type ScoreOptions,
@@ -14,7 +16,7 @@ import {
 import { api } from "@/src/utils/api";
 import { utcDateOffsetByDays } from "@/src/utils/dates";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
-import type { FilterState } from "@langfuse/shared";
+import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useSession } from "next-auth/react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
@@ -24,7 +26,12 @@ export type ScoresTableRow = {
   timestamp: string;
   source: string;
   name: string;
-  value: number;
+  dataType: ScoreDataType;
+  value: string;
+  author: {
+    image?: string;
+    name?: string;
+  };
   comment?: string;
   observationId?: string;
   traceName?: string;
@@ -172,6 +179,29 @@ export default function ScoresTable({
         ) : undefined;
       },
     },
+    {
+      accessorKey: "userId",
+      header: "Trace User ID",
+      id: "userId",
+      headerTooltip: {
+        description: "The user ID associated with the trace.",
+        href: "https://langfuse.com/docs/tracing-features/users",
+      },
+      enableHiding: true,
+      enableSorting: true,
+      cell: ({ row }) => {
+        const value = row.getValue("userId");
+        return typeof value === "string" ? (
+          <>
+            <TableLink
+              path={`/project/${projectId}/users/${value}`}
+              value={value}
+              truncateAt={40}
+            />
+          </>
+        ) : undefined;
+      },
+    },
     {
       accessorKey: "timestamp",
       header: "Timestamp",
@@ -193,38 +223,39 @@ export default function ScoresTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "dataType",
+      header: "Data Type",
+      id: "dataType",
+      enableHiding: true,
+      enableSorting: true,
+    },
     {
       accessorKey: "value",
       header: "Value",
       id: "value",
       enableHiding: true,
       enableSorting: true,
-      cell: ({ row }) => {
-        const value: number = row.getValue("value");
-        return value % 1 === 0 ? value : value.toFixed(4);
-      },
     },
     {
-      accessorKey: "userId",
-      header: "User ID",
-      id: "userId",
-      headerTooltip: {
-        description: "The user ID associated with the trace.",
-        href: "https://langfuse.com/docs/tracing-features/users",
-      },
-      enableHiding: true,
-      enableSorting: true,
+      accessorKey: "author",
+      id: "author",
+      header: "Author",
       cell: ({ row }) => {
-        const value = row.getValue("userId");
-        return typeof value === "string" ? (
-          <>
-            <TableLink
-              path={`/project/${projectId}/users/${value}`}
-              value={value}
-              truncateAt={40}
-            />
-          </>
-        ) : undefined;
+        const { name, image } = row.getValue(
+          "author",
+        ) as ScoresTableRow["author"];
+        return (
+          <div className="flex items-center space-x-2">
+            <Avatar className="h-7 w-7">
+              <AvatarImage
+                src={image ?? undefined}
+                alt={name ?? "User Avatar"}
+              />
+            </Avatar>
+            <span>{name}</span>
+          </div>
+        );
       },
     },
     {
@@ -279,12 +310,21 @@ export default function ScoresTable({
       timestamp: score.timestamp.toLocaleString(),
       source: score.source,
       name: score.name,
-      value: score.value,
+      dataType: score.dataType,
+      value: isNumericDataType(score.dataType)
+        ? score.value % 1 === 0
+          ? String(score.value)
+          : score.value.toFixed(4)
+        : score.stringValue ?? "",
+      author: {
+        image: score.authorUserImage ?? undefined,
+        name: score.authorUserName ?? undefined,
+      },
       comment: score.comment ?? undefined,
       observationId: score.observationId ?? undefined,
       traceId: score.traceId,
       traceName: score.traceName ?? undefined,
-      userId: score.userId ?? undefined,
+      userId: score.traceUserId ?? undefined,
       jobConfigurationId: score.jobConfigurationId ?? undefined,
     };
   };

web/src/components/trace/IOPreview.tsx

@@ -1,6 +1,6 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { z } from "zod";
-import { deepParseJson } from "@/src/utils/json";
+import { deepParseJson } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { useState } from "react";
 import { Button } from "@/src/components/ui/button";

web/src/components/trace/ObservationPreview.tsx

@@ -8,7 +8,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { Badge } from "@/src/components/ui/badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
@@ -22,6 +21,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
@@ -154,11 +154,12 @@ export const ObservationPreview = (props: {
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={props.projectId}
               traceId={preloadedObservation.traceId}
               observationId={preloadedObservation.id}
               scores={props.scores}
+              type="observation"
             />
             {observationWithInputAndOutput.data?.type === "GENERATION" && (
               <JumpToPlaygroundButton
@@ -237,7 +238,7 @@ const PromptBadge = (props: { promptId: string; projectId: string }) => {
   if (prompt.isLoading || !prompt.data) return null;
   return (
     <Link
-      href={`/project/${props.projectId}/prompts/${prompt.data.name}?version=${prompt.data.version}`}
+      href={`/project/${props.projectId}/prompts/${encodeURIComponent(prompt.data.name)}?version=${prompt.data.version}`}
     >
       <Badge>
         Prompt: {prompt.data.name}

web/src/components/trace/TracePreview.tsx

@@ -9,7 +9,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -19,6 +18,7 @@ import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const TracePreview = ({
   trace,
@@ -54,13 +54,13 @@ export const TracePreview = ({
           <TabsList className="bg-background py-0">
             <TabsTrigger
               value="preview"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Preview
             </TabsTrigger>
             <TabsTrigger
               value="scores"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Scores
             </TabsTrigger>
@@ -95,7 +95,7 @@ export const TracePreview = ({
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={trace.projectId}
               traceId={trace.id}
               scores={scores}

web/src/components/ui/CodeJsonViewer.tsx

@@ -4,7 +4,7 @@ import { Check, ChevronsDownUp, ChevronsUpDown, Copy } from "lucide-react";
 import { cn } from "@/src/utils/tailwind";
 import { default as React18JsonView } from "react18-json-view";
 import "react18-json-view/src/dark.css";
-import { deepParseJson } from "@/src/utils/json";
+import { deepParseJson } from "@langfuse/shared";
 import { Skeleton } from "@/src/components/ui/skeleton";
 import { useTheme } from "next-themes";
 

web/src/components/ui/drawer.tsx

@@ -0,0 +1,129 @@
+"use client";
+
+import * as React from "react";
+import { Drawer as DrawerPrimitive } from "vaul";
+
+import { cn } from "@/src/utils/tailwind";
+import { useMediaQuery } from "react-responsive";
+
+// https://tailwindcss.com/docs/responsive-design
+const TAILWIND_MD_MEDIA_QUERY = 768;
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => {
+  const isMediumScreen = useMediaQuery({
+    query: `(min-width: ${TAILWIND_MD_MEDIA_QUERY}px)`,
+  });
+  const direction = isMediumScreen ? "right" : "bottom";
+
+  return (
+    <DrawerPrimitive.Root
+      shouldScaleBackground={shouldScaleBackground}
+      direction={direction}
+      {...props}
+    />
+  );
+};
+Drawer.displayName = "Drawer";
+
+const DrawerTrigger = DrawerPrimitive.Trigger;
+
+const DrawerPortal = DrawerPrimitive.Portal;
+
+const DrawerClose = DrawerPrimitive.Close;
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-primary/20", className)}
+    {...props}
+  />
+));
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName;
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background md:inset-x-auto md:inset-y-0 md:right-0 md:mt-0 md:h-full md:w-2/5 md:rounded-l-[10px] md:rounded-t-[0px] lg:w-1/3 xl:w-1/4 2xl:w-1/4",
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+));
+DrawerContent.displayName = "DrawerContent";
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+);
+DrawerHeader.displayName = "DrawerHeader";
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+);
+DrawerFooter.displayName = "DrawerFooter";
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName;
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+));
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName;
+
+export {
+  Drawer,
+  DrawerPortal,
+  DrawerOverlay,
+  DrawerTrigger,
+  DrawerClose,
+  DrawerContent,
+  DrawerHeader,
+  DrawerFooter,
+  DrawerTitle,
+  DrawerDescription,
+};

web/src/components/ui/popover.tsx

@@ -28,4 +28,6 @@ const PopoverContent = React.forwardRef<
 ));
 PopoverContent.displayName = PopoverPrimitive.Content.displayName;
 
-export { Popover, PopoverTrigger, PopoverContent };
+const PopoverClose = PopoverPrimitive.Close;
+
+export { Popover, PopoverTrigger, PopoverContent, PopoverClose };

web/src/components/ui/toggle-group.tsx

@@ -0,0 +1,60 @@
+"use client";
+
+import * as React from "react";
+import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group";
+import { type VariantProps } from "class-variance-authority";
+import { cn } from "@/src/utils/tailwind";
+import { toggleVariants } from "@/src/components/ui/toggle";
+
+const ToggleGroupContext = React.createContext<
+  VariantProps<typeof toggleVariants>
+>({
+  size: "default",
+  variant: "default",
+});
+
+const ToggleGroup = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Root> &
+    VariantProps<typeof toggleVariants>
+>(({ className, variant, size, children, ...props }, ref) => (
+  <ToggleGroupPrimitive.Root
+    ref={ref}
+    className={cn("flex items-center justify-center gap-1", className)}
+    {...props}
+  >
+    <ToggleGroupContext.Provider value={{ variant, size }}>
+      {children}
+    </ToggleGroupContext.Provider>
+  </ToggleGroupPrimitive.Root>
+));
+
+ToggleGroup.displayName = ToggleGroupPrimitive.Root.displayName;
+
+const ToggleGroupItem = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Item> &
+    VariantProps<typeof toggleVariants>
+>(({ className, children, variant, size, ...props }, ref) => {
+  const context = React.useContext(ToggleGroupContext);
+
+  return (
+    <ToggleGroupPrimitive.Item
+      ref={ref}
+      className={cn(
+        toggleVariants({
+          variant: context.variant || variant,
+          size: context.size || size,
+        }),
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </ToggleGroupPrimitive.Item>
+  );
+});
+
+ToggleGroupItem.displayName = ToggleGroupPrimitive.Item.displayName;
+
+export { ToggleGroup, ToggleGroupItem };

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.47.2";
+export const VERSION = "v2.52.0";

web/src/ee/features/evals/components/eval-config-detail.tsx

@@ -11,7 +11,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { useState } from "react";
-import { Trash2 } from "lucide-react";
+import { Trash } from "lucide-react";
 import {
   Tabs,
   TabsContent,
@@ -156,7 +156,7 @@ export function DeactivateConfig({
           disabled={!hasAccess || config?.status !== "ACTIVE"}
           loading={isLoading}
         >
-          <Trash2 className="h-5 w-5" />
+          <Trash className="h-5 w-5" />
         </Button>
       </PopoverTrigger>
       <PopoverContent>

web/src/ee/features/evals/components/template-form.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState, useMemo } from "react";
+import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import * as z from "zod";
 import { Input } from "@/src/components/ui/input";
@@ -18,19 +18,12 @@ import { zodResolver } from "@hookform/resolvers/zod";
 import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
 import router from "next/router";
 import { type EvalTemplate } from "@prisma/client";
+import { ModelParameters } from "@/src/components/ModelParameters";
 import {
-  ModelParameters,
-  type ModelParamsContext,
-} from "@/src/components/ModelParameters";
-import {
-  EvalModelNames,
   OutputSchema,
-  evalLLMModels,
   type UIModelParams,
-  ModelProvider,
-  type OpenAIModel,
-  type OpenAIModelParams,
   type ModelParams,
+  ZodModelConfig,
 } from "@langfuse/shared";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
 import {
@@ -43,6 +36,7 @@ import {
 import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
 
 export const EvalTemplateForm = (props: {
   projectId: string;
@@ -102,15 +96,6 @@ export const EvalTemplateForm = (props: {
                     score: currentTemplate?.outputScore?.trim() ?? "",
                     reasoning: currentTemplate?.outputReasoning?.trim() ?? "",
                   },
-                  model: "gpt-3.5-turbo",
-                  modelParams: {
-                    model: "gpt-3.5-turbo",
-                    provider: ModelProvider.OpenAI,
-                    temperature: 1,
-                    maxTemperature: 2,
-                    max_tokens: 256,
-                    top_p: 1,
-                  },
                 }
               : props.existingEvalTemplate
                 ? {
@@ -121,10 +106,13 @@ export const EvalTemplateForm = (props: {
                       score: string;
                       reasoning: string;
                     },
-                    model: props.existingEvalTemplate.model as OpenAIModel,
-                    modelParams: props.existingEvalTemplate
-                      .modelParams as OpenAIModelParams & {
-                      maxTemperature: number;
+                    selectedModel: {
+                      provider: props.existingEvalTemplate.provider,
+                      model: props.existingEvalTemplate.model,
+                      modelParams: props.existingEvalTemplate
+                        .modelParams as ModelParams & {
+                        maxTemperature: number;
+                      },
                     },
                   }
                 : undefined
@@ -135,6 +123,12 @@ export const EvalTemplateForm = (props: {
   );
 };
 
+const selectedModelSchema = z.object({
+  provider: z.string().min(1, "Select a provider"),
+  model: z.string().min(1, "Select a model"),
+  modelParams: ZodModelConfig,
+});
+
 const formSchema = z.object({
   name: z.string().min(1, "Enter a name"),
   prompt: z
@@ -167,9 +161,12 @@ export type EvalTemplateFormPreFill = {
     score: string;
     reasoning: string;
   };
-  model: OpenAIModel;
-  modelParams: OpenAIModelParams & {
-    maxTemperature: number;
+  selectedModel?: {
+    provider: string;
+    model: string;
+    modelParams: ModelParams & {
+      maxTemperature: number;
+    };
   };
 };
 
@@ -186,27 +183,41 @@ export const InnerEvalTemplateForm = (props: {
 }) => {
   const capture = usePostHogClientCapture();
   const [formError, setFormError] = useState<string | null>(null);
-  const preFilledModel = useMemo(
-    () => getModelParamsWithEnabledFlag(props.preFilledFormValues),
-    [props.preFilledFormValues],
-  );
 
   // updates the model params based on the pre-filled data
   // either form update or from langfuse-generated template
-  const [modelParams, setModelParams] = useState<UIModelParams>(preFilledModel);
-  const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
-    key,
-    value,
-  ) => {
-    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], value } }));
-  };
+  const {
+    modelParams,
+    setModelParams,
+    updateModelParamValue,
+    setModelParamEnabled,
+    availableModels,
+    availableProviders,
+  } = useModelParams({ evalModelsOnly: true });
 
-  const setModelParamEnabled: ModelParamsContext["setModelParamEnabled"] = (
-    key,
-    enabled,
-  ) => {
-    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], enabled } }));
-  };
+  useEffect(() => {
+    if (props.preFilledFormValues?.selectedModel) {
+      const { provider, model, modelParams } =
+        props.preFilledFormValues.selectedModel;
+
+      const modelConfig = Object.entries(modelParams).reduce(
+        (acc, [key, value]) => {
+          return {
+            ...acc,
+            [key]: { value, enabled: true },
+          };
+        },
+        {} as UIModelParams,
+      );
+
+      setModelParams((prev) => ({
+        ...prev,
+        ...modelConfig,
+        provider: { value: provider, enabled: true },
+        model: { value: model, enabled: true },
+      }));
+    }
+  }, [props.preFilledFormValues?.selectedModel, setModelParams]);
 
   // updates the form based on the pre-filled data
   // either form update or from langfuse-generated template
@@ -243,33 +254,8 @@ export const InnerEvalTemplateForm = (props: {
         outputScore: OutputSchema.parse(props.preFilledFormValues.outputSchema)
           .score,
       });
-
-      // state for the model params is outside of the form, hence needs to be handled individually
-      // also set the context for the playground
-      const model = EvalModelNames.parse(preFilledModel.model.value);
-      updateModelParamValue("model", model);
-      setModelParams((prev) => ({
-        ...prev,
-        ...preFilledModel,
-      }));
-
-      const modelProvider = evalLLMModels.find((m) => m.model.value === model)
-        ?.provider.value;
-
-      if (modelProvider) {
-        updateModelParamValue("provider", modelProvider); // updating the provider based on the model
-        updateModelParamValue(
-          "maxTemperature",
-          modelProvider === ModelProvider.OpenAI ? 2 : 1,
-        ); // setting the max value of the slider based on the provider
-      }
     }
-  }, [
-    props.preFilledFormValues,
-    preFilledModel,
-    form,
-    props.existingEvalTemplateName,
-  ]);
+  }, [props.preFilledFormValues, form, props.existingEvalTemplateName]);
 
   const extractedVariables = form.watch("prompt")
     ? extractVariables(form.watch("prompt")).filter(getIsCharOrUnderscore)
@@ -288,26 +274,31 @@ export const InnerEvalTemplateForm = (props: {
         : "eval_templates:new_form_submit",
     );
 
-    const model = EvalModelNames.safeParse(modelParams.model.value);
+    const evalTemplate = {
+      name: values.name,
+      projectId: props.projectId,
+      prompt: values.prompt,
+      provider: modelParams.provider.value,
+      model: modelParams.model.value,
+      modelParams: getFinalModelParams(modelParams),
+      vars: extractedVariables ?? [],
+      outputSchema: {
+        score: values.outputScore,
+        reasoning: values.outputReasoning,
+      },
+    };
 
-    if (!model.success) {
-      setFormError("Please select a model.");
+    const parsedModel = selectedModelSchema.safeParse(evalTemplate);
+
+    if (!parsedModel.success) {
+      setFormError(
+        `${parsedModel.error.errors[0].path}: ${parsedModel.error.errors[0].message}`,
+      );
       return;
     }
 
     createEvalTemplateMutation
-      .mutateAsync({
-        name: values.name,
-        projectId: props.projectId,
-        prompt: values.prompt,
-        model: model.data,
-        modelParams: getFinalModelParams(modelParams),
-        vars: extractedVariables ?? [],
-        outputSchema: {
-          score: values.outputScore,
-          reasoning: values.outputReasoning,
-        },
-      })
+      .mutateAsync(evalTemplate)
       .then((res) => {
         props.onFormSuccess?.();
         form.reset();
@@ -429,10 +420,11 @@ export const InnerEvalTemplateForm = (props: {
             <ModelParameters
               {...{
                 modelParams,
+                availableModels,
+                availableProviders,
                 updateModelParamValue: updateModelParamValue,
                 setModelParamEnabled,
               }}
-              availableModels={[...evalLLMModels]}
               formDisabled={!props.isEditing}
             />
           </div>
@@ -456,33 +448,3 @@ export const InnerEvalTemplateForm = (props: {
     </Form>
   );
 };
-
-function getModelParamsWithEnabledFlag(
-  evalPreFill?: EvalTemplateFormPreFill,
-): UIModelParams {
-  const defaultModelParams: ModelParams & { maxTemperature: number } = {
-    model: evalPreFill?.model ?? "gpt-3.5-turbo",
-    provider: ModelProvider.OpenAI,
-    max_tokens: 100,
-    maxTemperature:
-      evalPreFill?.modelParams?.provider === ModelProvider.OpenAI ? 2 : 1,
-    top_p: 1,
-    temperature: 1,
-  };
-
-  return Object.entries({
-    ...defaultModelParams,
-    ...evalPreFill?.modelParams,
-  }).reduce(
-    (params, [key, value]) => ({
-      ...params,
-      [key]: {
-        enabled: Boolean(
-          !evalPreFill || evalPreFill.modelParams[key as keyof UIModelParams],
-        ),
-        value,
-      },
-    }),
-    {} as UIModelParams,
-  );
-}

web/src/ee/features/evals/server/router.ts

@@ -8,7 +8,6 @@ import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { DEFAULT_TRACE_JOB_DELAY, EvalTargetObject } from "@langfuse/shared";
 import {
-  EvalModelNames,
   ZodModelConfig,
   singleFilter,
   variableMapping,
@@ -19,7 +18,8 @@ export const CreateEvalTemplate = z.object({
   name: z.string().min(1),
   projectId: z.string(),
   prompt: z.string(),
-  model: EvalModelNames,
+  provider: z.string(),
+  model: z.string(),
   modelParams: ZodModelConfig,
   vars: z.array(z.string()),
   outputSchema: z.object({
@@ -332,6 +332,7 @@ export const evalRouter = createTRPCRouter({
           modelParams: input.modelParams,
           vars: input.vars,
           outputSchema: input.outputSchema,
+          provider: input.provider,
         },
       });
 

web/src/ee/features/playground/page/components/JumpToPlaygroundButton.tsx

@@ -1,24 +1,24 @@
 import { Terminal } from "lucide-react";
 import Link from "next/link";
+import { useEffect, useState } from "react";
+import { z } from "zod";
 
+import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import { Button } from "@/src/components/ui/button";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { type PlaygroundCache } from "@/src/ee/features/playground/page/types";
+import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import {
   ChatMessageRole,
-  supportedModels as playgroundSupportedModels,
   type Observation,
   type Prompt,
+  supportedModels as playgroundSupportedModels,
   type UIModelParams,
   ZodModelConfig,
 } from "@langfuse/shared";
-import { useEffect, useState } from "react";
-import { PromptType } from "@/src/features/prompts/server/utils/validation";
-import { ChatMessageListSchema } from "@/src/features/prompts/components/NewPromptForm/validation";
-import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 
 type JumpToPlaygroundButtonProps = (
   | {
@@ -76,9 +76,19 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
   );
 };
 
+const ParsedChatMessageListSchema = z.array(
+  z.object({
+    role: z.nativeEnum(ChatMessageRole),
+    content: z.union([
+      z.string(),
+      z.any().transform((v) => JSON.stringify(v, null, 2)),
+    ]),
+  }),
+);
+
 const parsePrompt = (prompt: Prompt): PlaygroundCache => {
   if (prompt.type === PromptType.Chat) {
-    const parsedMessages = ChatMessageListSchema.safeParse(prompt.prompt);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(prompt.prompt);
 
     return parsedMessages.success ? { messages: parsedMessages.data } : null;
   } else {
@@ -109,14 +119,16 @@ const parseGeneration = (generation: Observation): PlaygroundCache => {
   }
 
   if (typeof input === "object") {
-    const parsedMessages = ChatMessageListSchema.safeParse(input);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(input);
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };
   }
 
   if (typeof input === "object" && "messages" in input) {
-    const parsedMessages = ChatMessageListSchema.safeParse(input["messages"]);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(
+      input["messages"],
+    );
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };

web/src/ee/features/playground/page/components/PromptVariableComponent.tsx

@@ -1,5 +1,5 @@
 import { type ChangeEvent } from "react";
-import { CheckCircle2, Circle, Trash2Icon } from "lucide-react";
+import { CheckCircle2, Circle, TrashIcon } from "lucide-react";
 import { Button } from "@/src/components/ui/button";
 import { Textarea } from "@/src/components/ui/textarea";
 import { type PromptVariable } from "@langfuse/shared";
@@ -46,7 +46,7 @@ export const PromptVariableComponent: React.FC<{
           onClick={handleDeleteVariable}
           className="p-0"
         >
-          {!isUsed && <Trash2Icon size={16} />}
+          {!isUsed && <TrashIcon size={16} />}
         </Button>
       </div>
       <Textarea

web/src/ee/features/playground/page/components/SaveToPromptButton.tsx

@@ -49,8 +49,7 @@ export const SaveToPromptButton: React.FC = () => {
       )
       .data?.prompts.filter((prompt) => prompt.type === PromptType.Chat)
       .map((prompt) => ({
-        label:
-          prompt.name.slice(0, 20) + (prompt.name.length > 25 ? "..." : ""),
+        label: prompt.name,
         value: prompt.id,
       })) ?? [];
 
@@ -126,7 +125,9 @@ export const SaveToPromptButton: React.FC = () => {
                         : "opacity-0",
                     )}
                   />
-                  {promptName.label}
+                  <span className="overflow-hidden text-ellipsis whitespace-nowrap">
+                    {promptName.label}
+                  </span>
                 </CommandItem>
               ))}
             </CommandList>

web/src/ee/features/playground/page/context/index.tsx

@@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid";
 
 import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import useCommandEnter from "@/src/ee/features/playground/page/hooks/useCommandEnter";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -19,7 +20,6 @@ import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
   type ChatMessageWithId,
-  ModelProvider,
   type PromptVariable,
   type UIModelParams,
 } from "@langfuse/shared";
@@ -68,9 +68,14 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     createEmptyMessage(ChatMessageRole.System),
     createEmptyMessage(ChatMessageRole.User),
   ]);
-  const [modelParams, setModelParams] = useState<UIModelParams>(
-    getDefaultModelParams(ModelProvider.OpenAI),
-  );
+  const {
+    modelParams,
+    setModelParams,
+    availableProviders,
+    availableModels,
+    updateModelParamValue,
+    setModelParamEnabled,
+  } = useModelParams();
 
   // Load state from cache
   useEffect(() => {
@@ -97,11 +102,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     if (cachedPromptVariables) {
       setPromptVariables(cachedPromptVariables);
     }
-  }, [playgroundCache]);
-
-  useEffect(() => {
-    setModelParams(getDefaultModelParams(modelParams.provider.value));
-  }, [modelParams.provider.value]);
+  }, [playgroundCache, setModelParams]);
 
   const updatePromptVariables = useCallback(() => {
     const messageContents = messages.map((m) => m.content).join("\n");
@@ -169,6 +170,10 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
           finalMessages.map((m) => m.content).join("\n"),
         );
 
+        if (!modelParams.provider.value || !modelParams.model.value) {
+          throw new Error("Please select a model");
+        }
+
         if (leftOverVariables.length > 0) {
           throw Error("Error replacing variables. Please check your inputs.");
         }
@@ -210,24 +215,6 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
 
   useCommandEnter(!isStreaming, handleSubmit);
 
-  const updateModelParamValue: PlaygroundContextType["updateModelParamValue"] =
-    (key, value) => {
-      setModelParams((prev) => ({
-        ...prev,
-        [key]: { ...prev[key], value },
-      }));
-    };
-
-  const setModelParamEnabled: PlaygroundContextType["setModelParamEnabled"] = (
-    key,
-    enabled,
-  ) => {
-    setModelParams((prev) => ({
-      ...prev,
-      [key]: { ...prev[key], enabled },
-    }));
-  };
-
   const updatePromptVariableValue = (variable: string, value: string) => {
     setPromptVariables((prev) =>
       prev.map((v) => (v.name === variable ? { ...v, value } : v)),
@@ -258,6 +245,9 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
         outputJson,
         handleSubmit,
         isStreaming,
+
+        availableProviders,
+        availableModels,
       }}
     >
       {children}
@@ -342,38 +332,6 @@ function getFinalMessages(
   return finalMessages;
 }
 
-function getDefaultModelParams(provider: ModelProvider): UIModelParams {
-  switch (provider) {
-    // Docs: https://platform.openai.com/docs/api-reference/chat/create
-    case ModelProvider.OpenAI:
-      return {
-        provider: {
-          value: provider,
-          enabled: true,
-        },
-        model: { value: "gpt-3.5-turbo", enabled: true },
-        temperature: { value: 1, enabled: true },
-        maxTemperature: { value: 2, enabled: true },
-        max_tokens: { value: 256, enabled: true },
-        top_p: { value: 1, enabled: true },
-      };
-
-    // Docs: https://docs.anthropic.com/claude/reference/messages_post
-    case ModelProvider.Anthropic:
-      return {
-        provider: {
-          value: provider,
-          enabled: true,
-        },
-        model: { value: "claude-3-opus-20240229", enabled: true },
-        temperature: { value: 0, enabled: true },
-        maxTemperature: { value: 1, enabled: true },
-        max_tokens: { value: 256, enabled: true },
-        top_p: { value: 1, enabled: true },
-      };
-  }
-}
-
 function getOutputJson(
   output: string,
   messages: ChatMessageWithId[],

web/src/ee/features/playground/page/hooks/useModelParams.ts

@@ -0,0 +1,170 @@
+import { useEffect, useMemo, useState } from "react";
+
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { api } from "@/src/utils/api";
+import {
+  LLMAdapter,
+  evalModelsByAdapter,
+  supportedModels,
+  type UIModelParams,
+} from "@langfuse/shared";
+import { type ModelParamsContext } from "@/src/components/ModelParameters";
+
+export const useModelParams = ({
+  evalModelsOnly,
+}: {
+  evalModelsOnly?: boolean;
+} = {}) => {
+  const [modelParams, setModelParams] = useState<UIModelParams>({
+    ...getDefaultAdapterParams(LLMAdapter.OpenAI),
+    provider: { value: "", enabled: true },
+    model: { value: "", enabled: true },
+  });
+
+  // Set initial model params
+  const projectId = useProjectIdFromURL();
+  const availableLLMApiKeys = api.llmApiKey.all.useQuery(
+    {
+      projectId: projectId as string,
+    },
+    { enabled: Boolean(projectId) },
+  );
+
+  const availableProviders = useMemo(() => {
+    return availableLLMApiKeys.data?.data.map((key) => key.provider) ?? [];
+  }, [availableLLMApiKeys]);
+
+  const selectedProviderApiKey = availableLLMApiKeys.data?.data.find(
+    (key) => key.provider === modelParams.provider.value,
+  );
+
+  const availableModels = useMemo(
+    () =>
+      !selectedProviderApiKey
+        ? []
+        : evalModelsOnly
+          ? [
+              ...evalModelsByAdapter[selectedProviderApiKey.adapter],
+              ...selectedProviderApiKey.customModels,
+            ]
+          : selectedProviderApiKey.withDefaultModels
+            ? [
+                ...selectedProviderApiKey.customModels,
+                ...supportedModels[selectedProviderApiKey.adapter],
+              ]
+            : selectedProviderApiKey.customModels,
+    [selectedProviderApiKey, evalModelsOnly],
+  );
+
+  const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
+    key,
+    value,
+  ) => {
+    setModelParams((prev) => ({
+      ...prev,
+      [key]: { ...prev[key], value },
+    }));
+  };
+
+  const setModelParamEnabled: ModelParamsContext["setModelParamEnabled"] = (
+    key,
+    enabled,
+  ) => {
+    setModelParams((prev) => ({
+      ...prev,
+      [key]: { ...prev[key], enabled },
+    }));
+  };
+
+  // Set default provider and model
+  useEffect(() => {
+    if (availableProviders.length > 0 && !modelParams.provider.value) {
+      updateModelParamValue("provider", availableProviders[0]);
+    }
+  }, [availableProviders, modelParams.provider.value]);
+
+  useEffect(() => {
+    if (
+      (availableModels.length > 0 && !modelParams.model.value) ||
+      !availableModels.includes(modelParams.model.value)
+    ) {
+      updateModelParamValue("model", availableModels[0]);
+    }
+  }, [availableModels, modelParams.model.value]);
+
+  // Update adapter and max temperature when provider changes
+  useEffect(() => {
+    if (selectedProviderApiKey?.adapter) {
+      setModelParams((prev) => ({
+        ...prev,
+        adapter: {
+          value: selectedProviderApiKey.adapter,
+          enabled: true,
+        },
+        maxTemperature: getDefaultAdapterParams(selectedProviderApiKey.adapter)
+          .maxTemperature,
+        temperature: {
+          value: Math.min(
+            prev.temperature.value,
+            getDefaultAdapterParams(selectedProviderApiKey.adapter)
+              .maxTemperature.value,
+          ),
+          enabled: true,
+        },
+      }));
+    }
+  }, [selectedProviderApiKey?.adapter]);
+
+  return {
+    modelParams,
+    setModelParams,
+    availableProviders,
+    availableModels,
+    updateModelParamValue,
+    setModelParamEnabled,
+  };
+};
+
+function getDefaultAdapterParams(
+  adapter: LLMAdapter,
+): Omit<UIModelParams, "provider" | "model"> {
+  switch (adapter) {
+    // Docs: https://platform.openai.com/docs/api-reference/chat/create
+    case LLMAdapter.OpenAI:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 2, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+
+    case LLMAdapter.Azure:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 2, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+
+    // Docs: https://docs.anthropic.com/claude/reference/messages_post
+    case LLMAdapter.Anthropic:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 1, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+  }
+}

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -39,6 +39,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
       streaming: true,
       callbacks: [new PosthogCallbackHandler("playground", body, userId)],
       apiKey: decrypt(LLMApiKey.secretKey),
+      baseURL: LLMApiKey.baseURL || undefined,
     });
 
     return new StreamingTextResponse(stream);

web/src/ee/features/playground/server/validateChatCompletionBody.ts

@@ -1,31 +1,14 @@
 import { z } from "zod";
-import {
-  ChatMessageRole,
-  ModelProvider,
-  anthropicModels,
-  openAIModels,
-} from "@langfuse/shared";
+import { ChatMessageRole, LLMAdapter } from "@langfuse/shared";
 
-const OpenAIModelSchema = z.enum([...openAIModels]);
-const AnthropicModelSchema = z.enum([...anthropicModels]);
-const OpenAIModelParamsSchema = z.object({
-  provider: z.literal(ModelProvider.OpenAI),
-  model: OpenAIModelSchema,
-  max_tokens: z.number().optional(),
-  temperature: z.number().optional(),
-  top_p: z.number().optional(),
-});
-const AnthropicModelParamsSchema = z.object({
-  provider: z.literal(ModelProvider.Anthropic),
-  model: AnthropicModelSchema,
+const ModelParamsSchema = z.object({
+  provider: z.string(),
+  adapter: z.nativeEnum(LLMAdapter),
+  model: z.string(),
   temperature: z.number().optional(),
   max_tokens: z.number().optional(),
   top_p: z.number().optional(),
 });
-const ModelParamsSchema = z.union([
-  OpenAIModelParamsSchema,
-  AnthropicModelParamsSchema,
-]);
 const MessageSchema = z.object({
   role: z.nativeEnum(ChatMessageRole),
   content: z.string(),

web/src/env.mjs

@@ -30,14 +30,20 @@ export const env = createEnv({
     LANGFUSE_NEW_USER_SIGNUP_WEBHOOK: z.string().url().optional(),
     // Add `.min(1) on ID and SECRET if you want to make sure they're not empty
     LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: z.enum(["true", "false"]).optional(),
-    LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET: z.string().optional().refine((v) =>
-      v === undefined || !isNaN(Number.parseInt(v))
-      , {
-        message: "LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET must be a number when set"
-      }).transform((v) =>
-        v === undefined ? undefined : -Number.parseInt(v) // negative offset
+    LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET: z
+      .string()
+      .optional()
+      .refine((v) => v === undefined || !isNaN(Number.parseInt(v)), {
+        message:
+          "LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET must be a number when set",
+      })
+      .transform(
+        (v) => (v === undefined ? undefined : -Number.parseInt(v)), // negative offset
       ),
-    LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES: z.enum(["true", "false"]).optional().default("false"),
+    LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES: z
+      .enum(["true", "false"])
+      .optional()
+      .default("false"),
     SALT: z.string({
       required_error:
         "A strong Salt is required to encrypt API keys securely. See: https://langfuse.com/docs/deployment/self-host#deploy-the-container",
@@ -71,6 +77,11 @@ export const env = createEnv({
     AUTH_COGNITO_CLIENT_SECRET: z.string().optional(),
     AUTH_COGNITO_ISSUER: z.string().url().optional(),
     AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_CUSTOM_CLIENT_ID: z.string().optional(),
+    AUTH_CUSTOM_CLIENT_SECRET: z.string().optional(),
+    AUTH_CUSTOM_ISSUER: z.string().url().optional(),
+    AUTH_CUSTOM_NAME: z.string().optional(),
+    AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
@@ -88,9 +99,6 @@ export const env = createEnv({
     // Worker
     LANGFUSE_WORKER_HOST: z.string().optional(),
     LANGFUSE_WORKER_PASSWORD: z.string().optional(),
-    // Prompt playground
-    OPENAI_API_KEY: z.string().optional(),
-    ANTHROPIC_API_KEY: z.string().optional(),
     TURNSTILE_SECRET_KEY: z.string().optional(),
     // DB event log
     ENABLE_EVENT_LOG: z.enum(["true", "false"]).optional().default("true"),
@@ -174,7 +182,14 @@ export const env = createEnv({
     AUTH_COGNITO_CLIENT_ID: process.env.AUTH_COGNITO_CLIENT_ID,
     AUTH_COGNITO_CLIENT_SECRET: process.env.AUTH_COGNITO_CLIENT_SECRET,
     AUTH_COGNITO_ISSUER: process.env.AUTH_COGNITO_ISSUER,
-    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
+    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
+    AUTH_CUSTOM_CLIENT_ID: process.env.AUTH_CUSTOM_CLIENT_ID,
+    AUTH_CUSTOM_CLIENT_SECRET: process.env.AUTH_CUSTOM_CLIENT_SECRET,
+    AUTH_CUSTOM_ISSUER: process.env.AUTH_CUSTOM_ISSUER,
+    AUTH_CUSTOM_NAME: process.env.AUTH_CUSTOM_NAME,
+    AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
@@ -193,9 +208,6 @@ export const env = createEnv({
     // Worker
     LANGFUSE_WORKER_HOST: process.env.LANGFUSE_WORKER_HOST,
     LANGFUSE_WORKER_PASSWORD: process.env.LANGFUSE_WORKER_PASSWORD,
-    // Prompt playground
-    OPENAI_API_KEY: process.env.OPENAI_API_KEY,
-    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
     TURNSTILE_SECRET_KEY: process.env.TURNSTILE_SECRET_KEY,
     NEXT_PUBLIC_TURNSTILE_SITE_KEY: process.env.NEXT_PUBLIC_TURNSTILE_SITE_KEY,
     NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,

web/src/features/auth/lib/projectNameSchema.ts

@@ -1,4 +1,4 @@
-import { noHtmlCheck } from "@/src/utils/zod";
+import { noHtmlCheck } from "@langfuse/shared";
 import * as z from "zod";
 
 export const projectNameSchema = z.object({

web/src/features/auth/lib/signupSchema.ts

@@ -1,4 +1,4 @@
-import { noHtmlCheck } from "@/src/utils/zod";
+import { noHtmlCheck } from "@langfuse/shared";
 import * as z from "zod";
 
 export const signupSchema = z.object({

web/src/features/dashboard/components/BaseTimeSeriesChart.tsx

@@ -3,6 +3,10 @@ import { getColorsForCategories } from "@/src/features/dashboard/utils/getColors
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { cn } from "@/src/utils/tailwind";
 import { AreaChart, LineChart } from "@tremor/react";
+import {
+  CurrencyTooltip,
+  NumberTooltip,
+} from "@/src/features/dashboard/components/Tooltip";
 
 export type TimeSeriesChartDataPoint = {
   ts: number;
@@ -58,6 +62,9 @@ export function BaseTimeSeriesChart(props: {
   };
 
   const ChartComponent = props.chartType === "area" ? AreaChart : LineChart;
+  const TooltipComponent = props.valueFormatter
+    ? CurrencyTooltip
+    : NumberTooltip;
   const colors = getColorsForCategories(Array.from(labels));
 
   return (
@@ -74,6 +81,7 @@ export function BaseTimeSeriesChart(props: {
       showAnimation={true}
       onValueChange={() => {}}
       enableLegendSlider={true}
+      customTooltip={TooltipComponent}
     />
   );
 }

web/src/features/dashboard/components/TabsComponent.tsx

@@ -21,7 +21,7 @@ export const TabComponent = ({ tabs }: TabComponentProps) => {
         <select
           id="tabs"
           name="tabs"
-          className="focus:border-primary-accent focus:ring-primary-accent block w-full rounded-md border-border py-2 pl-3 pr-10 text-base focus:outline-none sm:text-sm"
+          className="block w-full rounded-md border-border bg-background py-2 pl-3 pr-10 text-base focus:border-primary-accent focus:outline-none focus:ring-primary-accent sm:text-sm"
           defaultValue={0}
           onChange={(e) => setSelectedIndex(Number(e.target.selectedIndex))}
         >

web/src/features/dashboard/components/Tooltip.tsx

@@ -0,0 +1,71 @@
+import {
+  ChartTooltipFrame,
+  ChartTooltipRow,
+} from "@tremor/react/dist/components/chart-elements/common/ChartTooltip";
+import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { cn } from "@/src/utils/tailwind";
+import { type CustomTooltipProps } from "@tremor/react";
+import { getRandomColor } from "@/src/features/dashboard/utils/getColorsForCategories";
+
+export const Tooltip = ({
+  payload,
+  active,
+  label,
+  formatter,
+}: CustomTooltipProps & { formatter: (value: number) => string }) => {
+  if (!active || !payload) return null;
+
+  // Filter out duplicates and sort by value in descending order
+  const uniquePayload = Array.from(
+    new Map(payload.map((category) => [category.name, category])).values(),
+  );
+
+  const sortedPayload = uniquePayload.sort(
+    (a, b) => (Number(b.value) ?? 0) - (Number(a.value) ?? 0),
+  );
+
+  return (
+    <ChartTooltipFrame>
+      <div
+        className={cn(
+          // light
+          "border-b border-tremor-border px-4 py-2",
+          // dark
+          "dark:border-dark-tremor-border",
+        )}
+      >
+        <p
+          className={cn(
+            // common
+            "font-medium",
+            // light
+            "text-tremor-content-emphasis",
+            // dark
+            "dark:text-dark-tremor-content-emphasis",
+          )}
+        >
+          {label}
+        </p>
+      </div>
+
+      <div className={cn("space-y-1 px-4 py-2")}>
+        {sortedPayload.map(({ name, value, color }, index) => (
+          <ChartTooltipRow
+            key={`${index}`}
+            value={formatter(Number(value))}
+            name={name?.toString() ?? ""}
+            color={color ?? getRandomColor()}
+          />
+        ))}
+      </div>
+    </ChartTooltipFrame>
+  );
+};
+
+export const NumberTooltip = (props: CustomTooltipProps) => (
+  <Tooltip {...props} formatter={compactNumberFormatter} />
+);
+
+export const CurrencyTooltip = (props: CustomTooltipProps) => (
+  <Tooltip {...props} formatter={usdFormatter} />
+);

web/src/features/dashboard/components/TracesBarListChart.tsx

@@ -91,7 +91,7 @@ export const TracesBarListChart = ({
             <BarList
               data={adjustedData}
               valueFormatter={(number: number) =>
-                Intl.NumberFormat("us").format(number).toString()
+                Intl.NumberFormat("en-US").format(number).toString()
               }
               className="mt-6"
               showAnimation={true}

web/src/features/dashboard/utils/getColorsForCategories.tsx

@@ -47,7 +47,7 @@ const predefinedColors: Color[] = [
   "stone",
 ];
 
-function getRandomColor() {
+export function getRandomColor() {
   return predefinedColors[Math.floor(Math.random() * predefinedColors.length)];
 }
 

web/src/features/datasets/components/DatasetRunsTable.tsx

@@ -12,7 +12,7 @@ import { useEffect } from "react";
 import { usdFormatter } from "../../../utils/numbers";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
-import { type Prisma } from "@langfuse/shared";
+import { ScoreDataType, type Prisma } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 
@@ -120,6 +120,7 @@ export function DatasetRunsTable(props: {
             scores={Object.entries(scores).map(([k, v]) => ({
               name: k,
               value: v,
+              dataType: ScoreDataType.NUMERIC, // numeric and boolean values treated as numeric
             }))}
             variant="headings"
           />

web/src/features/datasets/server/dataset-router.ts

@@ -12,7 +12,7 @@ import {
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { DB } from "@/src/server/db";
-import { paginationZod } from "@/src/utils/zod";
+import { paginationZod } from "@langfuse/shared";
 
 export const datasetRouter = createTRPCRouter({
   allDatasetMeta: protectedProjectProcedure
@@ -167,6 +167,7 @@ export const datasetRouter = createTRPCRouter({
                 JOIN traces t ON t.id = s.trace_id
               WHERE 
                 t.project_id = ${input.projectId}
+                AND s.data_type != 'CATEGORICAL'
                 AND ri.dataset_run_id = runs.id
               GROUP BY s.name
             ) s
@@ -529,18 +530,6 @@ export const datasetRouter = createTRPCRouter({
             },
           },
         },
-        include: {
-          observation: {
-            select: {
-              id: true,
-            },
-          },
-          trace: {
-            select: {
-              id: true,
-            },
-          },
-        },
         orderBy: {
           createdAt: "desc",
         },
@@ -548,6 +537,8 @@ export const datasetRouter = createTRPCRouter({
         skip: input.page * input.limit,
       });
 
+      if (runItems.length === 0) return { totalRunItems: 0, runItems: [] };
+
       const traceScores = await ctx.prisma.score.findMany({
         where: {
           projectId: ctx.session.projectId,
@@ -597,18 +588,37 @@ export const datasetRouter = createTRPCRouter({
         },
       });
 
-      const traces = await ctx.prisma.traceView.findMany({
-        where: {
-          id: {
-            in: runItems.map((ri) => ri.traceId).filter(Boolean) as string[],
-          },
-          projectId: ctx.session.projectId,
-        },
-        select: {
-          id: true,
-          duration: true,
-        },
-      });
+      // Directly access 'traces' table and calculate duration via lateral join
+      // Previously used 'traces_view' was not performant enough
+      const traceIdsSQL = Prisma.sql`ARRAY[${Prisma.join(runItems.map((ri) => ri.traceId))}]`;
+      const traces = await ctx.prisma.$queryRaw<
+        {
+          id: string;
+          duration: number;
+        }[]
+      >(
+        Prisma.sql`
+            SELECT
+              t.id,
+              o.duration
+            FROM
+              traces t
+              LEFT JOIN LATERAL (
+                SELECT
+                  EXTRACT(epoch FROM COALESCE(max(o1.end_time), max(o1.start_time)))::double precision - EXTRACT(epoch FROM min(o1.start_time))::double precision AS duration
+                FROM
+                  observations o1
+                WHERE
+                  o1.project_id = t.project_id
+                  AND o1.trace_id = t.id
+                GROUP BY
+                  o1.project_id,
+                  o1.trace_id) o ON TRUE
+            WHERE
+              t.project_id = ${input.projectId}
+              AND t.id = ANY(${traceIdsSQL})        
+        `,
+      );
 
       const items = runItems.map((ri) => {
         return {
@@ -628,6 +638,7 @@ export const datasetRouter = createTRPCRouter({
         };
       });
 
+      // Note: We early return in case of no run items, when adding parameters here, make sure to update the early return above
       return {
         totalRunItems,
         runItems: items,

web/src/features/filters/components/filter-builder.tsx

@@ -25,7 +25,7 @@ import {
   filterOperators,
   singleFilter,
 } from "@langfuse/shared";
-import { NonEmptyString } from "@/src/utils/zod";
+import { NonEmptyString } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 

web/src/features/filters/components/multi-select.tsx

@@ -104,7 +104,6 @@ export function MultiSelect({
                         selectedValues.add(option.value);
                       }
                       const filterValues = Array.from(selectedValues);
-
                       onValueChange(filterValues.length ? filterValues : []);
                     }}
                   >

web/src/features/filters/hooks/useFilterState.ts

@@ -15,6 +15,7 @@ import {
   withDefault,
 } from "use-query-params";
 import { promptsTableCols } from "@/src/server/api/definitions/promptsTable";
+import { usersTableCols } from "@/src/server/api/definitions/usersTable";
 
 const DEBUG_QUERY_STATE = false;
 
@@ -107,6 +108,7 @@ const tableCols = {
   sessions: sessionsViewCols,
   scores: scoresTableCols,
   prompts: promptsTableCols,
+  users: usersTableCols,
   dashboard: [
     { id: "traceName", name: "Trace Name" },
     { id: "tags", name: "Tags" },

web/src/features/llm-api-key/server/router.ts

@@ -1,32 +1,26 @@
 import { z } from "zod";
 
+import { env } from "@/src/env.mjs";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { CreateLlmApiKey } from "@/src/features/llm-api-key/types";
+import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import {
   createTRPCRouter,
   protectedProjectProcedure,
 } from "@/src/server/api/trpc";
-import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
-import { auditLog } from "@/src/features/audit-logs/auditLog";
-import { env } from "@/src/env.mjs";
-import { CreateLlmApiKey } from "@/src/features/llm-api-key/types";
+import {
+  type ChatMessage,
+  LLMApiKeySchema,
+  fetchLLMCompletion,
+  ChatMessageRole,
+  supportedModels,
+} from "@langfuse/shared";
 import { encrypt } from "@langfuse/shared/encryption";
 
 export function getDisplaySecretKey(secretKey: string) {
   return "..." + secretKey.slice(-4);
 }
 
-export const LlmApiKey = z
-  .object({
-    id: z.string(),
-    projectId: z.string(),
-    provider: z.string(),
-    createdAt: z.date(),
-    updatedAt: z.date(),
-    displaySecretKey: z.string(),
-  })
-  // strict mode to prevent extra keys. Thorws error otherwise
-  // https://github.com/colinhacks/zod?tab=readme-ov-file#strict
-  .strict();
-
 export const llmApiKeyRouter = createTRPCRouter({
   create: protectedProjectProcedure
     .input(CreateLlmApiKey)
@@ -45,8 +39,12 @@ export const llmApiKeyRouter = createTRPCRouter({
           data: {
             projectId: input.projectId,
             secretKey: encrypt(input.secretKey),
+            adapter: input.adapter,
             displaySecretKey: getDisplaySecretKey(input.secretKey),
             provider: input.provider,
+            baseURL: input.baseURL,
+            withDefaultModels: input.withDefaultModels,
+            customModels: input.customModels,
           },
         });
 
@@ -109,22 +107,28 @@ export const llmApiKeyRouter = createTRPCRouter({
         scope: "llmApiKeys:read",
       });
 
-      const apiKeys = z.array(LlmApiKey).parse(
-        await ctx.prisma.llmApiKeys.findMany({
-          // we must not return the secret key via the API, hence not selected
-          select: {
-            id: true,
-            createdAt: true,
-            updatedAt: true,
-            provider: true,
-            displaySecretKey: true,
-            projectId: true,
-          },
-          where: {
-            projectId: input.projectId,
-          },
-        }),
-      );
+      const apiKeys = z
+        .array(LLMApiKeySchema.extend({ secretKey: z.undefined() }))
+        .parse(
+          await ctx.prisma.llmApiKeys.findMany({
+            // we must not return the secret key via the API, hence not selected
+            select: {
+              id: true,
+              createdAt: true,
+              updatedAt: true,
+              provider: true,
+              displaySecretKey: true,
+              projectId: true,
+              adapter: true,
+              baseURL: true,
+              customModels: true,
+              withDefaultModels: true,
+            },
+            where: {
+              projectId: input.projectId,
+            },
+          }),
+        );
 
       const count = await ctx.prisma.llmApiKeys.count({
         where: {
@@ -137,4 +141,47 @@ export const llmApiKeyRouter = createTRPCRouter({
         totalCount: count,
       };
     }),
+
+  test: protectedProjectProcedure
+    .input(CreateLlmApiKey)
+    .mutation(async ({ input }) => {
+      if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === undefined) {
+        throw new Error("Evals available in cloud only");
+      }
+
+      try {
+        const model = input.customModels?.length
+          ? input.customModels[0]
+          : supportedModels[input.adapter][0];
+
+        if (!model) throw Error("No model found");
+
+        const testMessages: ChatMessage[] = [
+          { role: ChatMessageRole.System, content: "You are a bot" },
+          { role: ChatMessageRole.User, content: "How are you?" },
+        ];
+
+        await fetchLLMCompletion({
+          modelParams: {
+            adapter: input.adapter,
+            provider: input.provider,
+            model,
+          },
+          baseURL: input.baseURL,
+          apiKey: input.secretKey,
+          messages: testMessages,
+          streaming: false,
+          maxRetries: 1,
+        });
+
+        return { success: true };
+      } catch (err) {
+        console.log(err);
+
+        return {
+          success: false,
+          error: err instanceof Error ? err.message : "Unknown error",
+        };
+      }
+    }),
 });