chore: release v2.53.0

* add: MUI X tree dependency

* feat: initial timeline view

* add: alert and beta badge

* add: icons

* refactor: extract `TreeItemInner`

* feat: add drawer with detail content

* push

* improve styling

* style overflow

* use localStorage to perserve view

* sort nested observations

* rename to `TraceTimelineView`

* refactor: extract common functionality

* push

* fix: label width

* push

* push

* push

* push

* fix: hover for dark mode

* constant for step size

* fix: scroll behavior mobile

* Revert "add: MUI X tree dependency"

This reverts commit d4179dd7c1a99458b8f511834ec730121688e9ee.

* Revert "Revert "add: MUI X tree dependency""

This reverts commit 8dfe26ebc14ab2afabe9d5ac0a35d156bc10437c.

* push master pnpm equivalent

* reinstall: MUI X tree view

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+{
+	"name": "langfuse dev",
+    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+        "version": "latest",
+        "moby": true
+        }
+    },
+    "forwardPorts": [3000],
+
+    "onCreateCommand": "npm install -g pnpm@8.15.5",
+    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+}

.env.dev.example

@@ -34,10 +34,6 @@ S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 S3_REGION=
 
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Set during docker build of application
 # Used to disable environment verification at build time
 # DOCKER_BUILD=1

.env.local.example

@@ -13,11 +13,6 @@ NEXTAUTH_URL="http://localhost:3000"
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"
 
-
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379

.env.prod.example

@@ -62,7 +62,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
-
+# AUTH_CUSTOM_CLIENT_ID=
+# AUTH_CUSTOM_CLIENT_SECRET=
+# AUTH_CUSTOM_ISSUER=
+# AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -123,10 +127,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 

.github/workflows/pipeline.yml

@@ -40,19 +40,27 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build Web Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./web/Dockerfile
-          push: false
+      - name: Build and run both images from compose
+        run: |
+          docker compose -f docker-compose.build.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Build Worker Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: false
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Ensure no unhealthy status
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
 
   tests-web:
     runs-on: ubuntu-latest

.github/workflows/snyk.yml

@@ -0,0 +1,39 @@
+name: Snyk Container
+on: push
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker compose -f docker-compose.build.yml up -d
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-server
+          args: --file=web/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: web
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-worker
+          args: --file=worker/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: worker

.husky/pre-commit

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Commit will proceed
+    else
+        echo "Commit to $protected_branch branch has been canceled."
+        exit 1 # Commit will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the commit
+exit 0

.husky/pre-push

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to push to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Push will proceed
+    else
+        echo "Push to $protected_branch branch has been canceled."
+        exit 1 # Push will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the push
+exit 0

CONTRIBUTING.md

@@ -151,6 +151,8 @@ Requirements
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
 - Docker to run the database locally
 
+**Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
+
 **Steps**
 
 1. Fork the repository and clone it locally
@@ -191,6 +193,16 @@ Requirements
     pnpm run dev
    ```
 
+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -358,7 +370,9 @@ The background color of the following component will be `hsl(var(--primary))` an
 
 By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
 
-## Using secrets stored in 1Password
+## Maintainers
+
+### Using secrets stored in 1Password
 
 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
 

docker-compose.build.yml

@@ -20,6 +20,11 @@ services:
       - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     build:
@@ -41,6 +46,11 @@ services:
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:7.2.4

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   langfuse-server:
-    image: ghcr.io/langfuse/langfuse:latest
+    image: langfuse/langfuse:2
     depends_on:
       db:
         condition: service_healthy

ee/package.json

@@ -33,7 +33,7 @@
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

ee/src/sso/index.ts

@@ -11,6 +11,7 @@ import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { env } from "../env";
+import { CustomSSOProvider } from "@langfuse/shared/src/server/auth";
 
 // Local cache for SSO configurations
 let cachedSsoConfigs: {
@@ -173,6 +174,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "custom")
+    return CustomSSOProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -76,12 +76,26 @@ export const CognitoProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CustomProviderSchema = base.extend({
+  authProvider: z.literal("custom"),
+  authConfig: z
+    .object({
+      name: z.string(),
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
 export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -90,6 +104,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   OktaProviderSchema,
   AzureAdProviderSchema,
   CognitoProviderSchema,
+  CustomProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/ingestion.yml

@@ -7,28 +7,52 @@ service:
   base-path: /api/public
   endpoints:
     batch:
-      docs: Batched ingestion for Langfuse Tracing
+      docs: Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
       method: POST
       path: /ingestion
       request:
         name: IngestionRequest
         body:
           properties:
-            batch: list<IngestionEvent>
-      response: IngestionResponse # will be reportet as 200 response, but endpoint returns 207
+            batch:
+              type: list<IngestionEvent>
+              docs: "Batch of tracing events to be ingested. Discriminated by attribute `type`."
+            metadata:
+              type: optional<unknown>
+              docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
+      response:
+        type: IngestionResponse
+        status-code: 207
 
 types:
   IngestionEvent:
     discriminant: "type"
     union:
-      trace-create: TraceEvent
-      score-create: ScoreEvent
-      event-create: CreateEventEvent
-      generation-create: CreateGenerationEvent
-      generation-update: UpdateGenerationEvent
-      span-create: CreateSpanEvent
-      span-update: UpdateSpanEvent
-      sdk-log: SDKLogEvent
+      trace-create:
+        type: TraceEvent
+        docs: Creates a new trace. Upserts on id for updates if trace with id exists.
+      score-create:
+        type: ScoreEvent
+        docs: Creates a new score. Upserts on id for updates if score with id exists.
+      span-create:
+        type: CreateSpanEvent
+        docs: Creates a new span.
+      span-update:
+        type: UpdateSpanEvent
+        docs: Updates span based on id.
+      generation-create:
+        type: CreateGenerationEvent
+        docs: Creates a new generation.
+      generation-update:
+        type: UpdateGenerationEvent
+        docs: Updates a generation based on id.
+      event-create:
+        type: CreateEventEvent
+        docs: Creates an event.
+
+      sdk-log:
+        type: SDKLogEvent
+        docs: Langfuse SDKs only, used for debugging purposes.
 
       # both are legacy
       observation-create:
@@ -162,9 +186,15 @@ types:
 
   BaseEvent:
     properties:
-      id: string
-      timestamp: string
-      metadata: unknown
+      id:
+        type: string
+        docs: UUID v4 that identifies the event
+      timestamp:
+        type: string
+        docs: "Datetime (ISO 8601) of event creation in client. Should be as close to actual event creation in client as possible, this timestamp will be used for ordering of events in future release. Resolution: milliseconds (required), microseconds (optimal)."
+      metadata:
+        type: optional<unknown>
+        docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
 
   TraceEvent:
     extends: BaseEvent

fern/apis/server/definition/metrics.yml

@@ -48,12 +48,25 @@ types:
       date: date
       countTraces: integer
       countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
       model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD

fern/apis/server/definition/score.yml

@@ -31,7 +31,7 @@ service:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
-            type: commons.ScoreSource
+            type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
           operator:
             type: optional<string>
@@ -39,6 +39,9 @@ service:
           value:
             type: optional<double>
             docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
       response: Scores
     get-by-id:
       docs: Get a score

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
       path: /sessions/{sessionId}
       path-parameters:

fern/apis/server/definition/trace.yml

@@ -30,6 +30,7 @@ service:
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only traces newer than this datetime (ISO 8601).

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.21.0"
+  "version": "0.30.3"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.47.4",
+  "version": "2.53.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -17,26 +17,36 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "models:migrate": "turbo run models:migrate",
-    "release": "dotenv -e ../.env -- release-it"
+    "release": "dotenv -e ../.env -- release-it",
+    "prepare": "husky"
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
+    "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
+    "husky": "^9.0.11",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.1",
+    "release-it": "^17.3.0",
     "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {
       "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
+      "tagName": "v${version}",
+      "commitArgs": [
+        "--no-verify"
+      ],
+      "pushArgs": [
+        "--no-verify"
+      ]
     },
     "plugins": {
       "@release-it/bumper": {

packages/config-eslint/package.json

@@ -9,7 +9,7 @@
   ],
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",

packages/shared/package.json

@@ -52,6 +52,8 @@
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
     "langchain": "^0.1.36",
+    "lodash": "^4.17.21",
+    "next-auth": "^4.24.7",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -59,17 +61,17 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
     "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
     "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.0.3",
+    "nodemon": "^3.1.3",
     "prettier": "^3.2.5",
     "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",

packages/shared/prisma/generated/types.ts

@@ -32,20 +32,10 @@ export const ScoreSource = {
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const ScoreDataType = {
     CATEGORICAL: "CATEGORICAL",
-    NUMERIC: "NUMERIC"
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
-} as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -162,6 +152,7 @@ export type EvalTemplate = {
     version: number;
     prompt: string;
     model: string;
+    provider: string;
     model_params: unknown;
     vars: Generated<string[]>;
     output_schema: unknown;
@@ -209,8 +200,12 @@ export type LlmApiKeys = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     provider: string;
+    adapter: string;
     display_secret_key: string;
     secret_key: string;
+    base_url: string | null;
+    custom_models: string[];
+    with_default_models: Generated<boolean>;
     project_id: string;
 };
 export type MembershipInvitation = {
@@ -309,14 +304,6 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -470,7 +457,6 @@ export type DB = {
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;

packages/shared/prisma/migrations/20240603212024_dataset_items_add_index_source_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");

packages/shared/prisma/migrations/20240604133338_scores_add_index_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");

packages/shared/prisma/migrations/20240604133339_score_data_type_add_boolean/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';

packages/shared/prisma/migrations/20240604133340_backfill_manual_scores/migration.sql

@@ -0,0 +1,50 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	gen_random_uuid () AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,5 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240606093356_drop_unused_pricings_table/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";

packages/shared/prisma/migrations/20240606133011_remove_trace_fkey_datasetrunitems/migration.sql

@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";

packages/shared/prisma/migrations/20240611105521_llm_api_keys_custom_endpoints/migration.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+  ALTER TABLE "llm_api_keys" 
+    ADD COLUMN "base_url" TEXT,
+    ADD COLUMN "adapter" TEXT,
+    ADD COLUMN "custom_models" TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+    ADD COLUMN "with_default_models" BOOLEAN NOT NULL DEFAULT true;
+
+  UPDATE "llm_api_keys"
+    SET "adapter" = "provider";
+
+  ALTER TABLE "llm_api_keys" 
+    ALTER COLUMN "adapter" SET NOT NULL;
+
+  ALTER TABLE "eval_templates"
+    ADD COLUMN "provider" TEXT;
+  
+  UPDATE "eval_templates"
+    SET "provider" = 'openai';
+  
+  ALTER TABLE "eval_templates"
+    ALTER COLUMN "provider" SET NOT NULL;
+
+COMMIT;

packages/shared/prisma/schema.prisma

@@ -144,9 +144,13 @@ model LlmApiKeys {
     createdAt DateTime @default(now()) @map("created_at")
     updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
 
-    provider         String
-    displaySecretKey String @map("display_secret_key")
-    secretKey        String @map("secret_key")
+    provider          String
+    adapter           String // This controls the interface that is used to connect with the LLM, e.g. 'openai' or 'anthropic'
+    displaySecretKey  String   @map("display_secret_key")
+    secretKey         String   @map("secret_key")
+    baseURL           String?  @map("base_url")
+    customModels      String[] @map("custom_models")
+    withDefaultModels Boolean  @default(true) @map("with_default_models")
 
     projectId String  @map("project_id")
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -233,9 +237,8 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetRunItems DatasetRunItems[]
-    DatasetItem     DatasetItem[]
-    JobExecution    JobExecution[]
+    DatasetItem  DatasetItem[]
+    JobExecution JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -298,21 +301,20 @@ model Observation {
 
     // GENERATION ONLY
     model               String?
-    internalModel       String?           @map("internal_model")
+    internalModel       String?       @map("internal_model")
     modelParameters     Json?
     input               Json?
     output              Json?
-    promptTokens        Int               @default(0) @map("prompt_tokens")
-    completionTokens    Int               @default(0) @map("completion_tokens")
-    totalTokens         Int               @default(0) @map("total_tokens")
+    promptTokens        Int           @default(0) @map("prompt_tokens")
+    completionTokens    Int           @default(0) @map("completion_tokens")
+    totalTokens         Int           @default(0) @map("total_tokens")
     unit                String?
-    inputCost           Decimal?          @map("input_cost")
-    outputCost          Decimal?          @map("output_cost")
-    totalCost           Decimal?          @map("total_cost")
-    completionStartTime DateTime?         @map("completion_start_time")
-    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    inputCost           Decimal?      @map("input_cost")
+    outputCost          Decimal?      @map("output_cost")
+    totalCost           Decimal?      @map("total_cost")
+    completionStartTime DateTime?     @map("completion_start_time")
+    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
-    datasetRunItems     DatasetRunItems[]
 
     promptId String? @map("prompt_id")
     prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@ -419,6 +421,7 @@ model Score {
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
+    @@index([projectId, name])
     @@index([authorUserId])
     @@index([configId])
     @@index([traceId], type: Hash)
@@ -463,29 +466,7 @@ model ScoreConfig {
 enum ScoreDataType {
     CATEGORICAL
     NUMERIC
-}
-
-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
-}
-
-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
+    BOOLEAN
 }
 
 model CronJobs {
@@ -532,6 +513,7 @@ model DatasetItem {
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
     @@index([createdAt])
@@ -563,17 +545,15 @@ model DatasetRuns {
 }
 
 model DatasetRunItems {
-    id            String       @id @default(cuid())
-    datasetRunId  String       @map("dataset_run_id")
-    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String       @map("dataset_item_id")
-    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime     @default(now()) @map("created_at")
-    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String      @map("trace_id")
+    observationId String?     @map("observation_id")
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)
@@ -688,6 +668,7 @@ model EvalTemplate {
     version          Int
     prompt           String
     model            String
+    provider         String
     modelParams      Json               @map("model_params")
     vars             String[]           @default([])
     outputSchema     Json               @map("output_schema")

packages/shared/prisma/seed.ts

@@ -4,6 +4,7 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
+  ScoreDataType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -39,6 +40,7 @@ async function main() {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
 
@@ -144,6 +146,11 @@ async function main() {
       });
     }
 
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -157,7 +164,8 @@ async function main() {
       colorTags,
       project1,
       project2,
-      promptIds
+      promptIds,
+      configIdsAndNames
     );
 
     console.log(
@@ -176,6 +184,7 @@ async function main() {
           secretKey: encrypt(OPENAI_API_KEY),
           displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
           provider: "openai",
+          adapter: "openai",
         },
       });
     } else {
@@ -201,6 +210,7 @@ async function main() {
           "Please evaluate the toxicity of the following text {{input}} {{output}}",
         model: "gpt-3.5-turbo",
         vars: ["input", "output"],
+        provider: "openai",
         outputSchema: {
           score: "provide a score between 0 and 1",
           reasoning: "one sentence reasoning for the score",
@@ -453,13 +463,15 @@ function createObjects(
   colorTags: (string | null)[],
   project1: Project,
   project2: Project,
-  promptIds: Map<string, string[]>
+  promptIds: Map<string, string[]>,
+  configIdsAndNames: Map<string, { name: string; id: string }[]>
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
   const scores: Prisma.ScoreCreateManyInput[] = [];
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
+  const configs: Prisma.ScoreConfigCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -511,17 +523,28 @@ function createObjects(
 
     traces.push(trace);
 
+    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const randomIndex = Math.floor(Math.random() * 3);
+    const config =
+      configArray.length >= randomIndex - 1 && configArray[randomIndex];
+    const { name: annotationScoreName, id: configId } = config || {
+      name: "manual-score",
+      id: undefined,
+    };
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
             {
               traceId: trace.id,
-              name: "manual-score",
+              name: annotationScoreName,
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
+              dataType: ScoreDataType.NUMERIC,
+              ...(configId ? { configId } : {}),
             },
           ]
         : []),
@@ -534,6 +557,7 @@ function createObjects(
               timestamp: traceTs,
               source: ScoreSource.API,
               projectId,
+              dataType: ScoreDataType.NUMERIC,
             },
           ]
         : []),
@@ -772,6 +796,7 @@ function createObjects(
     traces,
     observations,
     scores,
+    configs,
     sessions: uniqueSessions,
     events,
   };
@@ -954,3 +979,80 @@ async function generatePrompts(project: Project) {
   }
   return promptIds;
 }
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
+    new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    })
+  );
+  return projectIdsToConfigs;
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: { name: string; id: string }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({ name: config.name, id: config.id });
+  }
+
+  return configNameAndId;
+}

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, UIModelParams } from "../..";
+import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,38 +77,15 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels: UIModelParams[] = [
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-3.5-turbo", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4-turbo-preview", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4o", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-] as const;
-
-export const EvalModelNames = z.enum([
-  "gpt-3.5-turbo",
-  "gpt-4-turbo-preview",
-  "gpt-4o",
-]);
+export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
+  [LLMAdapter.OpenAI]: [
+    "gpt-4o",
+    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+  ] as const,
+  [LLMAdapter.Anthropic]: [] as const,
+  [LLMAdapter.Azure]: [] as const,
+};
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/features/ingestion/types.ts

@@ -1,9 +1,9 @@
 import lodash from "lodash";
 import { z } from "zod";
 
-import { ModelUsageUnit } from "@langfuse/shared";
-import { NonEmptyString, jsonSchema } from "@/src/utils/zod";
-import { ObservationLevel } from "@langfuse/shared";
+import { NonEmptyString, jsonSchema } from "../../utils/zod";
+import { ModelUsageUnit } from "../../constants";
+import { ObservationLevel } from "@prisma/client";
 
 export const Usage = z.object({
   input: z.number().int().nullish(),
@@ -109,7 +109,7 @@ export const CreateGenerationBody = CreateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -131,7 +131,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -195,7 +195,7 @@ export const LegacyGenerationsCreateSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -219,7 +219,7 @@ export const LegacyGenerationPatchSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -243,7 +243,7 @@ export const LegacyObservationBody = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   input: jsonSchema.nullish(),

packages/shared/src/filterToPrisma.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import {
   ColumnDefinition,
-  type TableNames as TableName,
+  type TableNames,
 } from "./interfaces/tableDefinition";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,7 +43,7 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName
+  table: TableNames
 ) => {
   return column.name === "type" &&
     (table === "observations" ||

packages/shared/src/index.ts

@@ -9,6 +9,9 @@ export * from "./filterToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
 export * from "./observationsTable";
+export * from "./features/ingestion/types";
+export * from "./utils/zod";
+export * from "./utils/json";
 export { env } from "./env";
 
 // llm api

packages/shared/src/interfaces/tableDefinition.ts

@@ -42,6 +42,7 @@ export const tableNames = [
   "traces_parent_observation_scores",
   "sessions",
   "prompts",
+  "users",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/server/auth.ts

@@ -1,6 +1,7 @@
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
 
 export function generateSecretKey() {
   return `sk-lf-${randomUUID()}`;
@@ -48,3 +49,33 @@ export function createShaHash(privateKey: string, salt: string): string {
 
   return hash;
 }
+
+export interface CustomSSOUser extends Record<string, any> {
+  email: string;
+  id: string;
+  name: string;
+  verified: boolean;
+}
+
+export function CustomSSOProvider<P extends CustomSSOUser>(
+  options: OAuthUserConfig<P>
+): OAuthConfig<P> {
+  return {
+    id: "custom",
+    name: "CustomSSOProvider",
+    type: "oauth",
+    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
+    authorization: { params: { scope: "openid email profile" } },
+    checks: ["pkce", "state"],
+    idToken: true,
+    profile(profile) {
+      return {
+        id: profile.sub,
+        name: profile.name,
+        email: profile.email,
+        image: null,
+      };
+    },
+    options,
+  };
+}

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -17,7 +17,7 @@ import {
   ChatMessageRole,
   LLMFunctionCall,
   ModelParams,
-  ModelProvider,
+  LLMAdapter,
 } from "./types";
 import zodToJsonSchema from "zod-to-json-schema";
 import { JsonOutputFunctionsParser } from "langchain/output_parsers";
@@ -27,7 +27,9 @@ type LLMCompletionParams = {
   modelParams: ModelParams;
   functionCall?: LLMFunctionCall;
   callbacks?: BaseCallbackHandler[];
+  baseURL?: string;
   apiKey?: string;
+  maxRetries?: number;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -57,7 +59,16 @@ export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   // the apiKey must never be printed to the console
-  const { messages, modelParams, streaming, callbacks, apiKey } = params;
+  const {
+    messages,
+    modelParams,
+    streaming,
+    callbacks,
+    apiKey,
+    baseURL,
+    maxRetries,
+  } = params;
+
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -68,16 +79,18 @@ export async function fetchLLMCompletion(
   });
 
   let chatModel: ChatOpenAI | ChatAnthropic;
-  if (modelParams.provider === ModelProvider.Anthropic) {
+  if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
+      anthropicApiUrl: baseURL,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      clientOptions: { maxRetries },
     });
-  } else if (modelParams.provider === ModelProvider.OpenAI) {
+  } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
       openAIApiKey: apiKey,
       modelName: modelParams.model,
@@ -85,15 +98,29 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      maxRetries,
+      configuration: {
+        baseURL,
+      },
+    });
+  } else if (modelParams.adapter === LLMAdapter.Azure) {
+    chatModel = new ChatOpenAI({
+      azureOpenAIApiKey: apiKey,
+      azureOpenAIBasePath: baseURL,
+      azureOpenAIApiDeploymentName: modelParams.model,
+      azureOpenAIApiVersion: "2024-02-01",
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+      maxRetries,
     });
   } else {
     // eslint-disable-next-line no-unused-vars
-    const _exhaustiveCheck: never = modelParams;
+    const _exhaustiveCheck: never = modelParams.adapter;
     throw new Error("This model provider is not supported.");
   }
 
-  console.log("Making LLM call with params: ", modelParams);
-
   if (params.functionCall) {
     const functionCallingModel = chatModel.bind({
       functions: [

packages/shared/src/server/llm/types.ts

@@ -1,3 +1,4 @@
+import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
@@ -9,9 +10,10 @@ export type ChatMessage = {
 
 export type ChatMessageWithId = ChatMessage & { id: string };
 
-export enum ModelProvider {
+export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
+  Azure = "azure",
 }
 
 export enum ChatMessageRole {
@@ -19,32 +21,30 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
 }
-export type ModelParams = AnthropicModelParams | OpenAIModelParams;
+
+export type ModelParams = {
+  provider: string;
+  adapter: LLMAdapter;
+  model: string;
+} & ModelConfig;
 
 type RecordWithEnabledFlag<T> = {
   [K in keyof T]: { value: T[K]; enabled: boolean };
 };
 export type UIModelParams = RecordWithEnabledFlag<
-  Required<AnthropicModelParams | OpenAIModelParams> & {
+  Required<ModelParams> & {
     maxTemperature: number;
   }
 >;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
-
 export const ZodModelConfig = z.object({
   max_tokens: z.coerce.number().optional(),
   temperature: z.coerce.number().optional(),
   top_p: z.coerce.number().optional(),
 });
 
-// OpenAI
-export type OpenAIModelParams = {
-  provider: ModelProvider.OpenAI;
-  model: OpenAIModel;
-} & ModelConfig;
-
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-05-13",
@@ -64,12 +64,6 @@ export const openAIModels = [
 
 export type OpenAIModel = (typeof openAIModels)[number];
 
-// Anthropic
-export type AnthropicModelParams = {
-  provider: ModelProvider.Anthropic;
-  model: AnthropicModel;
-} & ModelConfig;
-
 export const anthropicModels = [
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
@@ -81,8 +75,9 @@ export const anthropicModels = [
 
 export type AnthropicModel = (typeof anthropicModels)[number];
 export const supportedModels = {
-  [ModelProvider.Anthropic]: anthropicModels,
-  [ModelProvider.OpenAI]: openAIModels,
+  [LLMAdapter.Anthropic]: anthropicModels,
+  [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.Azure]: [],
 } as const;
 
 export type LLMFunctionCall = {
@@ -90,3 +85,26 @@ export type LLMFunctionCall = {
   description: string;
   parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
 };
+
+export const LLMApiKeySchema = z
+  .object({
+    id: z.string(),
+    projectId: z.string(),
+    createdAt: z.date(),
+    updatedAt: z.date(),
+    adapter: z.nativeEnum(LLMAdapter),
+    provider: z.string(),
+    displaySecretKey: z.string(),
+    secretKey: z.string(),
+    baseURL: z.string().nullable(),
+    customModels: z.array(z.string()),
+    withDefaultModels: z.boolean(),
+  })
+  // strict mode to prevent extra keys. Thorws error otherwise
+  // https://github.com/colinhacks/zod?tab=readme-ov-file#strict
+  .strict();
+
+export type LLMApiKey =
+  z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
+    ? z.infer<typeof LLMApiKeySchema>
+    : never;

packages/shared/src/types.ts

@@ -30,4 +30,5 @@ export type TableName =
   | "sessions"
   | "scores"
   | "prompts"
-  | "dashboard";
+  | "dashboard"
+  | "users";

packages/shared/src/utils/json.ts

@@ -1,6 +1,6 @@
-import { type jsonSchema } from "@/src/utils/zod";
 import { type z } from "zod";
 import lodash from "lodash";
+import { jsonSchema } from "./zod";
 
 export const parseJson = (input: string) => {
   try {
@@ -37,7 +37,7 @@ export function deepParseJson(json: unknown): unknown {
         // Ensure we only iterate over the object's own properties
         if (Object.prototype.hasOwnProperty.call(json, key)) {
           (json as Record<string, unknown>)[key] = deepParseJson(
-            (json as Record<string, unknown>)[key],
+            (json as Record<string, unknown>)[key]
           );
         }
       }
@@ -50,7 +50,7 @@ export function deepParseJson(json: unknown): unknown {
 
 export const mergeJson = (
   json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
+  json2?: z.infer<typeof jsonSchema>
 ) => {
   if (json1 === undefined) {
     return json2;

packages/shared/src/utils/zod.ts

@@ -26,7 +26,7 @@ const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
     nestedLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 // Root schema that does not allow nulls at the root level
@@ -35,17 +35,17 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
     rootLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().default(1)
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().lte(100).default(50)
   ),
 };
 

web/Dockerfile

@@ -1,11 +1,11 @@
-FROM node:20-alpine AS alpine
+FROM node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
+RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.47.4",
+  "version": "2.53.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -27,7 +27,7 @@
     "@codemirror/lang-json": "^6.0.1",
     "@codemirror/lint": "^6.8.0",
     "@headlessui/react": "1.7.18",
-    "@headlessui/tailwindcss": "0.2.0",
+    "@headlessui/tailwindcss": "0.2.1",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
     "@langchain/anthropic": "^0.1.21",
@@ -36,6 +36,7 @@
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
+    "@mui/x-tree-view": "^7.6.2",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -56,28 +57,28 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
+    "@radix-ui/react-toggle-group": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.18",
-    "@react-email/render": "^0.0.14",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.113.0",
-    "@sentry/node": "^7.113.0",
-    "@sentry/profiling-node": "^7.113.0",
-    "@sentry/types": "^7.113.0",
+    "@sentry/nextjs": "^8.8.0",
+    "@sentry/node": "^8.8.0",
+    "@sentry/profiling-node": "^8.8.0",
+    "@sentry/types": "^8.8.0",
     "@t3-oss/env-nextjs": "^0.10.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
-    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
@@ -88,7 +89,7 @@
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.4",
+    "dompurify": "^3.1.5",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
@@ -116,6 +117,7 @@
     "tailwindcss-animate": "^1.0.7",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
@@ -123,21 +125,21 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
     "@tailwindcss/forms": "^0.5.7",
-    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/jest-dom": "^6.4.6",
     "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
@@ -148,7 +150,7 @@
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.14",
-    "tailwindcss": "^3.4.3",
+    "tailwindcss": "^3.4.4",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.2",

web/public/generated/api/openapi.yml

@@ -397,7 +397,10 @@ paths:
           description: ''
   /api/public/ingestion:
     post:
-      description: Batched ingestion for Langfuse Tracing
+      description: >-
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via
+        the API, such as to build your own Langfuse client implementation, this
+        is the only API route you need to implement.
       operationId: ingestion_batch
       tags:
         - Ingestion
@@ -446,6 +449,14 @@ paths:
                   type: array
                   items:
                     $ref: '#/components/schemas/IngestionEvent'
+                  description: >-
+                    Batch of tracing events to be ingested. Discriminated by
+                    attribute `type`.
+                metadata:
+                  nullable: true
+                  description: >-
+                    Optional. Metadata field used by the Langfuse SDKs for
+                    debugging.
               required:
                 - batch
   /api/public/metrics/daily:
@@ -997,9 +1008,10 @@ paths:
         - name: source
           in: query
           description: Retrieve only scores from a specific source.
-          required: true
+          required: false
           schema:
             $ref: '#/components/schemas/ScoreSource'
+            nullable: true
         - name: operator
           in: query
           description: Retrieve only scores with <operator> value.
@@ -1015,6 +1027,13 @@ paths:
             type: number
             format: double
             nullable: true
+        - name: scoreIds
+          in: query
+          description: Comma-separated list of score IDs to limit the results to.
+          required: false
+          schema:
+            type: string
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1137,7 +1156,10 @@ paths:
       security: *ref_0
   /api/public/sessions/{sessionId}:
     get:
-      description: Get a session
+      description: >-
+        Get a session. Please note that `traces` on this endpoint are not
+        paginated, if you plan to fetch large sessions, consider `GET
+        /api/public/traces?sessionId=<sessionId>`
       operationId: sessions_get
       tags:
         - Sessions
@@ -1262,6 +1284,12 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: sessionId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
         - name: fromTimestamp
           in: query
           description: Retrieve only traces newer than this datetime (ISO 8601).
@@ -1955,8 +1983,19 @@ components:
                 type:
                   type: string
                   enum:
-                    - event-create
-            - $ref: '#/components/schemas/CreateEventEvent'
+                    - span-create
+            - $ref: '#/components/schemas/CreateSpanEvent'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - span-update
+            - $ref: '#/components/schemas/UpdateSpanEvent'
           required:
             - type
         - type: object
@@ -1988,19 +2027,8 @@ components:
                 type:
                   type: string
                   enum:
-                    - span-create
-            - $ref: '#/components/schemas/CreateSpanEvent'
-          required:
-            - type
-        - type: object
-          allOf:
-            - type: object
-              properties:
-                type:
-                  type: string
-                  enum:
-                    - span-update
-            - $ref: '#/components/schemas/UpdateSpanEvent'
+                    - event-create
+            - $ref: '#/components/schemas/CreateEventEvent'
           required:
             - type
         - type: object
@@ -2323,13 +2351,20 @@ components:
       properties:
         id:
           type: string
+          description: UUID v4 that identifies the event
         timestamp:
           type: string
-        metadata: {}
+          description: >-
+            Datetime (ISO 8601) of event creation in client. Should be as close
+            to actual event creation in client as possible, this timestamp will
+            be used for ordering of events in future release. Resolution:
+            milliseconds (required), microseconds (optimal).
+        metadata:
+          nullable: true
+          description: Optional. Metadata field used by the Langfuse SDKs for debugging.
       required:
         - id
         - timestamp
-        - metadata
     TraceEvent:
       title: TraceEvent
       type: object
@@ -2499,6 +2534,7 @@ components:
         totalCost:
           type: number
           format: double
+          description: Total model cost in USD
         usage:
           type: array
           items:
@@ -2521,14 +2557,28 @@ components:
           nullable: true
         inputUsage:
           type: integer
+          description: Total number of generation input units (e.g. tokens)
         outputUsage:
           type: integer
+          description: Total number of generation output units (e.g. tokens)
         totalUsage:
           type: integer
+          description: Total number of generation total units (e.g. tokens)
+        countTraces:
+          type: integer
+        countObservations:
+          type: integer
+        totalCost:
+          type: number
+          format: double
+          description: Total model cost in USD
       required:
         - inputUsage
         - outputUsage
         - totalUsage
+        - countTraces
+        - countObservations
+        - totalCost
     Observations:
       title: Observations
       type: object

web/public/generated/postman/collection.json

@@ -337,7 +337,7 @@
           "_type": "endpoint",
           "name": "Batch",
           "request": {
-            "description": "Batched ingestion for Langfuse Tracing",
+            "description": "Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.",
             "url": {
               "raw": "{{baseUrl}}/api/public/ingestion",
               "host": [
@@ -356,7 +356,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
+              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ],\n    \"metadata\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -755,7 +755,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -804,6 +804,11 @@
                   "key": "value",
                   "value": "",
                   "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "scoreIds",
+                  "value": "",
+                  "description": "Comma-separated list of score IDs to limit the results to."
                 }
               ],
               "variable": []
@@ -890,7 +895,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a session",
+            "description": "Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`",
             "url": {
               "raw": "{{baseUrl}}/api/public/sessions/:sessionId",
               "host": [
@@ -963,7 +968,7 @@
           "request": {
             "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -993,6 +998,11 @@
                   "value": "",
                   "description": null
                 },
+                {
+                  "key": "sessionId",
+                  "value": "",
+                  "description": null
+                },
                 {
                   "key": "fromTimestamp",
                   "value": "",

web/sentry.client.config.ts

@@ -4,7 +4,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
   Sentry.init({
     dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
     // Replay may only be enabled for the client-side
-    integrations: [new Sentry.Replay()],
+    integrations: [Sentry.replayIntegration()],
 
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.

web/sentry.server.config.ts

@@ -1,8 +1,6 @@
-import { prisma } from "@langfuse/shared/src/db";
 import * as Sentry from "@sentry/nextjs";
-import { ProfilingIntegration } from "@sentry/profiling-node";
-import type { SamplingContext, TransactionEvent } from "@sentry/types";
-import { jsonSchema } from "@/src/utils/zod";
+import { jsonSchema } from "@langfuse/shared";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
 
 if (process.env.NEXT_PUBLIC_SENTRY_DSN)
   Sentry.init({
@@ -12,7 +10,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampler: (samplingContext: SamplingContext) => {
+    tracesSampler: (samplingContext) => {
       if (
         samplingContext.request &&
         samplingContext.request.url &&
@@ -23,9 +21,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       if (
         samplingContext.request &&
         samplingContext.request.url &&
-        samplingContext.request.url.includes("api/auth") &&
-        samplingContext.transactionContext.status !== "ok" &&
-        samplingContext.transactionContext.status !== "unauthenticated"
+        samplingContext.request.url.includes("api/auth")
       ) {
         return 0.1;
       }
@@ -35,14 +31,13 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations
-      new ProfilingIntegration(),
-      new Sentry.Integrations.Prisma({ client: prisma }),
-      Sentry.metrics.metricsAggregatorIntegration(),
+      Sentry.prismaIntegration(),
+      nodeProfilingIntegration(),
     ],
 
     // filter out passwords from the signup request body
     // transaction events are sentry transactions which include logs and spans.
-    beforeSendTransaction(transaction: TransactionEvent) {
+    beforeSendTransaction(transaction) {
       if (
         transaction.request &&
         typeof transaction.request.data === "string" &&

web/src/__tests__/llm-api-key.servertest.ts

@@ -3,7 +3,7 @@
 
 import type { Session } from "next-auth";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { ModelProvider } from "@langfuse/shared";
+import { LLMAdapter } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
@@ -38,11 +38,20 @@ describe("llmApiKey.all RPC", () => {
 
   it("should create an llm api key", async () => {
     const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;
 
     await caller.llmApiKey.create({
       projectId,
       secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
     });
 
     const llmApiKeys = await prisma.llmApiKeys.findMany({
@@ -55,18 +64,31 @@ describe("llmApiKey.all RPC", () => {
     expect(llmApiKeys[0].projectId).toBe(projectId);
     expect(llmApiKeys[0].secretKey).not.toBeNull();
     expect(llmApiKeys[0].secretKey).not.toEqual(secret);
-    expect(llmApiKeys[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
     // this has to be 3 dots and the last 4 characters of the secret
     expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
   });
 
   it("should create and get an llm api key", async () => {
     const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;
 
     await caller.llmApiKey.create({
       projectId,
       secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
     });
 
     const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
@@ -77,16 +99,23 @@ describe("llmApiKey.all RPC", () => {
 
     expect(dbLlmApiKeys.length).toBe(1);
 
-    const llmApiKeys = await caller.llmApiKey.all({
+    const { data: llmApiKeys } = await caller.llmApiKey.all({
       projectId,
     });
 
-    expect(llmApiKeys.data.length).toBe(1);
-    expect(llmApiKeys.data[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys.length).toBe(1);
+    expect(llmApiKeys[0].projectId).toBe(projectId);
+    expect(llmApiKeys[0].secretKey).not.toBeNull();
+    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
     // this has to be 3 dots and the last 4 characters of the secret
-    expect(llmApiKeys.data[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
+    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
 
     // response must not contain the secret key itself
-    expect(llmApiKeys.data[0]).not.toHaveProperty("secretKey");
+    expect(llmApiKeys[0]).not.toHaveProperty("secretKey");
   });
 });

web/src/__tests__/metrics-api.servertest.ts

@@ -68,12 +68,18 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 333,
         outputUsage: 0,
         totalUsage: 333,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
       {
         model: "modelC",
         inputUsage: 666,
         outputUsage: 777,
         totalUsage: 1443,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 1024.22,
       },
     ]);
 
@@ -88,6 +94,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 100,
         outputUsage: 200,
         totalUsage: 300,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
     ]);
   });

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
+          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id  WHERE  t."project_id" = `,
           ";",
         ],
       } as const,

web/src/__tests__/scores.servertest.ts

@@ -591,5 +591,37 @@ describe("/api/public/scores API Endpoint", () => {
         message: "Invalid request data",
       });
     });
+
+    it("should filter scores by score IDs", async () => {
+      const getScore = await makeAPICall<{
+        data: [
+          {
+            id: string;
+            name: string;
+            value: number;
+          },
+        ];
+        meta: object;
+      }>("GET", `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`);
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
   });
 });

web/src/__tests__/traces.servertest.ts

@@ -238,4 +238,62 @@ describe("/api/public/traces API Endpoint", () => {
     expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
     expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
   });
+
+  it("should filter traces by session ID", async () => {
+    const sessionId = "test-session-id";
+    const anotherSessionId = "another-session-id";
+
+    // Create traces with different session IDs
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-1",
+      name: "test-trace-1",
+      sessionId,
+      userId: "user-1",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-2",
+      name: "test-trace-2",
+      sessionId: anotherSessionId,
+      userId: "user-2",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    // Filter by session ID
+    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${sessionId}`,
+    );
+
+    expect(tracesBySessionId.status).toBe(200);
+    expect(tracesBySessionId.body.data).toHaveLength(1);
+    expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
+
+    // Filter by another session ID
+    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${anotherSessionId}`,
+    );
+
+    expect(tracesByAnotherSessionId.status).toBe(200);
+    expect(tracesByAnotherSessionId.body.data).toHaveLength(1);
+    expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
+
+    // Filter by non-existent session ID
+    const tracesByNonExistentSessionId =
+      await makeAPICall<GetTracesAPIResponse>(
+        "GET",
+        `/api/public/traces?sessionId=non-existent-session-id`,
+      );
+
+    expect(tracesByNonExistentSessionId.status).toBe(200);
+    expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);
+  });
 });

web/src/__tests__/zod.servertest.ts

@@ -1,4 +1,4 @@
-import { paginationZod } from "@/src/utils/zod";
+import { paginationZod } from "@langfuse/shared";
 import { ZodError } from "zod";
 
 // Create test cases

web/src/components/LangfuseLogo.tsx

@@ -1,8 +1,115 @@
-import { AlertTriangle, Check } from "lucide-react";
+import {
+  AlertTriangle,
+  Check,
+  Github,
+  HardDriveDownload,
+  Newspaper,
+} from "lucide-react";
 
 import { VERSION } from "@/src/constants";
 import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
+import {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+} from "@/src/components/ui/dropdown-menu";
+import { ArrowUp } from "lucide-react";
+import { api } from "@/src/utils/api";
+import { Button } from "@/src/components/ui/button";
+import Link from "next/link";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
+
+const VersionLabel = ({ className }: { className?: string }) => {
+  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
+    refetchOnMount: false,
+    refetchOnWindowFocus: false,
+    refetchOnReconnect: false,
+    enabled: !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION, // do not check for updates on Langfuse Cloud
+    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
+  });
+  const isEeVersion =
+    useIsEeEnabled() && !Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
+
+  const hasUpdate =
+    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+    checkUpdate.data &&
+    checkUpdate.data.updateType;
+
+  const color =
+    checkUpdate.data?.updateType === "major"
+      ? "text-dark-red"
+      : checkUpdate.data?.updateType === "minor"
+        ? "text-dark-yellow"
+        : undefined;
+
+  return (
+    <DropdownMenu>
+      <DropdownMenuTrigger asChild>
+        <Button variant="ghost" size="xs" className={className}>
+          {VERSION}
+          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
+        </Button>
+      </DropdownMenuTrigger>
+      <DropdownMenuContent>
+        {hasUpdate ? (
+          <>
+            <DropdownMenuLabel>
+              New {checkUpdate.data?.updateType} version:{" "}
+              {checkUpdate.data?.latestRelease}
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ? (
+          <>
+            <DropdownMenuLabel>This is the latest release</DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : null}
+        {isEeVersion && (
+          <>
+            <DropdownMenuLabel className="font-normal italic">
+              Enterprise Edition
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        )}
+        <DropdownMenuItem asChild>
+          <Link
+            href="https://github.com/langfuse/langfuse/releases"
+            target="_blank"
+          >
+            <Github size={16} className="mr-2" />
+            Releases
+          </Link>
+        </DropdownMenuItem>
+        <DropdownMenuItem asChild>
+          <Link href="https://langfuse.com/changelog" target="_blank">
+            <Newspaper size={16} className="mr-2" />
+            Changelog
+          </Link>
+        </DropdownMenuItem>
+        {hasUpdate && (
+          <>
+            <DropdownMenuSeparator />
+            <DropdownMenuItem asChild>
+              <Link
+                href="https://langfuse.com/docs/deployment/self-host#update"
+                target="_blank"
+              >
+                <HardDriveDownload size={16} className="mr-2" />
+                Update
+              </Link>
+            </DropdownMenuItem>
+          </>
+        )}
+      </DropdownMenuContent>
+    </DropdownMenu>
+  );
+};
 
 export const LangfuseIcon = ({
   size = 32,
@@ -65,22 +172,12 @@ export const LangfuseLogo = ({
       <span
         className={cn(
           "ml-2 font-mono font-semibold",
-          size === "sm" ? "text-sm" : "text-xl",
+          size === "sm" ? "text-sm" : "text-lg",
         )}
       >
         Langfuse
       </span>
-      {version && (
-        <a
-          href="https://github.com/langfuse/langfuse/releases"
-          target="_blank"
-          rel="noopener"
-          title="View releases on GitHub"
-          className="ml-2 text-xs text-muted-foreground"
-        >
-          {VERSION}
-        </a>
-      )}
+      {version && <VersionLabel className="ml-2" />}
     </div>
   </div>
 );

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -3,7 +3,7 @@ import Link from "next/link";
 import { Label } from "@/src/components/ui/label";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
-import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { type UIModelParams } from "@langfuse/shared";
 import { ArrowTopRightIcon } from "@radix-ui/react-icons";
 
 export const LLMApiKeyComponent = (p: {
@@ -39,11 +39,7 @@ export const LLMApiKeyComponent = (p: {
     );
   }
 
-  const model = p.modelParams.model.value;
-  const modelProvider = Object.entries(supportedModels).find((providerData) =>
-    (providerData[1] as any as string[]).includes(model),
-  )?.[0];
-
+  const modelProvider = p.modelParams.provider.value;
   const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
 
   return (
@@ -51,9 +47,11 @@ export const LLMApiKeyComponent = (p: {
       <Label className="text-xs font-semibold">API key</Label>
       <div>
         {apiKey ? (
-          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
-            {apiKey.displaySecretKey}
-          </span>
+          <Link href={`/project/${p.projectId}/settings#llm-api-keys`}>
+            <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+              {apiKey.displaySecretKey}
+            </span>
+          </Link>
         ) : undefined}
       </div>
       {/* Custom form message to include a link to the already existing prompt */}

web/src/components/ModelParameters/index.tsx

@@ -8,11 +8,12 @@ import {
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
 import { Switch } from "@/src/components/ui/switch";
+import { CreateLLMApiKeyDialog } from "@/src/features/public-api/components/CreateLLMApiKeyDialog";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { cn } from "@/src/utils/tailwind";
 import {
-  ModelProvider,
-  supportedModels,
+  type LLMAdapter,
+  type supportedModels,
   type UIModelParams,
 } from "@langfuse/shared";
 
@@ -20,7 +21,8 @@ import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
 
 export type ModelParamsContext = {
   modelParams: UIModelParams;
-  availableModels?: UIModelParams[];
+  availableProviders: string[];
+  availableModels: string[];
   updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
     value: UIModelParams[Key]["value"],
@@ -31,6 +33,7 @@ export type ModelParamsContext = {
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
+  availableProviders,
   availableModels,
   updateModelParamValue,
   setModelParamEnabled,
@@ -43,76 +46,71 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-4">
-        <ModelParamsSelect
-          title="Provider"
-          modelParamsKey="provider"
-          disabled={formDisabled}
-          value={modelParams.provider.value}
-          options={
-            availableModels
-              ? [...new Set(availableModels.map((m) => m.provider.value))]
-              : Object.values(ModelProvider)
-          }
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSelect
-          title="Model name"
-          modelParamsKey="model"
-          disabled={formDisabled}
-          value={modelParams.model.value}
-          options={Object.values(
-            availableModels
-              ? availableModels
-                  .filter(
-                    (m) => m.provider.value === modelParams.provider.value,
-                  )
-                  .map((m) => m.model.value)
-              : supportedModels[modelParams.provider.value],
-          )}
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Temperature"
-          modelParamsKey="temperature"
-          formDisabled={formDisabled}
-          enabled={modelParams.temperature.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.temperature.value}
-          min={0}
-          max={modelParams.maxTemperature.value}
-          step={0.01}
-          tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Output token limit"
-          modelParamsKey="max_tokens"
-          formDisabled={formDisabled}
-          enabled={modelParams.max_tokens.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.max_tokens.value}
-          min={1}
-          max={4096}
-          step={1}
-          tooltip="The maximum number of tokens that can be generated in the chat completion."
-          updateModelParam={updateModelParamValue}
-        />
-        <ModelParamsSlider
-          title="Top P"
-          modelParamsKey="top_p"
-          formDisabled={formDisabled}
-          enabled={modelParams.top_p.enabled}
-          setModelParamEnabled={setModelParamEnabled}
-          value={modelParams.top_p.value}
-          min={0}
-          max={1}
-          step={0.01}
-          tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
-          updateModelParam={updateModelParamValue}
-        />
-        <LLMApiKeyComponent {...{ projectId, modelParams }} />
-      </div>
+      {availableProviders.length === 0 ? (
+        <>
+          <p className="text-sm">No LLM API key set in project.</p>
+          <CreateLLMApiKeyDialog />
+        </>
+      ) : (
+        <div className="space-y-4">
+          <ModelParamsSelect
+            title="Provider"
+            modelParamsKey="provider"
+            disabled={formDisabled}
+            value={modelParams.provider.value}
+            options={availableProviders}
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSelect
+            title="Model name"
+            modelParamsKey="model"
+            disabled={formDisabled}
+            value={modelParams.model.value}
+            options={availableModels}
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Temperature"
+            modelParamsKey="temperature"
+            formDisabled={formDisabled}
+            enabled={modelParams.temperature.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.temperature.value}
+            min={0}
+            max={modelParams.maxTemperature.value}
+            step={0.01}
+            tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Output token limit"
+            modelParamsKey="max_tokens"
+            formDisabled={formDisabled}
+            enabled={modelParams.max_tokens.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.max_tokens.value}
+            min={1}
+            max={4096}
+            step={1}
+            tooltip="The maximum number of tokens that can be generated in the chat completion."
+            updateModelParam={updateModelParamValue}
+          />
+          <ModelParamsSlider
+            title="Top P"
+            modelParamsKey="top_p"
+            formDisabled={formDisabled}
+            enabled={modelParams.top_p.enabled}
+            setModelParamEnabled={setModelParamEnabled}
+            value={modelParams.top_p.value}
+            min={0}
+            max={1}
+            step={0.01}
+            tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
+            updateModelParam={updateModelParamValue}
+          />
+          <LLMApiKeyComponent {...{ projectId, modelParams }} />
+        </div>
+      )}
     </div>
   );
 };
@@ -148,7 +146,7 @@ const ModelParamsSelect = ({
         onValueChange={(value) =>
           updateModelParam(
             modelParamsKey,
-            value as (typeof supportedModels)[ModelProvider][number],
+            value as (typeof supportedModels)[LLMAdapter][number],
           )
         }
         value={value}

web/src/components/Support.tsx

@@ -67,7 +67,7 @@ const supportChannels = [
     title: "Slack Connect",
     description: "Get a dedicated support channel for you and your team.",
     href: "mailto:help@langfuse.com?subject=Slack%20Connect%20Request&body=I'd%20like%20to%20request%20a%20dedicated%20Slack%20Connect%20channel%20for%20me%20and%20my%20team.%0D%0A%0D%0AUsers%20(emails)%20to%20include%20besides%20mine%3A%0D%0A%0D%0A",
-    buttonText: "Request Slack Connect (via Email)",
+    buttonText: "Request via Email",
   },
 ];
 

web/src/components/deleteButton.tsx

@@ -53,16 +53,13 @@ export function DeleteButton({
     },
   });
 
-  if (!hasAccess) {
-    return null;
-  }
-
   return (
     <Popover key={itemId}>
       <PopoverTrigger asChild>
         <Button
           variant={isTableAction ? "ghost" : "outline"}
           size={isTableAction ? "xs" : "icon"}
+          disabled={!hasAccess}
           onClick={() =>
             type === "trace"
               ? capture("trace:delete_form_open", {

web/src/components/grouped-score-badge.tsx

@@ -4,9 +4,13 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import {
+  isBooleanDataType,
+  isCategoricalDataType,
+} from "@/src/features/manual-scoring/lib/helpers";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
 import { cn } from "@/src/utils/tailwind";
-import { MessageCircle } from "lucide-react";
+import { MessageCircleMore } from "lucide-react";
 
 export const GroupedScoreBadges = ({
   scores,
@@ -34,11 +38,13 @@ export const GroupedScoreBadges = ({
     <div className={cn("text-xs", props.className)}>
       {props.scores.map((s, i) => (
         <span key={i} className="group/score ml-1 first:ml-0">
-          {s.value.toFixed(2)}
+          {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
+            ? s.stringValue
+            : s.value.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
-                <MessageCircle size={12} />
+                <MessageCircleMore size={12} />
               </HoverCardTrigger>
               <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                 <p>{s.comment}</p>
@@ -73,9 +79,12 @@ export const GroupedScoreBadges = ({
             <Badge
               variant="outline"
               key={name}
-              className="break-all font-normal"
+              className="grid grid-cols-[1fr,auto] gap-1 font-normal"
             >
-              {name}: <ScoresOfGroup scores={scores} className="ml-2" />
+              <p className="truncate" title={name}>
+                {name}:
+              </p>
+              <ScoresOfGroup scores={scores} />
             </Badge>
           ))}
       </>

web/src/components/layouts/layout.tsx

@@ -32,6 +32,7 @@ import useLocalStorage from "@/src/components/useLocalStorage";
 import { ProjectNavigation } from "@/src/components/projectNavigation";
 import DOMPurify from "dompurify";
 import { ThemeToggle } from "@/src/features/theming/ThemeToggle";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 const signOutUser = async () => {
   localStorage.clear();
@@ -76,6 +77,7 @@ export default function Layout(props: PropsWithChildren) {
     session.data?.environment.enableExperimentalFeatures ?? false;
 
   const projectId = router.query.projectId as string | undefined;
+  const isEeEnabled = useIsEeEnabled();
 
   const mapNavigation = (route: Route): NavigationItem | null => {
     // Project-level routes
@@ -91,11 +93,14 @@ export default function Layout(props: PropsWithChildren) {
     )
       return null;
 
-    // cloud only
+    // check ee or cloud requirements
     if (
-      route.cloudOnly !== undefined &&
-      // the feature should be available in local development
-      route.cloudOnly !== (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined)
+      route.requires !== undefined &&
+      !(
+        (route.requires === "cloud" &&
+          Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)) ||
+        (route.requires === "cloud-or-ee" && isEeEnabled)
+      )
     )
       return null;
 
@@ -521,9 +526,9 @@ const MainNavigation: React.FC<{
   onNavitemClick?: () => void;
   className?: string;
 }> = ({ nav, onNavitemClick, className }) => {
-  const [isOpen, setIsOpen] = useLocalStorage(
-    "sidebar-tracing-default-open",
-    false,
+  const [isOpen, setIsOpen] = useLocalStorage<Record<string, boolean>>(
+    "sidebar-item-default-open",
+    {},
   );
 
   return (
@@ -572,14 +577,20 @@ const MainNavigation: React.FC<{
               <Disclosure
                 as="div"
                 defaultOpen={
-                  item.children.some((child) => child.current) || isOpen
+                  item.children.some((child) => child.current) ||
+                  isOpen[item.name]
                 }
               >
                 {({ open }) => (
                   <>
                     <Disclosure.Button
                       className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold hover:bg-primary-foreground hover:text-primary-accent"
-                      onClick={() => setIsOpen(!isOpen)}
+                      onClick={() =>
+                        setIsOpen((prev) => ({
+                          ...prev,
+                          [item.name]: !prev[item.name],
+                        }))
+                      }
                     >
                       {item.icon && (
                         <item.icon

web/src/components/layouts/routes.ts

@@ -24,7 +24,7 @@ export type Route = {
   children?: Array<Route>; // folder
   bottom?: boolean; // bottom of the sidebar, only for first level routes
   newTab?: boolean; // open in new tab
-  cloudOnly?: boolean; // only available in cloud
+  requires?: "cloud" | "cloud-or-ee"; // feature requires cloud or ee
 };
 
 export const ROUTES: Route[] = [
@@ -62,25 +62,25 @@ export const ROUTES: Route[] = [
   {
     name: "Evaluation",
     icon: Lightbulb,
-    cloudOnly: true,
+    requires: "cloud",
     label: "Beta",
     children: [
       {
         name: "Templates",
         pathname: `/project/[projectId]/evals/templates`,
-        cloudOnly: true,
+        requires: "cloud",
         rbacScope: "evalTemplate:read",
       },
       {
         name: "Configs",
         pathname: `/project/[projectId]/evals/configs`,
-        cloudOnly: true,
+        requires: "cloud",
         rbacScope: "evalJob:read",
       },
       {
         name: "Log",
         pathname: `/project/[projectId]/evals/log`,
-        cloudOnly: true,
+        requires: "cloud",
         rbacScope: "evalJobExecution:read",
       },
     ],
@@ -100,7 +100,7 @@ export const ROUTES: Route[] = [
     name: "Playground",
     pathname: "/project/[projectId]/playground",
     icon: TerminalIcon,
-    cloudOnly: true,
+    requires: "cloud-or-ee",
     label: "Beta",
   },
   {

web/src/components/session/index.tsx

@@ -7,38 +7,24 @@ import { IOPreview } from "@/src/components/trace/IOPreview";
 import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 import { Badge } from "@/src/components/ui/badge";
 import { Card } from "@/src/components/ui/card";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect, useRef } from "react";
-import { useWindowVirtualizer } from "@tanstack/react-virtual";
-import { type RouterOutput } from "@/src/utils/types";
-import {
-  DataTableRowHeightSwitch,
-  type RowHeight,
-  useRowHeightLocalStorage,
-} from "@/src/components/table/data-table-row-height-switch";
-import { ScrollArea } from "@/src/components/ui/scroll-area";
+import { useEffect, useState } from "react";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
+import { Button } from "@/src/components/ui/button";
 
-// do not use the usual table row heights here
-const rowHeightMapping: Record<RowHeight, number> = {
-  s: 200,
-  m: 350,
-  l: 700,
-};
+// some projects have thousands of traces in a sessions, paginate to avoid rendering all at once
+const PAGE_SIZE = 50;
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
-  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
-    "single-session",
-    "m",
-  );
+  const [visibleTraces, setVisibleTraces] = useState(PAGE_SIZE);
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -96,11 +82,6 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
-          <DataTableRowHeightSwitch
-            rowHeight={rowHeight}
-            setRowHeight={setRowHeight}
-            key="height"
-          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -121,97 +102,49 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      {session.data && (
-        <TraceCardList
-          session={session.data}
-          projectId={projectId}
-          rowHeight={rowHeightMapping[rowHeight]}
-        />
-      )}
-    </div>
-  );
-};
-
-const TraceCardList = ({
-  session,
-  projectId,
-  rowHeight,
-}: {
-  session: RouterOutput["sessions"]["byId"];
-  projectId: string;
-  rowHeight: number;
-}) => {
-  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
-
-  const virtualizer = useWindowVirtualizer({
-    count: session.traces.length,
-    estimateSize: () => rowHeight,
-    overscan: 5,
-    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
-    gap: 10,
-  });
-  useEffect(() => {
-    // re-measure when rowHeight changes to update the virtualizer
-    virtualizer.measure();
-  }, [rowHeight, virtualizer]);
-  return (
-    <div className="mt-5 border-t pt-5">
-      <div
-        ref={listVirtualizationRef}
-        style={{
-          height: `${virtualizer.getTotalSize()}px`,
-          position: "relative",
-        }}
-      >
-        {virtualizer
-          .getVirtualItems()
-          .map((virtualItem) => ({
-            virtualItem,
-            trace: session.traces[virtualItem.index],
-          }))
-          .map(({ virtualItem, trace }) => (
-            <Card
-              className="group grid w-full gap-3 overflow-hidden border-border p-2 shadow-none hover:border-ring   md:grid-cols-3"
-              key={virtualItem.key}
-              data-index={virtualItem.index}
-              style={{
-                position: "absolute",
-                top: 0,
-                left: 0,
-                height: `${rowHeight}px`,
-                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
-              }}
-            >
-              <ScrollArea className="col-span-2 pr-3">
-                <SessionIO traceId={trace.id} />
-              </ScrollArea>
-              <ScrollArea>
-                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
-                  <Link
-                    href={`/project/${projectId}/traces/${trace.id}`}
-                    className="text-xs hover:underline"
-                  >
-                    Trace: {trace.name} ({trace.id})&nbsp;↗
-                  </Link>
-                  <div className="text-xs text-muted-foreground">
-                    {trace.timestamp.toLocaleString()}
-                  </div>
-                  <div className="mb-1 mt-2 text-xs text-muted-foreground">
-                    Scores
-                  </div>
-                  <div className="flex flex-wrap content-start items-start gap-1">
-                    <GroupedScoreBadges scores={trace.scores} />
-                  </div>
-                  <ManualScoreButton
-                    projectId={projectId}
-                    traceId={trace.id}
-                    scores={trace.scores}
-                    variant="badge"
-                  />
-                </div>
-              </ScrollArea>
-            </Card>
-          ))}
+      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
+        {session.data?.traces.slice(0, visibleTraces).map((trace) => (
+          <Card
+            className="group grid gap-3 border-border p-2 shadow-none hover:border-ring md:grid-cols-3"
+            key={trace.id}
+          >
+            <SessionIO traceId={trace.id} />
+            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
+              <Link
+                href={`/project/${projectId}/traces/${trace.id}`}
+                className="text-xs hover:underline"
+              >
+                Trace: {trace.name} ({trace.id})&nbsp;↗
+              </Link>
+              <div className="text-xs text-muted-foreground">
+                {trace.timestamp.toLocaleString()}
+              </div>
+              <div className="mb-1 mt-2 text-xs text-muted-foreground">
+                Scores
+              </div>
+              <div className="mb-1 flex flex-wrap content-start items-start gap-1">
+                <GroupedScoreBadges scores={trace.scores} />
+              </div>
+              <AnnotateButton
+                projectId={projectId}
+                traceId={trace.id}
+                scores={trace.scores}
+                variant="badge"
+                type="session"
+                source="SessionDetail"
+              />
+            </div>
+          </Card>
+        ))}
+        {session.data?.traces && session.data.traces.length > visibleTraces && (
+          <Button
+            onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
+            variant="ghost"
+            className="self-center"
+          >
+            {`Load ${Math.min(session.data.traces.length - visibleTraces, PAGE_SIZE)} More`}
+          </Button>
+        )}
       </div>
     </div>
   );
@@ -230,9 +163,8 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
-
   return (
-    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
+    <div className="col-span-2 flex flex-col gap-2 p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"

web/src/components/table/data-table.tsx

@@ -50,6 +50,8 @@ interface DataTableProps<TData, TValue> {
   help?: { description: string; href: string };
   rowHeight?: RowHeight;
   className?: string;
+  paginationClassName?: string;
+  isBorderless?: boolean;
 }
 
 export interface AsyncTableData<T> {
@@ -71,6 +73,9 @@ export function DataTable<TData extends object, TValue>({
   orderBy,
   setOrderBy,
   rowHeight,
+  className,
+  paginationClassName,
+  isBorderless = false,
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight);
@@ -105,8 +110,18 @@ export function DataTable<TData extends object, TValue>({
 
   return (
     <>
-      <div className="flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto">
-        <div className="w-full overflow-auto rounded-md border">
+      <div
+        className={cn(
+          "flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto",
+          className,
+        )}
+      >
+        <div
+          className={cn(
+            "w-full overflow-auto",
+            isBorderless ? "" : "rounded-md border",
+          )}
+        >
           <Table>
             <TableHeader>
               {table.getHeaderGroups().map((headerGroup) => (
@@ -242,7 +257,12 @@ export function DataTable<TData extends object, TValue>({
         <div className="grow"></div>
       </div>
       {pagination !== undefined ? (
-        <div className="bg:background sticky bottom-0 z-10 flex w-full justify-end font-medium">
+        <div
+          className={cn(
+            "sticky bottom-0 z-10 flex w-full justify-end bg-background font-medium",
+            paginationClassName,
+          )}
+        >
           <DataTablePagination
             table={table}
             paginationOptions={pagination.options}

web/src/components/table/use-cases/models.tsx

@@ -1,6 +1,12 @@
 import { DataTable } from "@/src/components/table/data-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { Button } from "@/src/components/ui/button";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { useState } from "react";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
@@ -10,6 +16,7 @@ import { type Prisma, type Model } from "@langfuse/shared/src/db";
 import Decimal from "decimal.js";
 import { Trash } from "lucide-react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+import { cn } from "@/src/utils/tailwind";
 
 export type ModelTableRow = {
   modelId: string;
@@ -222,13 +229,12 @@ export default function ModelTable({ projectId }: { projectId: string }) {
       accessorKey: "actions",
       header: "Actions",
       cell: ({ row }) => {
-        return row.original.maintainer === "User" ? (
+        return (
           <DeleteModelButton
             projectId={projectId}
             modelId={row.original.modelId}
+            isBuiltIn={row.original.maintainer === "Langfuse"}
           />
-        ) : (
-          <div className="h-6" />
         );
       },
     },
@@ -287,10 +293,13 @@ export default function ModelTable({ projectId }: { projectId: string }) {
 const DeleteModelButton = ({
   modelId,
   projectId,
+  isBuiltIn,
 }: {
   modelId: string;
   projectId: string;
+  isBuiltIn?: boolean;
 }) => {
+  const [isOpen, setIsOpen] = useState(false);
   const utils = api.useUtils();
   const capture = usePostHogClientCapture();
   const mut = api.models.delete.useMutation({
@@ -304,32 +313,48 @@ const DeleteModelButton = ({
     scope: "models:CUD",
   });
 
-  if (!hasAccess) {
-    return null;
-  }
-
   return (
-    <Button
-      size="xs"
-      variant="ghost"
-      onClick={() => {
-        const confirmDelete = window.confirm(
-          "Are you sure you want to delete this model?",
-        );
-        if (confirmDelete) {
-          capture("models:delete_button_click");
-          mut
-            .mutateAsync({
-              projectId,
-              modelId,
-            })
-            .catch((error) => {
-              console.error(error);
-            });
-        }
-      }}
-    >
-      <Trash size={14} />
-    </Button>
+    <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
+      <PopoverTrigger asChild>
+        <Button
+          variant="ghost"
+          size="xs"
+          disabled={!hasAccess || isBuiltIn}
+          title={
+            isBuiltIn ? "Built-in models cannot be deleted" : "Delete model"
+          }
+          className={cn(
+            isBuiltIn &&
+              "disabled:pointer-events-auto disabled:cursor-not-allowed",
+          )}
+        >
+          <Trash className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action permanently deletes this model definition.
+        </p>
+        <div className="flex justify-end space-x-4">
+          <Button
+            type="button"
+            variant="destructive"
+            loading={mut.isLoading}
+            onClick={() => {
+              capture("models:delete_button_click");
+              mut.mutateAsync({
+                projectId,
+                modelId,
+              });
+
+              setIsOpen(false);
+            }}
+          >
+            Delete Model
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
   );
 };

web/src/components/table/use-cases/score-configs.tsx

@@ -0,0 +1,174 @@
+import React from "react";
+import { Card } from "@/src/components/ui/card";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import { DataTable } from "@/src/components/table/data-table";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { NumberParam, useQueryParams, withDefault } from "use-query-params";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
+
+type ScoreConfigTableRow = {
+  id: string;
+  name: string;
+  dataType: ScoreDataType;
+  createdAt: string;
+  updatedAt: string;
+  range: {
+    maxValue?: number | null;
+    minValue?: number | null;
+    categories?: Prisma.JsonValue | null;
+  };
+  description?: string | null;
+};
+
+function getConfigRange(
+  originalRow: ScoreConfigTableRow,
+): Prisma.JsonValue | undefined {
+  const { range, dataType } = originalRow;
+  if (isNumericDataType(dataType)) {
+    return [
+      { minValue: range.minValue ?? "-∞", maxValue: range.maxValue ?? "∞" },
+    ];
+  }
+  return range.categories;
+}
+
+export function ScoreConfigsTable({ projectId }: { projectId: string }) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "scoreConfigs",
+    "s",
+  );
+
+  const configs = api.scoreConfigs.all.useQuery({
+    projectId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  const totalCount = configs.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<ScoreConfigTableRow>[] = [
+    {
+      accessorKey: "name",
+      id: "name",
+      header: "Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "dataType",
+      id: "dataType",
+      header: "Data Type",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "range",
+      id: "range",
+      header: "Range",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const range = getConfigRange(row.original);
+
+        return !!range ? (
+          <IOTableCell data={range} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "description",
+      id: "description",
+      header: "Description",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const value = row.original.description;
+
+        return !!value ? (
+          <IOTableCell data={value} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "id",
+      id: "id",
+      header: "Config ID",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "createdAt",
+      id: "createdAt",
+      header: "Created At",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<ScoreConfigTableRow>(
+      "scoreConfigsColumnVisibility",
+      columns,
+    );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+      />
+      <Card className="mb-4 flex max-h-[calc(100dvh-40rem)] flex-col overflow-hidden">
+        <DataTable
+          columns={columns}
+          data={
+            configs.isLoading
+              ? { isLoading: true, isError: false }
+              : configs.isError
+                ? {
+                    isLoading: false,
+                    isError: true,
+                    error: configs.error.message,
+                  }
+                : {
+                    isLoading: false,
+                    isError: false,
+                    data: configs.data?.configs.map((config) => ({
+                      id: config.id,
+                      name: config.name,
+                      dataType: config.dataType,
+                      description: config.description,
+                      createdAt: config.createdAt.toLocaleString(),
+                      updatedAt: config.updatedAt.toLocaleString(),
+                      range: {
+                        maxValue: config.maxValue,
+                        minValue: config.minValue,
+                        categories: config.categories,
+                      },
+                    })),
+                  }
+          }
+          pagination={{
+            pageCount: Math.ceil(totalCount / paginationState.pageSize),
+            onChange: setPaginationState,
+            state: paginationState,
+          }}
+          columnVisibility={columnVisibility}
+          onColumnVisibilityChange={setColumnVisibility}
+          rowHeight={rowHeight}
+          className="gap-2"
+          paginationClassName="-mx-2 mb-2"
+          isBorderless
+        />
+      </Card>
+    </>
+  );
+}

web/src/components/table/use-cases/scores.tsx

@@ -4,8 +4,10 @@ import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { Avatar, AvatarImage } from "@/src/components/ui/avatar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
   type ScoreOptions,
@@ -14,7 +16,7 @@ import {
 import { api } from "@/src/utils/api";
 import { utcDateOffsetByDays } from "@/src/utils/dates";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
-import type { FilterState } from "@langfuse/shared";
+import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useSession } from "next-auth/react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
@@ -24,7 +26,12 @@ export type ScoresTableRow = {
   timestamp: string;
   source: string;
   name: string;
-  value: number;
+  dataType: ScoreDataType;
+  value: string;
+  author: {
+    image?: string;
+    name?: string;
+  };
   comment?: string;
   observationId?: string;
   traceName?: string;
@@ -172,6 +179,29 @@ export default function ScoresTable({
         ) : undefined;
       },
     },
+    {
+      accessorKey: "userId",
+      header: "Trace User ID",
+      id: "userId",
+      headerTooltip: {
+        description: "The user ID associated with the trace.",
+        href: "https://langfuse.com/docs/tracing-features/users",
+      },
+      enableHiding: true,
+      enableSorting: true,
+      cell: ({ row }) => {
+        const value = row.getValue("userId");
+        return typeof value === "string" ? (
+          <>
+            <TableLink
+              path={`/project/${projectId}/users/${value}`}
+              value={value}
+              truncateAt={40}
+            />
+          </>
+        ) : undefined;
+      },
+    },
     {
       accessorKey: "timestamp",
       header: "Timestamp",
@@ -193,38 +223,39 @@ export default function ScoresTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "dataType",
+      header: "Data Type",
+      id: "dataType",
+      enableHiding: true,
+      enableSorting: true,
+    },
     {
       accessorKey: "value",
       header: "Value",
       id: "value",
       enableHiding: true,
       enableSorting: true,
-      cell: ({ row }) => {
-        const value: number = row.getValue("value");
-        return value % 1 === 0 ? value : value.toFixed(4);
-      },
     },
     {
-      accessorKey: "userId",
-      header: "User ID",
-      id: "userId",
-      headerTooltip: {
-        description: "The user ID associated with the trace.",
-        href: "https://langfuse.com/docs/tracing-features/users",
-      },
-      enableHiding: true,
-      enableSorting: true,
+      accessorKey: "author",
+      id: "author",
+      header: "Author",
       cell: ({ row }) => {
-        const value = row.getValue("userId");
-        return typeof value === "string" ? (
-          <>
-            <TableLink
-              path={`/project/${projectId}/users/${value}`}
-              value={value}
-              truncateAt={40}
-            />
-          </>
-        ) : undefined;
+        const { name, image } = row.getValue(
+          "author",
+        ) as ScoresTableRow["author"];
+        return (
+          <div className="flex items-center space-x-2">
+            <Avatar className="h-7 w-7">
+              <AvatarImage
+                src={image ?? undefined}
+                alt={name ?? "User Avatar"}
+              />
+            </Avatar>
+            <span>{name}</span>
+          </div>
+        );
       },
     },
     {
@@ -279,12 +310,21 @@ export default function ScoresTable({
       timestamp: score.timestamp.toLocaleString(),
       source: score.source,
       name: score.name,
-      value: score.value,
+      dataType: score.dataType,
+      value: isNumericDataType(score.dataType)
+        ? score.value % 1 === 0
+          ? String(score.value)
+          : score.value.toFixed(4)
+        : score.stringValue ?? "",
+      author: {
+        image: score.authorUserImage ?? undefined,
+        name: score.authorUserName ?? undefined,
+      },
       comment: score.comment ?? undefined,
       observationId: score.observationId ?? undefined,
       traceId: score.traceId,
       traceName: score.traceName ?? undefined,
-      userId: score.userId ?? undefined,
+      userId: score.traceUserId ?? undefined,
       jobConfigurationId: score.jobConfigurationId ?? undefined,
     };
   };

web/src/components/table/utils/joinTableCoreAndMetrics.ts

@@ -0,0 +1,46 @@
+export function joinTableCoreAndMetrics<
+  Core extends { id: string },
+  Metric extends { id: string },
+>(
+  userCoreData?: Core[],
+  userMetricsData?: Metric[],
+): {
+  status: "loading" | "error" | "success";
+  rows: (Core & Partial<Metric>)[] | undefined;
+} {
+  if (!userCoreData) {
+    return { status: "error", rows: undefined };
+  }
+
+  const userCoreDataProcessed = userCoreData;
+
+  if (!userMetricsData) {
+    // create an object with all the keys of the UserMetrics type with undefined value
+
+    return {
+      status: "success",
+      rows: userCoreDataProcessed.map((u) => ({
+        ...u,
+        ...({} as Partial<Metric>),
+      })),
+    };
+  }
+
+  const metricsById = userMetricsData.reduce<Record<string, Metric>>(
+    (acc, metric) => {
+      acc[metric.id] = metric;
+      return acc;
+    },
+    {},
+  );
+
+  const joinedData = userCoreDataProcessed.map((userCore) => {
+    const metrics = metricsById[userCore.id];
+    return {
+      ...userCore,
+      ...(metrics ?? ({} as Partial<Metric>)),
+    };
+  });
+
+  return { status: "success", rows: joinedData };
+}

web/src/components/trace/IOPreview.tsx

@@ -1,6 +1,6 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { z } from "zod";
-import { deepParseJson } from "@/src/utils/json";
+import { deepParseJson } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { useState } from "react";
 import { Button } from "@/src/components/ui/button";

web/src/components/trace/ObservationPreview.tsx

@@ -8,7 +8,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { Badge } from "@/src/components/ui/badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
@@ -22,6 +21,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
@@ -154,11 +154,12 @@ export const ObservationPreview = (props: {
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={props.projectId}
               traceId={preloadedObservation.traceId}
               observationId={preloadedObservation.id}
               scores={props.scores}
+              type="observation"
             />
             {observationWithInputAndOutput.data?.type === "GENERATION" && (
               <JumpToPlaygroundButton
@@ -237,7 +238,7 @@ const PromptBadge = (props: { promptId: string; projectId: string }) => {
   if (prompt.isLoading || !prompt.data) return null;
   return (
     <Link
-      href={`/project/${props.projectId}/prompts/${prompt.data.name}?version=${prompt.data.version}`}
+      href={`/project/${props.projectId}/prompts/${encodeURIComponent(prompt.data.name)}?version=${prompt.data.version}`}
     >
       <Badge>
         Prompt: {prompt.data.name}

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,6 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, $Enums } from "@langfuse/shared";
+import { type Trace, type Score, type $Enums } from "@langfuse/shared";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -10,6 +10,10 @@ import { MinusCircle, MinusIcon, PlusCircleIcon, PlusIcon } from "lucide-react";
 import { Toggle } from "@/src/components/ui/toggle";
 import { Button } from "@/src/components/ui/button";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import {
+  nestObservations,
+  treeItemColors,
+} from "@/src/components/trace/lib/helpers";
 
 export const ObservationTree = (props: {
   observations: ObservationReturnType[];
@@ -257,53 +261,14 @@ const ObservationTreeNode = (props: {
 const ColorCodedObservationType = (props: {
   observationType: $Enums.ObservationType;
 }) => {
-  const colors: Record<$Enums.ObservationType, string> = {
-    [$Enums.ObservationType.SPAN]: "bg-muted-blue",
-    [$Enums.ObservationType.GENERATION]: "bg-muted-orange",
-    [$Enums.ObservationType.EVENT]: "bg-muted-green",
-  };
-
   return (
     <span
       className={cn(
         "self-start rounded-sm p-1 text-xs",
-        colors[props.observationType],
+        treeItemColors.get(props.observationType),
       )}
     >
       {props.observationType}
     </span>
   );
 };
-
-export function nestObservations(
-  list: ObservationReturnType[],
-): NestedObservation[] {
-  if (list.length === 0) return [];
-
-  // Step 1: Create a map where the keys are object IDs, and the values are
-  // the corresponding objects with an added 'children' property.
-  const map = new Map<string, NestedObservation>();
-  for (const obj of list) {
-    map.set(obj.id, { ...obj, children: [] });
-  }
-
-  // Step 2: Create another map for the roots of all trees.
-  const roots = new Map<string, NestedObservation>();
-
-  // Step 3: Populate the 'children' arrays and root map.
-  for (const obj of map.values()) {
-    if (obj.parentObservationId) {
-      const parent = map.get(obj.parentObservationId);
-      if (parent) {
-        parent.children.push(obj);
-      }
-    } else {
-      roots.set(obj.id, obj);
-    }
-  }
-
-  // TODO sum token amounts per level
-
-  // Step 4: Return the roots.
-  return Array.from(roots.values());
-}

web/src/components/trace/TracePreview.tsx

@@ -9,7 +9,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -19,6 +18,7 @@ import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const TracePreview = ({
   trace,
@@ -54,13 +54,13 @@ export const TracePreview = ({
           <TabsList className="bg-background py-0">
             <TabsTrigger
               value="preview"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Preview
             </TabsTrigger>
             <TabsTrigger
               value="scores"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Scores
             </TabsTrigger>
@@ -95,7 +95,7 @@ export const TracePreview = ({
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={trace.projectId}
               traceId={trace.id}
               scores={scores}

web/src/components/trace/TraceTimelineView.tsx

@@ -0,0 +1,383 @@
+import { Card } from "@/src/components/ui/card";
+import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+import { type Score, type Trace } from "@langfuse/shared";
+
+import React, { useEffect, useRef, useState } from "react";
+import { SimpleTreeView } from "@mui/x-tree-view/SimpleTreeView";
+import { TreeItem } from "@mui/x-tree-view/TreeItem";
+
+import { MinusIcon, PlusIcon, PanelRightOpen } from "lucide-react";
+import { nestObservations } from "@/src/components/trace/lib/helpers";
+import { type NestedObservation } from "@/src/utils/types";
+import { cn } from "@/src/utils/tailwind";
+import { Button } from "@/src/components/ui/button";
+import {
+  type TreeItemType,
+  treeItemColors,
+} from "@/src/components/trace/lib/helpers";
+import {
+  Drawer,
+  DrawerContent,
+  DrawerTrigger,
+} from "@/src/components/ui/drawer";
+import { TracePreview } from "@/src/components/trace/TracePreview";
+import { ObservationPreview } from "@/src/components/trace/ObservationPreview";
+import useSessionStorage from "@/src/components/useSessionStorage";
+
+// Fixed widths for styling for v1
+const SCALE_WIDTH = 800;
+const STEP_SIZE = 100;
+const CARD_PADDING = 60;
+const LABEL_WIDTH = 35;
+const MIN_LABEL_WIDTH = 250;
+const TREE_INDENTATION = 12; // default in MUI X TreeView
+
+const PREDEFINED_STEP_SIZES = [
+  0.25, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10,
+];
+
+const calculateStepSize = (latency: number, scaleWidth: number) => {
+  const calculatedStepSize = latency / (scaleWidth / STEP_SIZE);
+  return (
+    PREDEFINED_STEP_SIZES.find((step) => step >= calculatedStepSize) ||
+    PREDEFINED_STEP_SIZES[PREDEFINED_STEP_SIZES.length - 1]
+  );
+};
+
+function TreeItemInner({
+  latency,
+  totalScaleSpan,
+  type,
+  startOffset = 0,
+  name,
+  children,
+  setBackgroundColor,
+  level = 0,
+  cardWidth,
+}: {
+  latency: number;
+  totalScaleSpan: number;
+  type: TreeItemType;
+  startOffset?: number;
+  name?: string | null;
+  children?: React.ReactNode;
+  setBackgroundColor: (color: string) => void;
+  level?: number;
+  cardWidth: number;
+}) {
+  const itemWidth = (latency / totalScaleSpan) * SCALE_WIDTH;
+  const itemOffsetLabelWidth = itemWidth + startOffset + LABEL_WIDTH;
+  const customLabelWidth = cardWidth - SCALE_WIDTH - CARD_PADDING;
+
+  return (
+    <div className="group my-1 grid w-full min-w-fit grid-cols-[1fr,auto] items-center">
+      <div
+        className="flex flex-row items-center gap-2"
+        style={{
+          maxWidth: customLabelWidth - level * TREE_INDENTATION,
+          minWidth: MIN_LABEL_WIDTH - LABEL_WIDTH - level * TREE_INDENTATION,
+        }}
+      >
+        <span
+          className={cn("rounded-sm p-1 text-xs", treeItemColors.get(type))}
+        >
+          {type}
+        </span>
+        <span className="w-fit-content overflow-hidden text-ellipsis whitespace-nowrap break-all text-sm">
+          {name}
+        </span>
+        <div
+          className="w-6 flex-1"
+          onClick={(event) => {
+            event.stopPropagation();
+          }}
+        >
+          <Drawer
+            onOpenChange={(open) => setBackgroundColor(open ? "!bg-muted" : "")}
+          >
+            <DrawerTrigger asChild>
+              <Button
+                className="focus:none active:none hidden justify-start hover:!bg-transparent group-hover:block"
+                type="button"
+                size="xs"
+                variant="ghost"
+              >
+                <PanelRightOpen className="h-4 w-4"></PanelRightOpen>
+              </Button>
+            </DrawerTrigger>
+            <DrawerContent className="h-1/2 w-full overflow-hidden md:w-3/5 lg:w-3/5 xl:w-3/5 2xl:w-3/5">
+              {children}
+            </DrawerContent>
+          </Drawer>
+        </div>
+      </div>
+      <div className="flex items-center" style={{ width: `${SCALE_WIDTH}px` }}>
+        <div className={`relative w-[${SCALE_WIDTH}px]`}>
+          <div className="ml-4 mr-4 h-full border-r-2"></div>
+          <div
+            className={cn(
+              "flex h-5 items-center justify-end rounded-sm",
+              itemWidth
+                ? treeItemColors.get(type)
+                : "border border-dashed bg-muted",
+            )}
+            style={{
+              width: `${itemWidth || 10}px`,
+              marginLeft: `${startOffset}px`,
+            }}
+          >
+            <span
+              className={cn(
+                "hidden justify-end text-xs text-muted-foreground group-hover:block",
+                itemOffsetLabelWidth > SCALE_WIDTH
+                  ? "mr-1"
+                  : !!latency
+                    ? "-mr-9"
+                    : "-mr-6",
+              )}
+            >
+              {!!latency ? `${latency.toFixed(2)}s` : "n/a"}
+            </span>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+function TraceTreeItem({
+  observation,
+  level = 0,
+  traceStartTime,
+  totalScaleSpan,
+  projectId,
+  scores,
+  observations,
+  cardWidth,
+}: {
+  observation: NestedObservation;
+  level: number;
+  traceStartTime: Date;
+  totalScaleSpan: number;
+  projectId: string;
+  scores: Score[];
+  observations: Array<ObservationReturnType>;
+  cardWidth: number;
+}) {
+  const { startTime, endTime } = observation || {};
+  const [backgroundColor, setBackgroundColor] = useState("");
+
+  const latency = endTime
+    ? (endTime.getTime() - startTime.getTime()) / 1000
+    : 0;
+  const startOffset =
+    ((startTime.getTime() - traceStartTime.getTime()) / totalScaleSpan / 1000) *
+    SCALE_WIDTH;
+
+  return (
+    <TreeItem
+      classes={{
+        content: `border-l border-dashed !rounded-xs ${backgroundColor} !min-w-fit hover:!bg-muted`,
+        selected: "!bg-background !important hover:!bg-muted",
+        label: "!min-w-fit",
+      }}
+      key={observation.id}
+      itemId={observation.id}
+      label={
+        <TreeItemInner
+          latency={latency}
+          type={observation.type}
+          name={observation.name}
+          startOffset={startOffset}
+          totalScaleSpan={totalScaleSpan}
+          setBackgroundColor={setBackgroundColor}
+          level={level}
+          cardWidth={cardWidth}
+        >
+          <>
+            <h3 className="mb-6 px-8 pt-8 text-2xl font-semibold tracking-tight">
+              Detail view
+            </h3>
+            <div className="overflow-y-auto px-8 pb-8 pt-2">
+              <ObservationPreview
+                observations={observations}
+                scores={scores}
+                projectId={projectId}
+                currentObservationId={observation.id}
+                traceId={observation.traceId}
+              />
+            </div>
+          </>
+        </TreeItemInner>
+      }
+    >
+      {Array.isArray(observation.children)
+        ? observation.children.map((child) => (
+            <TraceTreeItem
+              key={child.id}
+              observation={child}
+              level={level + 1}
+              traceStartTime={traceStartTime}
+              totalScaleSpan={totalScaleSpan}
+              projectId={projectId}
+              scores={scores}
+              observations={observations}
+              cardWidth={cardWidth}
+            />
+          ))
+        : null}
+    </TreeItem>
+  );
+}
+
+export function TraceTimelineView({
+  trace,
+  observations,
+  projectId,
+  scores,
+}: {
+  trace: Trace & { latency?: number };
+  observations: Array<ObservationReturnType>;
+  projectId: string;
+  scores: Score[];
+}) {
+  const { latency, name, id } = trace;
+  const [backgroundColor, setBackgroundColor] = useState("");
+  const [expandedItems, setExpandedItems] = useSessionStorage<string[]>(
+    `${trace.id}-expanded`,
+    [trace.id],
+  );
+
+  const [cardWidth, setCardWidth] = useState(0);
+  const parentRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    const handleResize = () => {
+      if (parentRef.current) {
+        const availableWidth = parentRef.current.offsetWidth;
+        setCardWidth(availableWidth);
+      }
+    };
+
+    handleResize();
+    window.addEventListener("resize", handleResize); // Recalculate on window resize
+
+    return () => {
+      window.removeEventListener("resize", handleResize);
+    };
+  }, [parentRef]);
+
+  if (!latency) return null;
+
+  const nestedObservations = nestObservations(observations);
+  const stepSize = calculateStepSize(latency, SCALE_WIDTH);
+  const totalScaleSpan = stepSize * (SCALE_WIDTH / STEP_SIZE);
+
+  return (
+    <div ref={parentRef} className="w-full">
+      <Card
+        className="flex max-h-[calc(100dvh-24rem)] flex-col overflow-x-auto overflow-y-hidden"
+        style={{ width: cardWidth }}
+      >
+        <div className="grid w-full grid-cols-[1fr,auto] items-center p-2">
+          <h3
+            className="p-2 text-2xl font-semibold tracking-tight"
+            style={{
+              minWidth: `${MIN_LABEL_WIDTH}px`,
+            }}
+          >
+            Trace Timeline
+          </h3>
+          <div
+            className="relative mr-2 h-4"
+            style={{ width: `${SCALE_WIDTH}px` }}
+          >
+            {Array.from({ length: SCALE_WIDTH / STEP_SIZE + 1 }).map(
+              (_, index) => {
+                const step = stepSize * index;
+                const isLastStep = index === SCALE_WIDTH / STEP_SIZE;
+
+                return isLastStep ? (
+                  <span
+                    className="absolute -right-2 text-xs text-muted-foreground"
+                    key={index}
+                  >
+                    {step.toFixed(2)}s
+                  </span>
+                ) : (
+                  <div
+                    key={index}
+                    className="absolute h-full border border-l text-xs"
+                    style={{ left: `${index * STEP_SIZE}px` }}
+                  >
+                    <span className="absolute left-2 text-xs text-muted-foreground">
+                      {step.toFixed(2)}s
+                    </span>
+                  </div>
+                );
+              },
+            )}
+          </div>
+        </div>
+        <div className="min-w-fit overflow-y-auto p-2">
+          <SimpleTreeView
+            slots={{
+              expandIcon: PlusIcon,
+              collapseIcon: MinusIcon,
+            }}
+            expandedItems={expandedItems}
+            onExpandedItemsChange={(_, itemIds) => setExpandedItems(itemIds)}
+            itemChildrenIndentation={TREE_INDENTATION}
+          >
+            <TreeItem
+              key={id}
+              itemId={id}
+              classes={{
+                content: `${backgroundColor} !min-w-fit !hover:bg-muted`,
+                selected: "!bg-background !important hover:!bg-muted",
+                label: "!min-w-fit",
+              }}
+              label={
+                <TreeItemInner
+                  name={name}
+                  latency={latency}
+                  totalScaleSpan={totalScaleSpan}
+                  setBackgroundColor={setBackgroundColor}
+                  type="TRACE"
+                  cardWidth={cardWidth}
+                >
+                  <div className="overflow-y-auto p-8">
+                    <h3 className="mb-6 text-2xl font-semibold tracking-tight">
+                      Detail view
+                    </h3>
+                    <TracePreview
+                      trace={trace}
+                      observations={observations}
+                      scores={scores}
+                    />
+                  </div>
+                </TreeItemInner>
+              }
+            >
+              {Boolean(nestedObservations.length)
+                ? nestedObservations.map((observation) => (
+                    <TraceTreeItem
+                      key={observation.id}
+                      observation={observation}
+                      level={1}
+                      traceStartTime={nestedObservations[0].startTime}
+                      totalScaleSpan={totalScaleSpan}
+                      projectId={projectId}
+                      scores={scores}
+                      observations={observations}
+                      cardWidth={cardWidth}
+                    />
+                  ))
+                : null}
+            </TreeItem>
+          </SimpleTreeView>
+        </div>
+      </Card>
+    </div>
+  );
+}

web/src/components/trace/index.tsx

@@ -6,7 +6,7 @@ import { TracePreview } from "./TracePreview";
 import Header from "@/src/components/layouts/header";
 import { Badge } from "@/src/components/ui/badge";
 import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
-import { StringParam, useQueryParam } from "use-query-params";
+import { StringParam, useQueryParam, withDefault } from "use-query-params";
 import { PublishTraceSwitch } from "@/src/components/publish-object-switch";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { useRouter } from "next/router";
@@ -18,12 +18,22 @@ import { ErrorPage } from "@/src/components/error-page";
 import { TagTraceDetailsPopover } from "@/src/features/tag/components/TagTraceDetailsPopover";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { Toggle } from "@/src/components/ui/toggle";
-import { Award, ChevronsDownUp, ChevronsUpDown } from "lucide-react";
+import {
+  Award,
+  ChevronsDownUp,
+  ChevronsUpDown,
+  ListTree,
+  Network,
+  Terminal,
+} from "lucide-react";
 import { usdFormatter } from "@/src/utils/numbers";
 import Decimal from "decimal.js";
 import { useCallback, useState } from "react";
 import { DeleteButton } from "@/src/components/deleteButton";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
+import { Alert, AlertDescription, AlertTitle } from "@/src/components/ui/alert";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
@@ -199,6 +209,11 @@ export function TracePage({ traceId }: { traceId: string }) {
 
   const totalCost = calculateDisplayTotalCost(trace.data?.observations ?? []);
 
+  const [selectedTab, setSelectedTab] = useQueryParam(
+    "display",
+    withDefault(StringParam, "details"),
+  );
+
   if (trace.error?.data?.code === "UNAUTHORIZED")
     return <ErrorPage message="You do not have access to this trace." />;
   if (!trace.data) return <div>loading...</div>;
@@ -227,9 +242,17 @@ export function TracePage({ traceId }: { traceId: string }) {
             />
             <DetailPageNav
               currentId={traceId}
-              path={(id) =>
-                `/project/${router.query.projectId as string}/traces/${id}`
-              }
+              path={(id) => {
+                const { view, display, projectId } = router.query;
+                const queryParams = new URLSearchParams({
+                  ...(typeof view === "string" ? { view } : {}),
+                  ...(typeof display === "string" ? { display } : {}),
+                });
+                const queryParamString = Boolean(queryParams.size)
+                  ? `?${queryParams.toString()}`
+                  : "";
+                return `/project/${projectId as string}/traces/${id}${queryParamString}`;
+              }}
               listKey="traces"
             />
             <DeleteButton
@@ -281,15 +304,67 @@ export function TracePage({ traceId }: { traceId: string }) {
           />
         </div>
       </div>
-      <div className="mt-5 flex-1 overflow-hidden border-t pt-5">
-        <Trace
-          key={trace.data.id}
-          trace={trace.data}
-          scores={trace.data.scores}
-          projectId={trace.data.projectId}
-          observations={trace.data.observations}
-        />
-      </div>
+      <Tabs
+        value={selectedTab}
+        onValueChange={setSelectedTab}
+        className="flex w-full justify-end border-b bg-background"
+      >
+        <TabsList className="bg-background py-0">
+          <TabsTrigger
+            value="details"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+          >
+            <Network className="mr-1 h-4 w-4"></Network>
+            Tree
+          </TabsTrigger>
+          <TabsTrigger
+            value="timeline"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+          >
+            <ListTree className="mr-1 h-4 w-4"></ListTree>
+            Timeline
+            <Badge className="pointer-events-none ml-2 px-1.5">Beta</Badge>
+          </TabsTrigger>
+        </TabsList>
+      </Tabs>
+      {selectedTab === "details" && (
+        <div className="mt-5 flex-1 overflow-hidden border-t pt-5">
+          <Trace
+            key={trace.data.id}
+            trace={trace.data}
+            scores={trace.data.scores}
+            projectId={trace.data.projectId}
+            observations={trace.data.observations}
+          />
+        </div>
+      )}
+      {selectedTab === "timeline" && (
+        <div className="mt-5 flex-1 flex-col space-y-5 overflow-hidden">
+          <Alert>
+            <Terminal className="h-4 w-4" />
+            <AlertTitle>New Trace Design Beta</AlertTitle>
+            <AlertDescription>
+              We value your feedback! Share your thoughts on{" "}
+              <a
+                href="https://github.com/orgs/langfuse/discussions/2195"
+                target="_blank"
+                className="underline"
+                rel="noopener noreferrer"
+              >
+                GitHub discussions
+              </a>
+              .
+            </AlertDescription>
+          </Alert>
+          <TraceTimelineView
+            key={trace.data.id}
+            trace={trace.data}
+            scores={trace.data.scores}
+            observations={trace.data.observations}
+            projectId={trace.data.projectId}
+          />
+        </div>
+      )}
     </div>
   );
 }

web/src/components/trace/lib/helpers.ts

@@ -0,0 +1,50 @@
+import { $Enums } from "@langfuse/shared";
+import { type NestedObservation } from "@/src/utils/types";
+import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+
+export type TreeItemType = $Enums.ObservationType | "TRACE";
+
+export const treeItemColors: Map<TreeItemType, string> = new Map([
+  [$Enums.ObservationType.SPAN, "bg-muted-blue"],
+  [$Enums.ObservationType.GENERATION, "bg-muted-orange"],
+  [$Enums.ObservationType.EVENT, "bg-muted-green"],
+  ["TRACE", "bg-input"],
+]);
+
+export function nestObservations(
+  list: ObservationReturnType[],
+): NestedObservation[] {
+  if (list.length === 0) return [];
+
+  // Step 1: Create a map where the keys are object IDs, and the values are
+  // the corresponding objects with an added 'children' property.
+  const map = new Map<string, NestedObservation>();
+  for (const obj of list) {
+    map.set(obj.id, { ...obj, children: [] });
+  }
+
+  // Step 2: Create another map for the roots of all trees.
+  const roots = new Map<string, NestedObservation>();
+
+  // Step 3: Populate the 'children' arrays and root map.
+  for (const obj of map.values()) {
+    if (obj.parentObservationId) {
+      const parent = map.get(obj.parentObservationId);
+      if (parent) {
+        parent.children.push(obj);
+      }
+    } else {
+      roots.set(obj.id, obj);
+    }
+  }
+
+  // Step 4: Sort children by start time for each parent
+  for (const obj of map.values()) {
+    obj.children.sort((a, b) => a.startTime.getTime() - b.startTime.getTime());
+  }
+
+  // TODO sum token amounts per level
+
+  // Step 5: Return the roots.
+  return Array.from(roots.values());
+}

web/src/components/ui/CodeJsonViewer.tsx

@@ -4,7 +4,7 @@ import { Check, ChevronsDownUp, ChevronsUpDown, Copy } from "lucide-react";
 import { cn } from "@/src/utils/tailwind";
 import { default as React18JsonView } from "react18-json-view";
 import "react18-json-view/src/dark.css";
-import { deepParseJson } from "@/src/utils/json";
+import { deepParseJson } from "@langfuse/shared";
 import { Skeleton } from "@/src/components/ui/skeleton";
 import { useTheme } from "next-themes";
 

web/src/components/ui/drawer.tsx

@@ -0,0 +1,129 @@
+"use client";
+
+import * as React from "react";
+import { Drawer as DrawerPrimitive } from "vaul";
+
+import { cn } from "@/src/utils/tailwind";
+import { useMediaQuery } from "react-responsive";
+
+// https://tailwindcss.com/docs/responsive-design
+const TAILWIND_MD_MEDIA_QUERY = 768;
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => {
+  const isMediumScreen = useMediaQuery({
+    query: `(min-width: ${TAILWIND_MD_MEDIA_QUERY}px)`,
+  });
+  const direction = isMediumScreen ? "right" : "bottom";
+
+  return (
+    <DrawerPrimitive.Root
+      shouldScaleBackground={shouldScaleBackground}
+      direction={direction}
+      {...props}
+    />
+  );
+};
+Drawer.displayName = "Drawer";
+
+const DrawerTrigger = DrawerPrimitive.Trigger;
+
+const DrawerPortal = DrawerPrimitive.Portal;
+
+const DrawerClose = DrawerPrimitive.Close;
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-primary/20", className)}
+    {...props}
+  />
+));
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName;
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background md:inset-x-auto md:inset-y-0 md:right-0 md:mt-0 md:h-full md:w-2/5 md:rounded-l-[10px] md:rounded-t-[0px] lg:w-1/3 xl:w-1/4 2xl:w-1/4",
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+));
+DrawerContent.displayName = "DrawerContent";
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+);
+DrawerHeader.displayName = "DrawerHeader";
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+);
+DrawerFooter.displayName = "DrawerFooter";
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName;
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+));
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName;
+
+export {
+  Drawer,
+  DrawerPortal,
+  DrawerOverlay,
+  DrawerTrigger,
+  DrawerClose,
+  DrawerContent,
+  DrawerHeader,
+  DrawerFooter,
+  DrawerTitle,
+  DrawerDescription,
+};

web/src/components/ui/popover.tsx

@@ -28,4 +28,6 @@ const PopoverContent = React.forwardRef<
 ));
 PopoverContent.displayName = PopoverPrimitive.Content.displayName;
 
-export { Popover, PopoverTrigger, PopoverContent };
+const PopoverClose = PopoverPrimitive.Close;
+
+export { Popover, PopoverTrigger, PopoverContent, PopoverClose };

web/src/components/ui/toggle-group.tsx

@@ -0,0 +1,60 @@
+"use client";
+
+import * as React from "react";
+import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group";
+import { type VariantProps } from "class-variance-authority";
+import { cn } from "@/src/utils/tailwind";
+import { toggleVariants } from "@/src/components/ui/toggle";
+
+const ToggleGroupContext = React.createContext<
+  VariantProps<typeof toggleVariants>
+>({
+  size: "default",
+  variant: "default",
+});
+
+const ToggleGroup = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Root> &
+    VariantProps<typeof toggleVariants>
+>(({ className, variant, size, children, ...props }, ref) => (
+  <ToggleGroupPrimitive.Root
+    ref={ref}
+    className={cn("flex items-center justify-center gap-1", className)}
+    {...props}
+  >
+    <ToggleGroupContext.Provider value={{ variant, size }}>
+      {children}
+    </ToggleGroupContext.Provider>
+  </ToggleGroupPrimitive.Root>
+));
+
+ToggleGroup.displayName = ToggleGroupPrimitive.Root.displayName;
+
+const ToggleGroupItem = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Item> &
+    VariantProps<typeof toggleVariants>
+>(({ className, children, variant, size, ...props }, ref) => {
+  const context = React.useContext(ToggleGroupContext);
+
+  return (
+    <ToggleGroupPrimitive.Item
+      ref={ref}
+      className={cn(
+        toggleVariants({
+          variant: context.variant || variant,
+          size: context.size || size,
+        }),
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </ToggleGroupPrimitive.Item>
+  );
+});
+
+ToggleGroupItem.displayName = ToggleGroupPrimitive.Item.displayName;
+
+export { ToggleGroup, ToggleGroupItem };

web/src/components/useSessionStorage.tsx

@@ -0,0 +1,58 @@
+import { useState, useEffect } from "react";
+
+/**
+ * useSessionStorage is a hook for managing data with the sessionStorage API.
+ *
+ * @param {string} sessionStorageKey - The key under which the value is stored in sessionStorage.
+ * @param {T} initialValue - The initial value of the data to be stored.
+ *
+ * Note: The object T should be able to be stringified, as it will be stored in sessionStorage as a string.
+ *
+ * @return An array with three elements:
+ *     value: Current value
+ *     setValue: Function to update the value
+ *     clearValue: Function to remove value from the session storage.
+ *                This function will also reset the value to initial value
+ *
+ * @template T - The type of the data to be stored in sessionStorage. It should be a type that can be stringified.
+ *
+ * @throws Will throw an error if the stringifying the value or accessing session storage fails.
+ */
+function useSessionStorage<T>(
+  sessionStorageKey: string,
+  initialValue: T,
+): [T, React.Dispatch<React.SetStateAction<T>>, () => void] {
+  const [value, setValue] = useState<T>(() => {
+    if (typeof window === "undefined") {
+      return initialValue;
+    }
+    try {
+      const storedValue = sessionStorage.getItem(sessionStorageKey);
+      return storedValue ? (JSON.parse(storedValue) as T) : initialValue;
+    } catch (error) {
+      console.error("Error reading from session storage", error);
+      return initialValue;
+    }
+  });
+
+  const clearValue = () => {
+    try {
+      sessionStorage.removeItem(sessionStorageKey);
+      setValue(initialValue);
+    } catch (error) {
+      console.error("Error clearing session storage", error);
+    }
+  };
+
+  useEffect(() => {
+    try {
+      sessionStorage.setItem(sessionStorageKey, JSON.stringify(value));
+    } catch (error) {
+      console.error("Error writing to session storage", error);
+    }
+  }, [sessionStorageKey, value]);
+
+  return [value, setValue, clearValue] as const;
+}
+
+export default useSessionStorage;

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.47.4";
+export const VERSION = "v2.53.0";

web/src/ee/features/evals/components/eval-config-detail.tsx

@@ -11,7 +11,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { useState } from "react";
-import { Trash2 } from "lucide-react";
+import { Trash } from "lucide-react";
 import {
   Tabs,
   TabsContent,
@@ -156,7 +156,7 @@ export function DeactivateConfig({
           disabled={!hasAccess || config?.status !== "ACTIVE"}
           loading={isLoading}
         >
-          <Trash2 className="h-5 w-5" />
+          <Trash className="h-5 w-5" />
         </Button>
       </PopoverTrigger>
       <PopoverContent>

web/src/ee/features/evals/components/template-form.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState, useMemo } from "react";
+import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import * as z from "zod";
 import { Input } from "@/src/components/ui/input";
@@ -18,19 +18,12 @@ import { zodResolver } from "@hookform/resolvers/zod";
 import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
 import router from "next/router";
 import { type EvalTemplate } from "@prisma/client";
+import { ModelParameters } from "@/src/components/ModelParameters";
 import {
-  ModelParameters,
-  type ModelParamsContext,
-} from "@/src/components/ModelParameters";
-import {
-  EvalModelNames,
   OutputSchema,
-  evalLLMModels,
   type UIModelParams,
-  ModelProvider,
-  type OpenAIModel,
-  type OpenAIModelParams,
   type ModelParams,
+  ZodModelConfig,
 } from "@langfuse/shared";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
 import {
@@ -43,6 +36,7 @@ import {
 import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
 
 export const EvalTemplateForm = (props: {
   projectId: string;
@@ -102,15 +96,6 @@ export const EvalTemplateForm = (props: {
                     score: currentTemplate?.outputScore?.trim() ?? "",
                     reasoning: currentTemplate?.outputReasoning?.trim() ?? "",
                   },
-                  model: "gpt-3.5-turbo",
-                  modelParams: {
-                    model: "gpt-3.5-turbo",
-                    provider: ModelProvider.OpenAI,
-                    temperature: 1,
-                    maxTemperature: 2,
-                    max_tokens: 256,
-                    top_p: 1,
-                  },
                 }
               : props.existingEvalTemplate
                 ? {
@@ -121,10 +106,13 @@ export const EvalTemplateForm = (props: {
                       score: string;
                       reasoning: string;
                     },
-                    model: props.existingEvalTemplate.model as OpenAIModel,
-                    modelParams: props.existingEvalTemplate
-                      .modelParams as OpenAIModelParams & {
-                      maxTemperature: number;
+                    selectedModel: {
+                      provider: props.existingEvalTemplate.provider,
+                      model: props.existingEvalTemplate.model,
+                      modelParams: props.existingEvalTemplate
+                        .modelParams as ModelParams & {
+                        maxTemperature: number;
+                      },
                     },
                   }
                 : undefined
@@ -135,6 +123,12 @@ export const EvalTemplateForm = (props: {
   );
 };
 
+const selectedModelSchema = z.object({
+  provider: z.string().min(1, "Select a provider"),
+  model: z.string().min(1, "Select a model"),
+  modelParams: ZodModelConfig,
+});
+
 const formSchema = z.object({
   name: z.string().min(1, "Enter a name"),
   prompt: z
@@ -167,9 +161,12 @@ export type EvalTemplateFormPreFill = {
     score: string;
     reasoning: string;
   };
-  model: OpenAIModel;
-  modelParams: OpenAIModelParams & {
-    maxTemperature: number;
+  selectedModel?: {
+    provider: string;
+    model: string;
+    modelParams: ModelParams & {
+      maxTemperature: number;
+    };
   };
 };
 
@@ -186,27 +183,41 @@ export const InnerEvalTemplateForm = (props: {
 }) => {
   const capture = usePostHogClientCapture();
   const [formError, setFormError] = useState<string | null>(null);
-  const preFilledModel = useMemo(
-    () => getModelParamsWithEnabledFlag(props.preFilledFormValues),
-    [props.preFilledFormValues],
-  );
 
   // updates the model params based on the pre-filled data
   // either form update or from langfuse-generated template
-  const [modelParams, setModelParams] = useState<UIModelParams>(preFilledModel);
-  const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
-    key,
-    value,
-  ) => {
-    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], value } }));
-  };
+  const {
+    modelParams,
+    setModelParams,
+    updateModelParamValue,
+    setModelParamEnabled,
+    availableModels,
+    availableProviders,
+  } = useModelParams({ evalModelsOnly: true });
 
-  const setModelParamEnabled: ModelParamsContext["setModelParamEnabled"] = (
-    key,
-    enabled,
-  ) => {
-    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], enabled } }));
-  };
+  useEffect(() => {
+    if (props.preFilledFormValues?.selectedModel) {
+      const { provider, model, modelParams } =
+        props.preFilledFormValues.selectedModel;
+
+      const modelConfig = Object.entries(modelParams).reduce(
+        (acc, [key, value]) => {
+          return {
+            ...acc,
+            [key]: { value, enabled: true },
+          };
+        },
+        {} as UIModelParams,
+      );
+
+      setModelParams((prev) => ({
+        ...prev,
+        ...modelConfig,
+        provider: { value: provider, enabled: true },
+        model: { value: model, enabled: true },
+      }));
+    }
+  }, [props.preFilledFormValues?.selectedModel, setModelParams]);
 
   // updates the form based on the pre-filled data
   // either form update or from langfuse-generated template
@@ -243,33 +254,8 @@ export const InnerEvalTemplateForm = (props: {
         outputScore: OutputSchema.parse(props.preFilledFormValues.outputSchema)
           .score,
       });
-
-      // state for the model params is outside of the form, hence needs to be handled individually
-      // also set the context for the playground
-      const model = EvalModelNames.parse(preFilledModel.model.value);
-      updateModelParamValue("model", model);
-      setModelParams((prev) => ({
-        ...prev,
-        ...preFilledModel,
-      }));
-
-      const modelProvider = evalLLMModels.find((m) => m.model.value === model)
-        ?.provider.value;
-
-      if (modelProvider) {
-        updateModelParamValue("provider", modelProvider); // updating the provider based on the model
-        updateModelParamValue(
-          "maxTemperature",
-          modelProvider === ModelProvider.OpenAI ? 2 : 1,
-        ); // setting the max value of the slider based on the provider
-      }
     }
-  }, [
-    props.preFilledFormValues,
-    preFilledModel,
-    form,
-    props.existingEvalTemplateName,
-  ]);
+  }, [props.preFilledFormValues, form, props.existingEvalTemplateName]);
 
   const extractedVariables = form.watch("prompt")
     ? extractVariables(form.watch("prompt")).filter(getIsCharOrUnderscore)
@@ -288,26 +274,31 @@ export const InnerEvalTemplateForm = (props: {
         : "eval_templates:new_form_submit",
     );
 
-    const model = EvalModelNames.safeParse(modelParams.model.value);
+    const evalTemplate = {
+      name: values.name,
+      projectId: props.projectId,
+      prompt: values.prompt,
+      provider: modelParams.provider.value,
+      model: modelParams.model.value,
+      modelParams: getFinalModelParams(modelParams),
+      vars: extractedVariables ?? [],
+      outputSchema: {
+        score: values.outputScore,
+        reasoning: values.outputReasoning,
+      },
+    };
 
-    if (!model.success) {
-      setFormError("Please select a model.");
+    const parsedModel = selectedModelSchema.safeParse(evalTemplate);
+
+    if (!parsedModel.success) {
+      setFormError(
+        `${parsedModel.error.errors[0].path}: ${parsedModel.error.errors[0].message}`,
+      );
       return;
     }
 
     createEvalTemplateMutation
-      .mutateAsync({
-        name: values.name,
-        projectId: props.projectId,
-        prompt: values.prompt,
-        model: model.data,
-        modelParams: getFinalModelParams(modelParams),
-        vars: extractedVariables ?? [],
-        outputSchema: {
-          score: values.outputScore,
-          reasoning: values.outputReasoning,
-        },
-      })
+      .mutateAsync(evalTemplate)
       .then((res) => {
         props.onFormSuccess?.();
         form.reset();
@@ -429,10 +420,11 @@ export const InnerEvalTemplateForm = (props: {
             <ModelParameters
               {...{
                 modelParams,
+                availableModels,
+                availableProviders,
                 updateModelParamValue: updateModelParamValue,
                 setModelParamEnabled,
               }}
-              availableModels={[...evalLLMModels]}
               formDisabled={!props.isEditing}
             />
           </div>
@@ -456,33 +448,3 @@ export const InnerEvalTemplateForm = (props: {
     </Form>
   );
 };
-
-function getModelParamsWithEnabledFlag(
-  evalPreFill?: EvalTemplateFormPreFill,
-): UIModelParams {
-  const defaultModelParams: ModelParams & { maxTemperature: number } = {
-    model: evalPreFill?.model ?? "gpt-3.5-turbo",
-    provider: ModelProvider.OpenAI,
-    max_tokens: 100,
-    maxTemperature:
-      evalPreFill?.modelParams?.provider === ModelProvider.OpenAI ? 2 : 1,
-    top_p: 1,
-    temperature: 1,
-  };
-
-  return Object.entries({
-    ...defaultModelParams,
-    ...evalPreFill?.modelParams,
-  }).reduce(
-    (params, [key, value]) => ({
-      ...params,
-      [key]: {
-        enabled: Boolean(
-          !evalPreFill || evalPreFill.modelParams[key as keyof UIModelParams],
-        ),
-        value,
-      },
-    }),
-    {} as UIModelParams,
-  );
-}

web/src/ee/features/evals/server/router.ts

@@ -8,7 +8,6 @@ import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { DEFAULT_TRACE_JOB_DELAY, EvalTargetObject } from "@langfuse/shared";
 import {
-  EvalModelNames,
   ZodModelConfig,
   singleFilter,
   variableMapping,
@@ -19,7 +18,8 @@ export const CreateEvalTemplate = z.object({
   name: z.string().min(1),
   projectId: z.string(),
   prompt: z.string(),
-  model: EvalModelNames,
+  provider: z.string(),
+  model: z.string(),
   modelParams: ZodModelConfig,
   vars: z.array(z.string()),
   outputSchema: z.object({
@@ -332,6 +332,7 @@ export const evalRouter = createTRPCRouter({
           modelParams: input.modelParams,
           vars: input.vars,
           outputSchema: input.outputSchema,
+          provider: input.provider,
         },
       });
 

web/src/ee/features/playground/page/components/JumpToPlaygroundButton.tsx

@@ -1,24 +1,24 @@
 import { Terminal } from "lucide-react";
 import Link from "next/link";
+import { useEffect, useState } from "react";
+import { z } from "zod";
 
+import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import { Button } from "@/src/components/ui/button";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { type PlaygroundCache } from "@/src/ee/features/playground/page/types";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import {
   ChatMessageRole,
-  supportedModels as playgroundSupportedModels,
   type Observation,
   type Prompt,
+  supportedModels as playgroundSupportedModels,
   type UIModelParams,
   ZodModelConfig,
 } from "@langfuse/shared";
-import { useEffect, useState } from "react";
-import { PromptType } from "@/src/features/prompts/server/utils/validation";
-import { ChatMessageListSchema } from "@/src/features/prompts/components/NewPromptForm/validation";
-import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 type JumpToPlaygroundButtonProps = (
   | {
@@ -42,6 +42,7 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
   const projectId = useProjectIdFromURL();
   const { setPlaygroundCache } = usePlaygroundCache();
   const [capturedState, setCapturedState] = useState<PlaygroundCache>(null);
+  const isEeEnabled = useIsEeEnabled();
 
   useEffect(() => {
     if (props.source === "prompt") {
@@ -56,7 +57,7 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
     setPlaygroundCache(capturedState);
   };
 
-  if (!getIsCloudEnvironment()) return null;
+  if (!isEeEnabled) return null;
 
   return (
     <Button
@@ -76,9 +77,19 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
   );
 };
 
+const ParsedChatMessageListSchema = z.array(
+  z.object({
+    role: z.nativeEnum(ChatMessageRole),
+    content: z.union([
+      z.string(),
+      z.any().transform((v) => JSON.stringify(v, null, 2)),
+    ]),
+  }),
+);
+
 const parsePrompt = (prompt: Prompt): PlaygroundCache => {
   if (prompt.type === PromptType.Chat) {
-    const parsedMessages = ChatMessageListSchema.safeParse(prompt.prompt);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(prompt.prompt);
 
     return parsedMessages.success ? { messages: parsedMessages.data } : null;
   } else {
@@ -109,14 +120,16 @@ const parseGeneration = (generation: Observation): PlaygroundCache => {
   }
 
   if (typeof input === "object") {
-    const parsedMessages = ChatMessageListSchema.safeParse(input);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(input);
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };
   }
 
   if (typeof input === "object" && "messages" in input) {
-    const parsedMessages = ChatMessageListSchema.safeParse(input["messages"]);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(
+      input["messages"],
+    );
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };

web/src/ee/features/playground/page/components/PromptVariableComponent.tsx

@@ -1,5 +1,5 @@
 import { type ChangeEvent } from "react";
-import { CheckCircle2, Circle, Trash2Icon } from "lucide-react";
+import { CheckCircle2, Circle, TrashIcon } from "lucide-react";
 import { Button } from "@/src/components/ui/button";
 import { Textarea } from "@/src/components/ui/textarea";
 import { type PromptVariable } from "@langfuse/shared";
@@ -46,7 +46,7 @@ export const PromptVariableComponent: React.FC<{
           onClick={handleDeleteVariable}
           className="p-0"
         >
-          {!isUsed && <Trash2Icon size={16} />}
+          {!isUsed && <TrashIcon size={16} />}
         </Button>
       </div>
       <Textarea

web/src/ee/features/playground/page/components/ResetPlaygroundButton.tsx

@@ -3,9 +3,10 @@ import { useRouter } from "next/router";
 
 import { Button } from "@/src/components/ui/button";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 export const ResetPlaygroundButton: React.FC = () => {
+  const isEeEnabled = useIsEeEnabled();
   const router = useRouter();
   const { setPlaygroundCache } = usePlaygroundCache();
 
@@ -15,7 +16,9 @@ export const ResetPlaygroundButton: React.FC = () => {
     router.reload();
   };
 
-  return getIsCloudEnvironment() ? (
+  if (!isEeEnabled) return null;
+
+  return (
     <Button
       variant={"outline"}
       title="Reset playground state"
@@ -24,5 +27,5 @@ export const ResetPlaygroundButton: React.FC = () => {
       <ListRestartIcon className="mr-1 h-5 w-5" />
       <span>Reset playground</span>
     </Button>
-  ) : null;
+  );
 };

web/src/ee/features/playground/page/components/SaveToPromptButton.tsx

@@ -19,14 +19,15 @@ import {
 } from "@/src/components/ui/popover";
 import { usePlaygroundContext } from "@/src/ee/features/playground/page/context";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
 import { cn } from "@/src/utils/tailwind";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 export const SaveToPromptButton: React.FC = () => {
+  const isEeEnabled = useIsEeEnabled();
   const [open, setOpen] = useState(false);
   const [selectedPromptId, setSelectedPromptId] = useState("");
   const { modelParams, messages, output, promptVariables } =
@@ -83,7 +84,7 @@ export const SaveToPromptButton: React.FC = () => {
     );
   };
 
-  if (!getIsCloudEnvironment()) return null;
+  if (!isEeEnabled) return null;
 
   return (
     <Popover>

web/src/ee/features/playground/page/context/index.tsx

@@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid";
 
 import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import useCommandEnter from "@/src/ee/features/playground/page/hooks/useCommandEnter";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -19,7 +20,6 @@ import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
   type ChatMessageWithId,
-  ModelProvider,
   type PromptVariable,
   type UIModelParams,
 } from "@langfuse/shared";
@@ -68,9 +68,14 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     createEmptyMessage(ChatMessageRole.System),
     createEmptyMessage(ChatMessageRole.User),
   ]);
-  const [modelParams, setModelParams] = useState<UIModelParams>(
-    getDefaultModelParams(ModelProvider.OpenAI),
-  );
+  const {
+    modelParams,
+    setModelParams,
+    availableProviders,
+    availableModels,
+    updateModelParamValue,
+    setModelParamEnabled,
+  } = useModelParams();
 
   // Load state from cache
   useEffect(() => {
@@ -97,11 +102,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     if (cachedPromptVariables) {
       setPromptVariables(cachedPromptVariables);
     }
-  }, [playgroundCache]);
-
-  useEffect(() => {
-    setModelParams(getDefaultModelParams(modelParams.provider.value));
-  }, [modelParams.provider.value]);
+  }, [playgroundCache, setModelParams]);
 
   const updatePromptVariables = useCallback(() => {
     const messageContents = messages.map((m) => m.content).join("\n");
@@ -169,6 +170,10 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
           finalMessages.map((m) => m.content).join("\n"),
         );
 
+        if (!modelParams.provider.value || !modelParams.model.value) {
+          throw new Error("Please select a model");
+        }
+
         if (leftOverVariables.length > 0) {
           throw Error("Error replacing variables. Please check your inputs.");
         }
@@ -210,24 +215,6 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
 
   useCommandEnter(!isStreaming, handleSubmit);
 
-  const updateModelParamValue: PlaygroundContextType["updateModelParamValue"] =
-    (key, value) => {
-      setModelParams((prev) => ({
-        ...prev,
-        [key]: { ...prev[key], value },
-      }));
-    };
-
-  const setModelParamEnabled: PlaygroundContextType["setModelParamEnabled"] = (
-    key,
-    enabled,
-  ) => {
-    setModelParams((prev) => ({
-      ...prev,
-      [key]: { ...prev[key], enabled },
-    }));
-  };
-
   const updatePromptVariableValue = (variable: string, value: string) => {
     setPromptVariables((prev) =>
       prev.map((v) => (v.name === variable ? { ...v, value } : v)),
@@ -258,6 +245,9 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
         outputJson,
         handleSubmit,
         isStreaming,
+
+        availableProviders,
+        availableModels,
       }}
     >
       {children}
@@ -342,38 +332,6 @@ function getFinalMessages(
   return finalMessages;
 }
 
-function getDefaultModelParams(provider: ModelProvider): UIModelParams {
-  switch (provider) {
-    // Docs: https://platform.openai.com/docs/api-reference/chat/create
-    case ModelProvider.OpenAI:
-      return {
-        provider: {
-          value: provider,
-          enabled: true,
-        },
-        model: { value: "gpt-3.5-turbo", enabled: true },
-        temperature: { value: 1, enabled: true },
-        maxTemperature: { value: 2, enabled: true },
-        max_tokens: { value: 256, enabled: true },
-        top_p: { value: 1, enabled: true },
-      };
-
-    // Docs: https://docs.anthropic.com/claude/reference/messages_post
-    case ModelProvider.Anthropic:
-      return {
-        provider: {
-          value: provider,
-          enabled: true,
-        },
-        model: { value: "claude-3-opus-20240229", enabled: true },
-        temperature: { value: 0, enabled: true },
-        maxTemperature: { value: 1, enabled: true },
-        max_tokens: { value: 256, enabled: true },
-        top_p: { value: 1, enabled: true },
-      };
-  }
-}
-
 function getOutputJson(
   output: string,
   messages: ChatMessageWithId[],

web/src/ee/features/playground/page/hooks/useModelParams.ts

@@ -0,0 +1,170 @@
+import { useEffect, useMemo, useState } from "react";
+
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { api } from "@/src/utils/api";
+import {
+  LLMAdapter,
+  evalModelsByAdapter,
+  supportedModels,
+  type UIModelParams,
+} from "@langfuse/shared";
+import { type ModelParamsContext } from "@/src/components/ModelParameters";
+
+export const useModelParams = ({
+  evalModelsOnly,
+}: {
+  evalModelsOnly?: boolean;
+} = {}) => {
+  const [modelParams, setModelParams] = useState<UIModelParams>({
+    ...getDefaultAdapterParams(LLMAdapter.OpenAI),
+    provider: { value: "", enabled: true },
+    model: { value: "", enabled: true },
+  });
+
+  // Set initial model params
+  const projectId = useProjectIdFromURL();
+  const availableLLMApiKeys = api.llmApiKey.all.useQuery(
+    {
+      projectId: projectId as string,
+    },
+    { enabled: Boolean(projectId) },
+  );
+
+  const availableProviders = useMemo(() => {
+    return availableLLMApiKeys.data?.data.map((key) => key.provider) ?? [];
+  }, [availableLLMApiKeys]);
+
+  const selectedProviderApiKey = availableLLMApiKeys.data?.data.find(
+    (key) => key.provider === modelParams.provider.value,
+  );
+
+  const availableModels = useMemo(
+    () =>
+      !selectedProviderApiKey
+        ? []
+        : evalModelsOnly
+          ? [
+              ...evalModelsByAdapter[selectedProviderApiKey.adapter],
+              ...selectedProviderApiKey.customModels,
+            ]
+          : selectedProviderApiKey.withDefaultModels
+            ? [
+                ...selectedProviderApiKey.customModels,
+                ...supportedModels[selectedProviderApiKey.adapter],
+              ]
+            : selectedProviderApiKey.customModels,
+    [selectedProviderApiKey, evalModelsOnly],
+  );
+
+  const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
+    key,
+    value,
+  ) => {
+    setModelParams((prev) => ({
+      ...prev,
+      [key]: { ...prev[key], value },
+    }));
+  };
+
+  const setModelParamEnabled: ModelParamsContext["setModelParamEnabled"] = (
+    key,
+    enabled,
+  ) => {
+    setModelParams((prev) => ({
+      ...prev,
+      [key]: { ...prev[key], enabled },
+    }));
+  };
+
+  // Set default provider and model
+  useEffect(() => {
+    if (availableProviders.length > 0 && !modelParams.provider.value) {
+      updateModelParamValue("provider", availableProviders[0]);
+    }
+  }, [availableProviders, modelParams.provider.value]);
+
+  useEffect(() => {
+    if (
+      (availableModels.length > 0 && !modelParams.model.value) ||
+      !availableModels.includes(modelParams.model.value)
+    ) {
+      updateModelParamValue("model", availableModels[0]);
+    }
+  }, [availableModels, modelParams.model.value]);
+
+  // Update adapter and max temperature when provider changes
+  useEffect(() => {
+    if (selectedProviderApiKey?.adapter) {
+      setModelParams((prev) => ({
+        ...prev,
+        adapter: {
+          value: selectedProviderApiKey.adapter,
+          enabled: true,
+        },
+        maxTemperature: getDefaultAdapterParams(selectedProviderApiKey.adapter)
+          .maxTemperature,
+        temperature: {
+          value: Math.min(
+            prev.temperature.value,
+            getDefaultAdapterParams(selectedProviderApiKey.adapter)
+              .maxTemperature.value,
+          ),
+          enabled: true,
+        },
+      }));
+    }
+  }, [selectedProviderApiKey?.adapter]);
+
+  return {
+    modelParams,
+    setModelParams,
+    availableProviders,
+    availableModels,
+    updateModelParamValue,
+    setModelParamEnabled,
+  };
+};
+
+function getDefaultAdapterParams(
+  adapter: LLMAdapter,
+): Omit<UIModelParams, "provider" | "model"> {
+  switch (adapter) {
+    // Docs: https://platform.openai.com/docs/api-reference/chat/create
+    case LLMAdapter.OpenAI:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 2, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+
+    case LLMAdapter.Azure:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 2, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+
+    // Docs: https://docs.anthropic.com/claude/reference/messages_post
+    case LLMAdapter.Anthropic:
+      return {
+        adapter: {
+          value: adapter,
+          enabled: true,
+        },
+        temperature: { value: 0, enabled: true },
+        maxTemperature: { value: 1, enabled: true },
+        max_tokens: { value: 256, enabled: true },
+        top_p: { value: 1, enabled: true },
+      };
+  }
+}

web/src/ee/features/playground/page/hooks/usePlaygroundCache.ts

@@ -1,13 +1,13 @@
 import { useEffect, useState } from "react";
 
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 import { type PlaygroundCache } from "../types";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 const playgroundCacheKey = "playgroundCache";
 
 export default function usePlaygroundCache() {
   const [cache, setCache] = useState<PlaygroundCache>(null);
-
+  const isEeEnabled = useIsEeEnabled();
   const setPlaygroundCache = (cache: PlaygroundCache) => {
     sessionStorage.setItem(playgroundCacheKey, JSON.stringify(cache));
   };
@@ -24,7 +24,7 @@ export default function usePlaygroundCache() {
   }, []);
 
   return {
-    playgroundCache: getIsCloudEnvironment() ? cache : null,
-    setPlaygroundCache: getIsCloudEnvironment() ? setPlaygroundCache : () => {},
+    playgroundCache: isEeEnabled ? cache : null,
+    setPlaygroundCache: isEeEnabled ? setPlaygroundCache : () => {},
   };
 }

web/src/ee/features/playground/page/index.tsx

@@ -3,10 +3,12 @@ import { ResetPlaygroundButton } from "@/src/ee/features/playground/page/compone
 import { SaveToPromptButton } from "@/src/ee/features/playground/page/components/SaveToPromptButton";
 import { PlaygroundProvider } from "@/src/ee/features/playground/page/context";
 import Playground from "@/src/ee/features/playground/page/playground";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 export default function PlaygroundPage() {
-  return getIsCloudEnvironment() ? (
+  const isEeAvailable = useIsEeEnabled();
+  if (!isEeAvailable) return null;
+  return (
     <PlaygroundProvider>
       <div className="flex h-[95vh] flex-col">
         <Header
@@ -28,5 +30,5 @@ export default function PlaygroundPage() {
         </div>
       </div>
     </PlaygroundProvider>
-  ) : null;
+  );
 }

web/src/ee/features/playground/server/authorizeRequest.ts

@@ -1,9 +1,9 @@
 import { getServerSession } from "next-auth";
 
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 import { getAuthOptions } from "@/src/server/auth";
 import { isProjectMemberOrAdmin } from "@/src/server/utils/checkProjectMembershipOrAdmin";
 import { ApiError, ForbiddenError, UnauthorizedError } from "@langfuse/shared";
+import { isEeEnabled } from "@/src/ee/utils/isEeEnabled";
 
 export type AuthorizeRequestResult = {
   userId: string;
@@ -12,8 +12,10 @@ export type AuthorizeRequestResult = {
 export const authorizeRequestOrThrow = async (
   projectId: string,
 ): Promise<AuthorizeRequestResult> => {
-  if (!getIsCloudEnvironment())
-    throw new ApiError("This endpoint is available in Langfuse cloud only.");
+  if (!isEeEnabled)
+    throw new ApiError(
+      "LLM Playground is not yet available in the v2 open-source version.",
+    );
 
   const authOptions = await getAuthOptions();
   const session = await getServerSession(authOptions);

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -39,6 +39,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
       streaming: true,
       callbacks: [new PosthogCallbackHandler("playground", body, userId)],
       apiKey: decrypt(LLMApiKey.secretKey),
+      baseURL: LLMApiKey.baseURL || undefined,
     });
 
     return new StreamingTextResponse(stream);

web/src/ee/features/playground/server/validateChatCompletionBody.ts

@@ -1,31 +1,14 @@
 import { z } from "zod";
-import {
-  ChatMessageRole,
-  ModelProvider,
-  anthropicModels,
-  openAIModels,
-} from "@langfuse/shared";
+import { ChatMessageRole, LLMAdapter } from "@langfuse/shared";
 
-const OpenAIModelSchema = z.enum([...openAIModels]);
-const AnthropicModelSchema = z.enum([...anthropicModels]);
-const OpenAIModelParamsSchema = z.object({
-  provider: z.literal(ModelProvider.OpenAI),
-  model: OpenAIModelSchema,
-  max_tokens: z.number().optional(),
-  temperature: z.number().optional(),
-  top_p: z.number().optional(),
-});
-const AnthropicModelParamsSchema = z.object({
-  provider: z.literal(ModelProvider.Anthropic),
-  model: AnthropicModelSchema,
+const ModelParamsSchema = z.object({
+  provider: z.string(),
+  adapter: z.nativeEnum(LLMAdapter),
+  model: z.string(),
   temperature: z.number().optional(),
   max_tokens: z.number().optional(),
   top_p: z.number().optional(),
 });
-const ModelParamsSchema = z.union([
-  OpenAIModelParamsSchema,
-  AnthropicModelParamsSchema,
-]);
 const MessageSchema = z.object({
   role: z.nativeEnum(ChatMessageRole),
   content: z.string(),

web/src/ee/utils/getIsCloudEnvironment.ts

@@ -1,4 +0,0 @@
-import { env } from "@/src/env.mjs";
-
-export const getIsCloudEnvironment = () =>
-  Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);