chore: release v2.47.3

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

.env.dev.example

@@ -47,5 +47,5 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
-# openssl rand -base64 32 used only here
+# openssl rand -hex 32 used only here
 ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.prod.example

@@ -58,6 +58,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -96,7 +101,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
-# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
 
 # NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
 # NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"

.github/workflows/porter_stack_worker.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

CONTRIBUTING.md

@@ -290,6 +290,74 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Theming
+
+At Langfuse, we utilize CSS variables to manage our theme settings across the platform.
+
+Our approach leverages separate CSS variables for backgrounds (--background) and foregrounds (--foreground), fully adhering to the [shadcn/ui](https://ui.shadcn.com/docs/theming) color conventions. The background suffix can be omitted if the variable is used for the background color of the component. We recommend using HSL values for these colors to enhance consistency and customization. There is no need to manually handle dark mode styling with "dark:" prefixes, as next-themes automatically manages the theme switching.
+
+Given the following CSS variables:
+
+```
+--primary: 222.2 47.4% 11.2%; // e.g. background-color
+--primary-foreground: 210 40% 98%; // e.g. text-color
+```
+
+The background color of the following component will be `hsl(var(--primary))` and the foreground color will be `hsl(var(--primary-foreground))`.
+
+```
+<div class="bg-primary text-primary-foreground">Hello</div>
+```
+
+### Color Variables
+
+| Variable                 | Description                                                        | Examples                         |
+| ------------------------ | ------------------------------------------------------------------ | -------------------------------- |
+| --background             | Background color                                                   | Default background color of body |
+| --foreground             | Foreground color                                                   | Default text color of body       |
+| --muted                  | Muted background color                                             | TabsList, Skeleton and Switch    |
+| --muted-foreground       | Muted foreground color                                             |                                  |
+| --popover                | Popover background color                                           | DropdownMenu, HoverCard, Popover |
+| --popover-foreground     | Popover foreground color                                           |                                  |
+| --card                   | Card background color                                              | Card                             |
+| --card-foreground        | Card foreground color                                              |                                  |
+| --border                 | Border color                                                       | Default border color             |
+| --input                  | Input field border color                                           | Input, Select, Textarea          |
+| --primary                | Primary button background colors                                   | Button variant="primary"         |
+| --primary-foreground     | Primary button foreground color                                    |                                  |
+| --secondary              | Secondary button background color                                  | Button variant="secondary"       |
+| --secondary-foreground   | Secondary button foreground color                                  |                                  |
+| --accent                 | Used for accents such as hover effects                             | DropdownMenuItem, SelectItem     |
+| --accent-foreground      | Used for texts on hover effects                                    | DropdownMenuItem, SelectItem     |
+| --destructive            | Destructive action color for background                            | Button variant="destructive"     |
+| --destructive-foreground | Destructive action color for text                                  |                                  |
+| --ring                   | Focus ring color                                                   | MultiSelect                      |
+| --primary-accent         | Primary accent color used for branding                             | Layout                           |
+| --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
+| --muted-green            | Muted green for Event label                                        | ObservationTree                  |
+| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
+| --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
+| --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
+| --accent-dark-green      | Dark green accent for border of output and assistant messages      | CodeJsonViewer and IOPReview     |
+| --light-red              | Light red for error background                                     | level-color and StatusBadge      |
+| --dark-red               | Dark red for error text and error badge dot color                  | level-color and ErrorPage        |
+| --light-yellow           | Light yellow for warning background                                | LevelColor                       |
+| --dark-yellow            | Dark yellow for warning text                                       | LevelColor                       |
+| --light-green            | Light green for success status badge background                    | StatusBadge                      |
+| --dark-green             | Dark green for success status badge text and dot                   | StatusBadge                      |
+| --light-blue             | Light blue for background of Staging label                         | LangfuseLogo                     |
+| --dark-blue              | Dark blue for text and border of Staging label                     | LangfuseLogo                     |
+| --accent-light-blue      | Light blue accent for table link hover effect                      | TableLink                        |
+| --accent-dark-blue       | Dark blue accent for table link text                               | TableLink                        |
+
+### Adding New Colors
+
+1. Global Definitions: Add new CSS variable definitions in the global.css file.
+2. Tailwind Configuration: Reflect these new colors in the tailwind.config.js to maintain alignment with Tailwind's utility classes.
+
+By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
+
 ## Using secrets stored in 1Password
 
 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.

README.md

@@ -106,22 +106,26 @@ See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Lan
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                 |
-| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
+| Integration                                              | Supports                   | Description                                                                                                                                      |
+| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
 
-External projects/packages that integrate with Langfuse:
+Packages that integrate with Langfuse:
 
-| Name                                                            | Description                                                                                                                                      |
-| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
-| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
+| Name                                                       | Description                                                                                                             |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
+| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
+| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
+| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
 
 ## Questions and feedback
 

careers

@@ -0,0 +1,14 @@
+_We are Hiring_
+Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+
+_Open Roles_
+
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
+_More Info_
+
+- https://langfuse.com/careers
+- https://langfuse.com/docs
+- https://langfuse.com/changelog

docker-compose.yml

@@ -4,7 +4,8 @@ services:
   langfuse-server:
     image: ghcr.io/langfuse/langfuse:latest
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     ports:
       - "3000:3000"
     environment:
@@ -18,6 +19,11 @@ services:
   db:
     image: postgres
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres

ee/src/sso/index.ts

@@ -2,10 +2,11 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
-import { prisma } from "@langfuse/shared/src/db";
+import { type SsoConfig, prisma } from "@langfuse/shared/src/db";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
@@ -25,14 +26,36 @@ let cachedSsoConfigs: {
 async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
   if (!isEeAvailable) return [];
   const CACHE_TTL = 60 * 1000; // 1 minute
+  const DB_MAX_WAIT = 2000; // 2 seconds
+  const DB_TIMEOUT = 3000; // 3 seconds
 
   // Set/refresh the cache if it's empty or expired
   if (
     cachedSsoConfigs === null ||
     Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
   ) {
+    // findMany with custom timeout via $transaction
+    let dbConfigs: SsoConfig[] = [];
+    try {
+      dbConfigs = await prisma.$transaction(
+        async (prisma) => prisma.ssoConfig.findMany(),
+        {
+          maxWait: DB_MAX_WAIT,
+          timeout: DB_TIMEOUT,
+        }
+      );
+    } catch (e) {
+      // cache empty array to prevent repeated DB calls on error
+      cachedSsoConfigs = {
+        data: [],
+        timestamp: Date.now(),
+      };
+
+      // caught and logged in the caller
+      throw e;
+    }
+
     // transform into zod object
-    const dbConfigs = await prisma.ssoConfig.findMany();
     const parsedSsoConfigs = dbConfigs
       .map((v) => {
         try {
@@ -144,6 +167,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,24 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +89,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/commons.yml

@@ -209,8 +209,8 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
 
 errors:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.43.2",
+  "version": "2.47.3",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -28,7 +28,7 @@
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
-    "dotenv-cli": "^7.4.1",
+    "dotenv-cli": "^7.4.2",
     "prettier": "^3.2.5",
     "release-it": "^17.2.1",
     "turbo": "^1.13.3"

packages/shared/package.json

@@ -44,9 +44,9 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.16",
+    "@langchain/anthropic": "^0.1.21",
     "@langchain/core": "^0.1.61",
-    "@langchain/openai": "^0.0.28",
+    "@langchain/openai": "^0.0.33",
     "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
@@ -59,9 +59,9 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.5",
+    "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.7.0",
     "eslint": "^8.57.0",

packages/shared/prisma/generated/types.ts

@@ -25,11 +25,16 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC"
+} as const;
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const PricingUnit = {
     PER_1000_TOKENS: "PER_1000_TOKENS",
     PER_1000_CHARS: "PER_1000_CHARS"
@@ -246,6 +251,7 @@ export type Observation = {
     status_message: string | null;
     version: string | null;
     created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
     modelParameters: unknown | null;
@@ -293,6 +299,7 @@ export type ObservationView = {
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
     latency: number | null;
+    time_to_first_token: number | null;
 };
 export type PosthogIntegration = {
     project_id: string;
@@ -346,9 +353,28 @@ export type Score = {
     name: string;
     value: number;
     source: ScoreSource;
+    author_user_id: string | null;
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -448,6 +474,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240513082205_observations_view_add_time_to_first_token/migration.sql

@@ -0,0 +1,61 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240522081254_scores_add_author_user_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "author_user_id" TEXT;

packages/shared/prisma/migrations/20240522095738_scores_add_author_user_id_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_author_user_id_idx" ON "scores"("author_user_id");

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/migrations/20240528214726_add_cursor_new_columns_observations/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214727_add_cursor_new_columns_scores/migration.sql

@@ -0,0 +1,4 @@
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214728_add_cursor_index_01/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "audit_logs_updated_at_idx" ON "audit_logs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_02/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_created_at_idx" ON "dataset_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_03/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_updated_at_idx" ON "dataset_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_04/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_created_at_idx" ON "dataset_run_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_05/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_updated_at_idx" ON "dataset_run_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_06/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_created_at_idx" ON "dataset_runs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_07/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_updated_at_idx" ON "dataset_runs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_08/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_created_at_idx" ON "datasets"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_09/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_updated_at_idx" ON "datasets"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_10/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_updated_at_idx" ON "job_executions"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_11/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_updated_at_idx" ON "observations"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_12/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_created_at_idx" ON "prompts"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_13/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_updated_at_idx" ON "prompts"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_14/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_created_at_idx" ON "score_configs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_15/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_updated_at_idx" ON "score_configs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_16/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_created_at_idx" ON "scores"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_17/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_updated_at_idx" ON "scores"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_18/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "trace_sessions_updated_at_idx" ON "trace_sessions"("updated_at");

packages/shared/prisma/migrations/20240530212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/schema.prisma

@@ -114,6 +114,7 @@ model Project {
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
     Score              Score[]
+    scoreConfig        ScoreConfig[]
 
     @@map("projects")
 }
@@ -205,6 +206,7 @@ model TraceSession {
     @@id([id, projectId])
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("trace_sessions")
 }
 
@@ -292,6 +294,7 @@ model Observation {
     statusMessage       String?          @map("status_message")
     version             String?
     createdAt           DateTime         @default(now()) @map("created_at")
+    updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -323,6 +326,7 @@ model Observation {
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
+    @@index(updatedAt)
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
@@ -373,6 +377,7 @@ view ObservationView {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
     latency              Float?   @map("latency")
+    timeToFirstToken     Float?   @map("time_to_first_token")
 
     @@map("observations_view")
 }
@@ -398,27 +403,68 @@ model Score {
     name          String
     value         Float
     source        ScoreSource
+    authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    createdAt     DateTime       @default(now()) @map("created_at")
+    updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
     JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
 
     @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
+    @@index([authorUserId])
+    @@index([configId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("scores")
 }
 
 enum ScoreSource {
+    ANNOTATION
     API
-    REVIEW
     EVAL
 }
 
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@index([createdAt])
+    @@index([updatedAt])
+    @@map("score_configs")
+}
+
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+}
+
 enum PricingUnit {
     PER_1000_TOKENS
     PER_1000_CHARS
@@ -465,6 +511,8 @@ model Dataset {
 
     @@unique([projectId, name])
     @@index([projectId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("datasets")
 }
 
@@ -486,6 +534,8 @@ model DatasetItem {
 
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_items")
 }
 
@@ -507,6 +557,8 @@ model DatasetRuns {
 
     @@unique([datasetId, name])
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_runs")
 }
 
@@ -527,6 +579,8 @@ model DatasetRunItems {
     @@index([datasetItemId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([traceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_run_items")
 }
 
@@ -569,6 +623,8 @@ model Prompt {
     @@index([projectId, name, version])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("prompts")
 }
@@ -616,6 +672,7 @@ model AuditLog {
 
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("audit_logs")
 }
 
@@ -709,6 +766,11 @@ model JobExecution {
     @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("job_executions")
 }
 

packages/shared/prisma/seed.ts

@@ -12,6 +12,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -165,6 +166,24 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
     // add eval objects
     const evalTemplate = await prisma.evalTemplate.upsert({
       where: {
@@ -500,8 +519,9 @@ function createObjects(
               name: "manual-score",
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
               projectId,
+              authorUserId: `user-${i}`,
             },
           ]
         : []),
@@ -528,9 +548,9 @@ function createObjects(
       const spanTsStart = new Date(
         traceTs.getTime() + Math.floor(Math.random() * 30)
       );
-      // random duration of upto 30ms
+      // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 30)
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
       );
 
       const span = {
@@ -574,6 +594,13 @@ function createObjects(
                 (spanTsEnd.getTime() - generationTsStart.getTime())
             )
         );
+        // somewhere in the middle
+        const generationTsCompletionStart = new Date(
+          generationTsStart.getTime() +
+            Math.floor(
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
+            )
+        );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
         const completionTokens = Math.floor(Math.random() * 500) + 100;
@@ -602,6 +629,8 @@ function createObjects(
           id: `generation-${v4()}`,
           startTime: generationTsStart,
           endTime: generationTsEnd,
+          completionStartTime:
+            Math.random() > 0.5 ? generationTsCompletionStart : undefined,
           name: `generation-${i}-${j}-${k}`,
           projectId: trace.projectId,
           promptId: promptId,

packages/shared/src/server/llm/types.ts

@@ -34,9 +34,9 @@ export type UIModelParams = RecordWithEnabledFlag<
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
 
 export const ZodModelConfig = z.object({
-  max_tokens: z.number().optional(),
-  temperature: z.number().optional(),
-  top_p: z.number().optional(),
+  max_tokens: z.coerce.number().optional(),
+  temperature: z.coerce.number().optional(),
+  top_p: z.coerce.number().optional(),
 });
 
 // OpenAI

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.43.2",
+  "version": "2.47.3",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -25,12 +25,14 @@
     "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
-    "@headlessui/react": "^1.7.19",
+    "@codemirror/lint": "^6.8.0",
+    "@headlessui/react": "1.7.18",
+    "@headlessui/tailwindcss": "0.2.0",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.16",
+    "@langchain/anthropic": "^0.1.21",
     "@langchain/core": "^0.1.61",
-    "@langchain/openai": "^0.0.28",
+    "@langchain/openai": "^0.0.33",
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
@@ -55,26 +57,27 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.17",
-    "@react-email/render": "^0.0.13",
+    "@react-email/components": "^0.0.18",
+    "@react-email/render": "^0.0.14",
+    "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
     "@sentry/nextjs": "^7.113.0",
     "@sentry/node": "^7.113.0",
     "@sentry/profiling-node": "^7.113.0",
     "@sentry/types": "^7.113.0",
-    "@t3-oss/env-nextjs": "^0.8.0",
+    "@t3-oss/env-nextjs": "^0.10.1",
     "@tailwindcss/container-queries": "^0.1.1",
-    "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
     "@tanstack/react-virtual": "^3.5.0",
-    "@tremor/react": "3.11.1",
+    "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
@@ -85,7 +88,7 @@
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.2",
+    "dompurify": "^3.1.4",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
@@ -95,6 +98,7 @@
     "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
+    "next-themes": "^0.3.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
@@ -102,8 +106,8 @@
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.3",
-    "react-icons": "^5.0.1",
+    "react-hook-form": "^7.51.5",
+    "react-icons": "^5.2.1",
     "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",
@@ -118,23 +122,24 @@
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
-    "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^15.0.6",
+    "@tailwindcss/forms": "^0.5.7",
+    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "20.10.5",
-    "@types/nodemailer": "^6.4.14",
+    "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.7.0",
     "autoprefixer": "^10.4.19",
-    "dotenv-cli": "^7.3.0",
+    "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
     "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",

web/public/generated/api/openapi.yml

@@ -1825,8 +1825,8 @@ components:
       title: ScoreSource
       type: string
       enum:
+        - ANNOTATION
         - API
-        - REVIEW
         - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest

web/sentry.server.config.ts

@@ -18,12 +18,21 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.3;
+        return 0.1;
       }
-      return 0.1;
+      if (
+        samplingContext.request &&
+        samplingContext.request.url &&
+        samplingContext.request.url.includes("api/auth") &&
+        samplingContext.transactionContext.status !== "ok" &&
+        samplingContext.transactionContext.status !== "unauthenticated"
+      ) {
+        return 0.1;
+      }
+      return 0.01;
     },
 
-    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

web/src/__tests__/ingestion.servertest.ts

@@ -727,6 +727,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             metadata: { key: "value" },
             release: "1.0.0",
             version: "2.0.0",
+            tags: ["tag-1", "tag-2", "tag-2"],
           },
         },
       ],
@@ -743,6 +744,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             id: traceId,
             name: "trace-name",
             userId: "user-2",
+            tags: ["tag-1", "tag-4", "tag-3"],
           },
         },
       ],
@@ -763,6 +765,8 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace[0]?.externalId).toBeNull();
     expect(dbTrace[0]?.version).toBe("2.0.0");
     expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(dbTrace[0]?.tags).toEqual(["tag-1", "tag-2", "tag-3", "tag-4"]);
+    expect(dbTrace[0]?.tags.length).toBe(4);
   });
 
   it("should fail for wrong event formats", async () => {

web/src/__tests__/scores.servertest.ts

@@ -489,16 +489,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_2,
           name: scoreName,
           value: 50.5,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator >=", async () => {
@@ -514,16 +514,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
       ]);
     });
     it("test operator !=", async () => {
@@ -539,16 +539,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator =", async () => {

web/src/__tests__/traces.servertest.ts

@@ -182,6 +182,32 @@ describe("/api/public/traces API Endpoint", () => {
       endTime: "2021-01-01T00:20:00.000Z",
     });
 
+    // Simulate scores on the trace
+    const scoreId1 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId1,
+      name: "score-1",
+      value: 75.0,
+      traceId: traceId,
+      comment: "First score",
+    });
+    const scoreId2 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId2,
+      name: "score-2",
+      value: 85.5,
+      traceId: traceId,
+      comment: "Second score",
+    });
+    const scoreId3 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId3,
+      name: "score-3",
+      value: 95.0,
+      traceId: traceId,
+      comment: "Third score",
+    });
+
     // GET traces
     // Retrieve the trace with totalCost and latency
     const traces = await makeAPICall<GetTracesAPIResponse>(

web/src/app/api/chatCompletion/route.ts

@@ -0,0 +1,6 @@
+import chatCompletionHandler from "@/src/ee/features/playground/server/chatCompletionHandler";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 60;
+
+export const POST = chatCompletionHandler;

web/src/components/ChatMessages/ChatMessageComponent.tsx

@@ -63,6 +63,7 @@ export const ChatMessageComponent: React.FC<ChatMessageProps> = ({
 
         <Textarea
           ref={textAreaRef}
+          id={message.id}
           className="height-[auto] min-h-8 w-full pt-3  font-mono text-xs focus:outline-none"
           placeholder={placeholder}
           value={message.content}

web/src/components/LangfuseLogo.tsx

@@ -41,10 +41,10 @@ export const LangfuseLogo = ({
         className={cn(
           "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
           env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
-            ? "bg-blue-100 text-blue-500 ring-blue-500"
+            ? "bg-light-blue text-dark-blue ring-dark-blue"
             : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
-              ? "bg-green-100 text-green-500 ring-green-500"
-              : "bg-red-100 text-red-500 ring-red-500",
+              ? "bg-light-green text-dark-green ring-dark-green"
+              : "bg-light-red text-dark-red ring-dark-red",
         )}
       >
         {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
@@ -76,7 +76,7 @@ export const LangfuseLogo = ({
           target="_blank"
           rel="noopener"
           title="View releases on GitHub"
-          className="ml-2 text-xs text-gray-400"
+          className="ml-2 text-xs text-muted-foreground"
         >
           {VERSION}
         </a>

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -0,0 +1,77 @@
+import Link from "next/link";
+
+import { Label } from "@/src/components/ui/label";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+export const LLMApiKeyComponent = (p: {
+  projectId: string;
+  modelParams: UIModelParams;
+}) => {
+  const hasAccess = useHasAccess({
+    projectId: p.projectId,
+    scope: "llmApiKeys:read",
+  });
+
+  if (!hasAccess) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">
+          LLM API Key only visible to Owner and Admin roles.
+        </p>
+      </div>
+    );
+  }
+
+  const apiKeys = api.llmApiKey.all.useQuery({
+    projectId: p.projectId,
+  });
+
+  if (apiKeys.isLoading) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">Loading...</p>
+      </div>
+    );
+  }
+
+  const model = p.modelParams.model.value;
+  const modelProvider = Object.entries(supportedModels).find((providerData) =>
+    (providerData[1] as any as string[]).includes(model),
+  )?.[0];
+
+  const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
+
+  return (
+    <div className="space-y-2 text-xs">
+      <Label className="text-xs font-semibold">API key</Label>
+      <div>
+        {apiKey ? (
+          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+            {apiKey.displaySecretKey}
+          </span>
+        ) : undefined}
+      </div>
+      {/* Custom form message to include a link to the already existing prompt */}
+      {!apiKey ? (
+        <div className="flex flex-col font-medium text-destructive">
+          {`No LLM API key found for provider ${modelProvider}.`}
+
+          <Link
+            href={`/project/${p.projectId}/settings`}
+            className="flex flex-row"
+          >
+            Create a new LLM API key here. <ArrowTopRightIcon />
+          </Link>
+        </div>
+      ) : undefined}
+      <p className="text-muted-foreground">
+        The LLM API key is used for each execution and will incur costs.
+      </p>
+    </div>
+  );
+};

web/src/components/ModelParameters/index.tsx

@@ -8,6 +8,7 @@ import {
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
 import { Switch } from "@/src/components/ui/switch";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { cn } from "@/src/utils/tailwind";
 import {
   ModelProvider,
@@ -15,6 +16,8 @@ import {
   type UIModelParams,
 } from "@langfuse/shared";
 
+import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
+
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
@@ -33,10 +36,14 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   setModelParamEnabled,
   formDisabled = false,
 }) => {
+  const projectId = useProjectIdFromURL();
+
+  if (!projectId) return null;
+
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-6">
+      <div className="space-y-4">
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
@@ -104,6 +111,7 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
           updateModelParam={updateModelParamValue}
         />
+        <LLMApiKeyComponent {...{ projectId, modelParams }} />
       </div>
     </div>
   );
@@ -127,7 +135,12 @@ const ModelParamsSelect = ({
 }: ModelParamsSelectProps) => {
   return (
     <div className="space-y-2">
-      <p className={cn("text-xs font-semibold", disabled && "text-gray-400")}>
+      <p
+        className={cn(
+          "text-xs font-semibold",
+          disabled && "text-muted-foreground",
+        )}
+      >
         {title}
       </p>
       <Select
@@ -187,7 +200,7 @@ const ModelParamsSlider = ({
         <p
           className={cn(
             "flex-1 text-xs font-semibold",
-            (!enabled || formDisabled) && "text-gray-400",
+            (!enabled || formDisabled) && "text-muted-foreground",
           )}
         >
           {title}

web/src/components/Slider.tsx

@@ -7,27 +7,23 @@ export const Slider = (props: {
   loading?: boolean;
   onChecked?: (checked: boolean) => void;
   isChecked?: boolean; // whether the slider is active
-}) => {
-  console.log("props", props);
-
-  return (
-    <Switch
-      checked={props.isChecked}
-      disabled={props.loading || props.disabled}
-      onChange={props.onChecked}
+}) => (
+  <Switch
+    checked={props.isChecked}
+    disabled={props.loading || props.disabled}
+    onChange={props.onChecked}
+    className={cn(
+      props.isChecked ? "bg-background" : "bg-input",
+      "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-foreground focus:ring-offset-2",
+    )}
+  >
+    <span className="sr-only">Use setting</span>
+    <span
+      aria-hidden="true"
       className={cn(
-        props.isChecked ? "bg-black" : "bg-gray-200",
-        "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-black focus:ring-offset-2",
+        props.isChecked ? "translate-x-5" : "translate-x-0",
+        "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-background shadow ring-0 transition duration-200 ease-in-out",
       )}
-    >
-      <span className="sr-only">Use setting</span>
-      <span
-        aria-hidden="true"
-        className={cn(
-          props.isChecked ? "translate-x-5" : "translate-x-0",
-          "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out",
-        )}
-      />
-    </Switch>
-  );
-};
+    />
+  </Switch>
+);

web/src/components/Support.tsx

@@ -8,14 +8,7 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { chatAvailable, openChat } from "@/src/features/support-chat/chat";
-import {
-  Book,
-  Calendar,
-  Github,
-  Mail,
-  MessageSquare,
-  Slack,
-} from "lucide-react";
+import { Book, Github, Mail, MessageSquare, Slack } from "lucide-react";
 import Link from "next/link";
 import { SiDiscord } from "react-icons/si";
 
@@ -30,7 +23,7 @@ const supportChannels = [
   },
   {
     icon: Github,
-    title: "GitHub",
+    title: "GitHub Issues",
     description:
       "Create an issue on Github to report bugs or request new features.",
     href: "https://github.com/langfuse/langfuse/issues/new/choose",
@@ -38,12 +31,11 @@ const supportChannels = [
     primary: true,
   },
   {
-    icon: SiDiscord,
-    title: "Discord",
-    description:
-      "Get support from community and maintainers. Follow announcements to stay up to date with new features.",
-    href: "https://langfuse.com/discord",
-    buttonText: "Join Discord",
+    icon: Github,
+    title: "GitHub Support",
+    description: "Create a support ticket via GitHub discussions.",
+    href: "https://github.com/orgs/langfuse/discussions/categories/support",
+    buttonText: "Submit question",
     primary: true,
   },
   {
@@ -57,17 +49,18 @@ const supportChannels = [
   {
     icon: MessageSquare,
     title: "Chat",
-    description: "Get quick support directly from the team.",
+    description: "Get support directly from the team.",
     onClick: () => openChat(),
     available: chatAvailable,
     buttonText: "Launch Chat",
   },
   {
-    icon: Calendar,
-    title: "Schedule call",
-    description: "Schedule a call with one of the founders.",
-    href: "https://cal.com/marc-kl/office-hours",
-    buttonText: "Schedule Call",
+    icon: SiDiscord,
+    title: "Discord",
+    description:
+      "Get support from community. Follow announcements to stay up to date with new features.",
+    href: "https://langfuse.com/discord",
+    buttonText: "Join Discord",
   },
   {
     icon: Slack,

web/src/components/error-page.tsx

@@ -18,7 +18,7 @@ export const ErrorPage = ({
 
   return (
     <div className="flex h-screen flex-col items-center justify-center">
-      <AlertCircle className="mb-4 h-12 w-12 text-red-500" />
+      <AlertCircle className="text-dark-red mb-4 h-12 w-12" />
       <h1 className="mb-4 text-xl font-bold">{title}</h1>
       <p className="mb-8 text-center">{message}</p>
       {session.status === "unauthenticated" ? (

web/src/components/grouped-score-badge.tsx

@@ -58,7 +58,7 @@ export const GroupedScoreBadges = ({
           .sort(([a], [b]) => (a < b ? -1 : 1))
           .map(([name, scores]) => (
             <div key={name}>
-              <div className="text-xs text-gray-500">{name}</div>
+              <div className="text-xs text-muted-foreground">{name}</div>
               <ScoresOfGroup scores={scores} />
             </div>
           ))}

web/src/components/json-editor.tsx

@@ -1,7 +1,11 @@
 import CodeMirror, { EditorView } from "@uiw/react-codemirror";
 import { githubLight } from "@uiw/codemirror-theme-github";
-import { json } from "@codemirror/lang-json";
+import { tokyoNight } from "@uiw/codemirror-theme-tokyo-night";
+import { json, jsonParseLinter } from "@codemirror/lang-json";
+import { linter } from "@codemirror/lint";
+import { useTheme } from "next-themes";
 import { cn } from "@/src/utils/tailwind";
+import { useState } from "react";
 
 // todo: add json linting
 
@@ -18,17 +22,31 @@ export function JsonEditor({
   lineWrapping?: boolean;
   className?: string;
 }) {
+  const { resolvedTheme } = useTheme();
+  const codeMirrorTheme = resolvedTheme === "dark" ? tokyoNight : githubLight;
+  
+  // used to disable linter when field is empty
+  const [linterEnabled, setLinterEnabled] = useState<boolean>(
+    !!defaultValue && defaultValue !== "",
+  );
   return (
     <CodeMirror
       value={defaultValue}
-      theme={githubLight}
+      theme={codeMirrorTheme}
       basicSetup={{
         foldGutter: true,
       }}
       lang={"json"}
-      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      extensions={[
+        json(),
+        ...(linterEnabled ? [linter(jsonParseLinter())] : []),
+        ...(lineWrapping ? [EditorView.lineWrapping] : []),
+      ]}
       defaultValue={defaultValue}
-      onChange={onChange}
+      onChange={(c) => {
+        if (onChange) onChange(c);
+        setLinterEnabled(c !== "");
+      }}
       className={cn("overflow-hidden rounded-md border", className)}
       editable={editable}
     />

web/src/components/layouts/doc-popup.tsx

@@ -4,6 +4,7 @@ import {
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { cn } from "@/src/utils/tailwind";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
 
@@ -40,13 +41,16 @@ export default function DocPopup({
         }
       }}
     >
-      <HoverCardTrigger className="mx-1 cursor-pointer" asChild>
+      <HoverCardTrigger
+        className={cn("mx-1", href ? "cursor-pointer" : "cursor-default")}
+        asChild
+      >
         {href ? (
           <Link
             href={href}
             rel="noopener"
             target="_blank"
-            className="inline-block whitespace-nowrap text-gray-500 sm:pl-0"
+            className="inline-block whitespace-nowrap text-muted-foreground sm:pl-0"
             onClick={() => {
               capture("help_popup:href_clicked", {
                 href: href,
@@ -62,7 +66,7 @@ export default function DocPopup({
             }
           </Link>
         ) : (
-          <div className="inline-block whitespace-nowrap text-gray-500 sm:pl-0">
+          <div className="inline-block whitespace-nowrap text-muted-foreground sm:pl-0">
             {
               {
                 question: <HelpCircle className={sizes[size]} />,
@@ -74,7 +78,7 @@ export default function DocPopup({
       </HoverCardTrigger>
       <HoverCardContent>
         {typeof description === "string" ? (
-          <div className="whitespace-break-spaces text-xs font-normal text-gray-800 sm:pl-0">
+          <div className="whitespace-break-spaces text-xs font-normal text-primary sm:pl-0">
             {description}
           </div>
         ) : (
@@ -98,7 +102,7 @@ export function Popup({ triggerContent, description }: PopupProps) {
       </HoverCardTrigger>
       <HoverCardContent>
         {typeof description === "string" ? (
-          <div className="whitespace-break-spaces text-xs font-normal text-gray-800 sm:pl-0">
+          <div className="whitespace-break-spaces text-xs font-normal text-primary sm:pl-0">
             {description}
           </div>
         ) : (

web/src/components/layouts/header.tsx

@@ -49,10 +49,10 @@ export default function Header({
           <nav className="sm:hidden" aria-label="Back">
             <Link
               href={backHref}
-              className="flex items-center text-sm font-medium text-gray-500 hover:text-gray-700"
+              className="flex items-center text-sm font-medium text-muted-foreground hover:text-primary"
             >
               <ChevronLeftIcon
-                className="-ml-1 mr-1 h-5 w-5 flex-shrink-0 text-gray-400"
+                className="-ml-1 mr-1 h-5 w-5 flex-shrink-0 text-muted-foreground"
                 aria-hidden="true"
               />
               Back
@@ -67,19 +67,19 @@ export default function Header({
                   <div className="flex items-center">
                     {index !== 0 && (
                       <ChevronRightIcon
-                        className="mr-4 h-5 w-5 flex-shrink-0 text-gray-400"
+                        className="mr-4 h-5 w-5 flex-shrink-0 text-muted-foreground"
                         aria-hidden="true"
                       />
                     )}
                     {href ? (
                       <Link
                         href={href}
-                        className="text-sm font-medium text-gray-500 hover:text-gray-700"
+                        className="text-sm font-medium text-muted-foreground hover:text-primary"
                       >
                         {name}
                       </Link>
                     ) : (
-                      <div className="text-sm font-medium text-gray-500">
+                      <div className="text-sm font-medium text-muted-foreground">
                         {name}
                       </div>
                     )}
@@ -94,11 +94,11 @@ export default function Header({
         <div className="flex items-center gap-3 md:gap-5">
           <div className="flex min-w-0 flex-row justify-center align-middle">
             {level === "h2" ? (
-              <h2 className="text-2xl font-bold leading-7 text-gray-900 sm:truncate sm:text-3xl sm:tracking-tight">
+              <h2 className="text-2xl font-bold leading-7 sm:truncate sm:text-3xl sm:tracking-tight">
                 {props.title}
               </h2>
             ) : (
-              <h3 className="text-lg font-bold leading-7 text-gray-900 sm:truncate sm:text-xl sm:tracking-tight">
+              <h3 className="text-lg font-bold leading-7 sm:truncate sm:text-xl sm:tracking-tight">
                 {props.title}
               </h3>
             )}

web/src/components/layouts/layout.tsx

@@ -31,14 +31,26 @@ import { ChevronDownIcon } from "@heroicons/react/20/solid";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { ProjectNavigation } from "@/src/components/projectNavigation";
 import DOMPurify from "dompurify";
+import { ThemeToggle } from "@/src/features/theming/ThemeToggle";
+
+const signOutUser = async () => {
+  localStorage.clear();
+  sessionStorage.clear();
+
+  await signOut({
+    callbackUrl: "/auth/sign-in",
+  });
+};
 
 const userNavigation = [
+  {
+    name: "Theme",
+    onClick: () => {},
+    content: <ThemeToggle />,
+  },
   {
     name: "Sign out",
-    onClick: () =>
-      signOut({
-        callbackUrl: "/auth/sign-in",
-      }),
+    onClick: signOutUser,
   },
 ];
 
@@ -135,9 +147,8 @@ export default function Layout(props: PropsWithChildren) {
     !publishablePaths.includes(router.pathname) &&
     !router.pathname.startsWith("/public/")
   ) {
-    void signOut({
-      callbackUrl: "/auth/sign-in",
-    });
+    signOutUser();
+
     return <Spinner message="Redirecting" />;
   }
 
@@ -178,7 +189,7 @@ export default function Layout(props: PropsWithChildren) {
     router.pathname.startsWith("/public/");
   if (hideNavigation)
     return (
-      <main className="min-h-screen bg-gray-50 px-4 py-4 sm:px-6 lg:px-8">
+      <main className="min-h-screen bg-primary-foreground px-4 py-4 sm:px-6 lg:px-8">
         {props.children}
       </main>
     );
@@ -222,7 +233,7 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="opacity-100"
               leaveTo="opacity-0"
             >
-              <div className="fixed inset-0 bg-gray-900/80" />
+              <div className="fixed inset-0 bg-primary/80" />
             </Transition.Child>
 
             <div className="fixed inset-0 flex">
@@ -253,14 +264,14 @@ export default function Layout(props: PropsWithChildren) {
                       >
                         <span className="sr-only">Close sidebar</span>
                         <XMarkIcon
-                          className="h-6 w-6 text-white"
+                          className="h-5 w-5 text-background"
                           aria-hidden="true"
                         />
                       </button>
                     </div>
                   </Transition.Child>
                   {/* Sidebar component, swap this element with another sidebar if you like */}
-                  <div className="flex grow flex-col gap-y-5 overflow-y-auto bg-white px-6 py-4">
+                  <div className="flex grow flex-col gap-y-5 overflow-y-auto bg-background px-6 py-4">
                     <LangfuseLogo
                       version
                       size="xl"
@@ -273,7 +284,7 @@ export default function Layout(props: PropsWithChildren) {
                         <MainNavigation nav={navigation} />
                       </ul>
                       <div className="mb-2 flex flex-row place-content-between items-center">
-                        <div className="text-xs font-semibold leading-6 text-gray-400">
+                        <div className="text-xs font-semibold text-muted-foreground">
                           Project
                         </div>
                         <NewProjectButton size="xs" />
@@ -291,9 +302,9 @@ export default function Layout(props: PropsWithChildren) {
         </Transition.Root>
 
         {/* Static sidebar for desktop */}
-        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-60 lg:flex-col">
+        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-56 lg:flex-col">
           {/* Sidebar component, swap this element with another sidebar if you like */}
-          <div className="flex h-screen grow flex-col border-r border-gray-200 bg-white pt-7">
+          <div className="flex h-screen grow flex-col border-r border-border bg-background pt-7">
             <LangfuseLogo
               version
               size="xl"
@@ -312,16 +323,16 @@ export default function Layout(props: PropsWithChildren) {
                   description="What do you think about this project? What can be improved?"
                   type="feedback"
                 >
-                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
+                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-1.5 text-sm font-semibold text-primary hover:bg-primary-foreground hover:text-primary-accent">
                     <MessageSquarePlus
-                      className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                      className="h-5 w-5 shrink-0 text-muted-foreground group-hover:text-primary-accent"
                       aria-hidden="true"
                     />
                     Feedback
                   </li>
                 </FeedbackButtonWrapper>
                 <div className="mb-2 flex flex-row place-content-between items-center">
-                  <div className="text-xs font-semibold leading-6 text-gray-400">
+                  <div className="text-xs font-semibold text-muted-foreground">
                     Project
                   </div>
                   <NewProjectButton size="xs" />
@@ -334,9 +345,9 @@ export default function Layout(props: PropsWithChildren) {
             </nav>
 
             <Menu as="div" className="relative">
-              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+              <Menu.Button className="flex w-full items-center gap-x-2 overflow-hidden p-1.5 py-3 pl-6 pr-8 text-sm font-semibold text-primary hover:bg-primary-foreground">
                 <span className="sr-only">Open user menu</span>
-                <Avatar className="h-8 w-8">
+                <Avatar className="h-7 w-7">
                   <AvatarImage src={session.data?.user?.image ?? undefined} />
                   <AvatarFallback>
                     {session.data?.user?.name
@@ -348,12 +359,12 @@ export default function Layout(props: PropsWithChildren) {
                       : null}
                   </AvatarFallback>
                 </Avatar>
-                <span className="flex-shrink truncate text-sm font-semibold leading-6 text-gray-900">
+                <span className="flex-shrink truncate text-sm font-semibold text-primary">
                   {session.data?.user?.name}
                 </span>
                 <div className="flex-1" />
                 <ChevronDownIcon
-                  className="h-5 w-5 text-gray-400"
+                  className="h-5 w-5 text-muted-foreground"
                   aria-hidden="true"
                 />
               </Menu.Button>
@@ -366,8 +377,8 @@ export default function Layout(props: PropsWithChildren) {
                 leaveFrom="transform opacity-100 scale-100"
                 leaveTo="transform opacity-0 scale-95"
               >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                <Menu.Items className="absolute -top-full bottom-1 right-0 z-10 overflow-hidden rounded-md bg-background py-2 shadow-lg ring-1 ring-border focus:outline-none">
+                  <span className="block border-b px-3 pb-2 text-sm leading-6 text-muted-foreground">
                     {session.data?.user?.email}
                   </span>
                   {userNavigation.map((item) => (
@@ -376,11 +387,12 @@ export default function Layout(props: PropsWithChildren) {
                         <a
                           onClick={() => void item.onClick()}
                           className={cn(
-                            active ? "bg-gray-50" : "",
-                            "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                            active ? "bg-primary-foreground" : "",
+                            "flex cursor-pointer items-center justify-between px-2 py-0.5 text-sm leading-6 text-primary",
                           )}
                         >
                           {item.name}
+                          {item.content}
                         </a>
                       )}
                     </Menu.Item>
@@ -391,14 +403,14 @@ export default function Layout(props: PropsWithChildren) {
           </div>
         </div>
 
-        <div className="sticky top-0 z-40 flex items-center gap-x-6 bg-white px-4 py-4 shadow-sm sm:px-6 lg:hidden">
+        <div className="sticky top-0 z-40 flex items-center gap-x-6 bg-background px-4 py-4 shadow-sm sm:px-6 lg:hidden">
           <button
             type="button"
-            className="-m-2.5 p-2.5 text-gray-700 lg:hidden"
+            className="-m-2.5 p-2.5 text-primary lg:hidden"
             onClick={() => setSidebarOpen(true)}
           >
             <span className="sr-only">Open sidebar</span>
-            <Bars3Icon className="h-6 w-6" aria-hidden="true" />
+            <Bars3Icon className="h-5 w-5" aria-hidden="true" />
           </button>
           <LangfuseLogo
             version
@@ -406,7 +418,7 @@ export default function Layout(props: PropsWithChildren) {
             showEnvLabel={session.data?.user?.email?.endsWith("@langfuse.com")}
           />
           <Menu as="div" className="relative">
-            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold leading-6 text-gray-900">
+            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold text-primary">
               <span className="sr-only">Open user menu</span>
               <Avatar className="h-8 w-8">
                 <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -430,8 +442,8 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="transform opacity-100 scale-100"
               leaveTo="transform opacity-0 scale-95"
             >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-background py-2 pb-1 shadow-lg ring-1 ring-border focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-muted-foreground">
                   {session.data?.user?.email}
                 </span>
                 {userNavigation.map((item) => (
@@ -440,11 +452,12 @@ export default function Layout(props: PropsWithChildren) {
                       <a
                         onClick={() => void item.onClick()}
                         className={cn(
-                          active ? "bg-gray-50" : "",
-                          "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                          active ? "bg-primary-foreground" : "",
+                          "flex cursor-pointer items-center justify-between px-2 py-1 text-sm leading-6 text-primary",
                         )}
                       >
                         {item.name}
+                        {item.content}
                       </a>
                     )}
                   </Menu.Item>
@@ -453,13 +466,13 @@ export default function Layout(props: PropsWithChildren) {
             </Transition>
           </Menu>
         </div>
-        <div className="lg:pl-60">
+        <div className="lg:pl-56">
           {env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           projectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING" ||
             env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU") &&
           !session.data?.user?.email?.endsWith("@langfuse.com") ? (
-            <div className="flex w-full items-center border-b border-yellow-500  bg-yellow-100 px-4 py-2 lg:sticky lg:top-0 lg:z-40">
+            <div className="flex w-full items-center border-b border-dark-yellow  bg-light-yellow px-4 py-2 lg:sticky lg:top-0 lg:z-40">
               <div className="flex flex-1 flex-wrap gap-1">
                 <div className="flex items-center gap-1">
                   <Info className="h-4 w-4" />
@@ -523,9 +536,9 @@ const MainNavigation: React.FC<{
                 href={item.href}
                 className={clsx(
                   item.current
-                    ? "bg-gray-50 text-indigo-600"
-                    : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
+                    ? "bg-primary-foreground text-primary-accent"
+                    : "text-primary hover:bg-primary-foreground hover:text-primary-accent",
+                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold",
                 )}
                 onClick={onNavitemClick}
                 target={item.newTab ? "_blank" : undefined}
@@ -534,9 +547,9 @@ const MainNavigation: React.FC<{
                   <item.icon
                     className={clsx(
                       item.current
-                        ? "text-indigo-600"
-                        : "text-gray-400 group-hover:text-indigo-600",
-                      "h-6 w-6 shrink-0",
+                        ? "text-primary-accent"
+                        : "text-muted-foreground group-hover:text-primary-accent",
+                      "h-5 w-5 shrink-0",
                     )}
                     aria-hidden="true"
                   />
@@ -545,10 +558,10 @@ const MainNavigation: React.FC<{
                 {item.label && (
                   <span
                     className={cn(
-                      "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                      "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                       item.current
-                        ? "border-indigo-600 text-indigo-600"
-                        : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
+                        ? "border-primary-accent text-primary-accent"
+                        : "border-border text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent",
                     )}
                   >
                     {item.label}
@@ -565,12 +578,12 @@ const MainNavigation: React.FC<{
                 {({ open }) => (
                   <>
                     <Disclosure.Button
-                      className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold leading-6 hover:bg-gray-50 hover:text-indigo-600"
+                      className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold hover:bg-primary-foreground hover:text-primary-accent"
                       onClick={() => setIsOpen(!isOpen)}
                     >
                       {item.icon && (
                         <item.icon
-                          className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                          className="h-5 w-5 shrink-0 text-muted-foreground group-hover:text-primary-accent"
                           aria-hidden="true"
                         />
                       )}
@@ -578,10 +591,10 @@ const MainNavigation: React.FC<{
                       {item.label && (
                         <span
                           className={cn(
-                            "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                            "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                             item.current
-                              ? "border-indigo-600 text-indigo-600"
-                              : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
+                              ? "border-primary-accent text-primary-accent"
+                              : "border-border text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent",
                           )}
                         >
                           {item.label}
@@ -589,13 +602,15 @@ const MainNavigation: React.FC<{
                       )}
                       <ChevronRightIcon
                         className={clsx(
-                          open ? "rotate-90 text-gray-500" : "text-gray-400",
+                          open
+                            ? "rotate-90 text-muted-foreground"
+                            : "text-muted-foreground",
                           "ml-auto h-5 w-5 shrink-0",
                         )}
                         aria-hidden="true"
                       />
                     </Disclosure.Button>
-                    <Disclosure.Panel as="ul" className="mt-1 px-2">
+                    <Disclosure.Panel as="ul" className="mt-1 space-y-1 px-2">
                       {item.children?.map((subItem) => (
                         <li key={subItem.name}>
                           {/* 44px */}
@@ -603,15 +618,15 @@ const MainNavigation: React.FC<{
                             href={subItem.href ?? "#"}
                             className={clsx(
                               subItem.current
-                                ? "bg-gray-50 text-indigo-600"
-                                : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
+                                ? "bg-primary-foreground text-primary-accent"
+                                : "text-primary hover:bg-primary-foreground hover:text-primary-accent",
+                              "ml-0.5 flex w-full items-center gap-x-3 rounded-md p-1.5 pl-7 pr-2 text-sm",
                             )}
                             target={subItem.newTab ? "_blank" : undefined}
                           >
                             {subItem.name}
                             {subItem.label && (
-                              <span className="self-center whitespace-nowrap break-keep rounded-sm border border-gray-200 px-1 py-0.5 text-xs text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600">
+                              <span className="self-center whitespace-nowrap break-keep rounded-sm border border-border px-1 py-0.5 text-xs text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent">
                                 {subItem.label}
                               </span>
                             )}

web/src/components/layouts/spinner.tsx

@@ -5,7 +5,7 @@ export function Spinner(props: { message: string }) {
     <div className="flex min-h-full flex-1 flex-col justify-center py-12 sm:px-6 lg:px-8">
       <div className="sm:mx-auto sm:w-full sm:max-w-md">
         <LangfuseIcon className="mx-auto motion-safe:animate-spin" size={42} />
-        <h2 className="mt-5 text-center text-2xl font-bold leading-9 tracking-tight text-gray-900">
+        <h2 className="mt-5 text-center text-2xl font-bold leading-9 tracking-tight text-primary">
           {props.message} ...
         </h2>
       </div>

web/src/components/layouts/status-badge.tsx

@@ -11,22 +11,22 @@ export type Status =
   (typeof statusCategories)[keyof typeof statusCategories][number];
 
 export const StatusBadge = (props: { className?: string; type: Status }) => {
-  let badgeColor = "bg-gray-100 text-gray-800";
-  let dotColor = "bg-gray-500";
-  let dotPingColor = "bg-gray-600";
+  let badgeColor = "bg-muted-gray text-primary";
+  let dotColor = "bg-muted-foreground";
+  let dotPingColor = "bg-muted-foreground";
   let showDot = true;
 
   if (statusCategories.active.includes(props.type)) {
-    badgeColor = "bg-green-100 text-green-600";
-    dotColor = "animate-ping bg-green-500";
-    dotPingColor = "bg-green-600";
+    badgeColor = "bg-light-green text-dark-green";
+    dotColor = "animate-ping bg-dark-green";
+    dotPingColor = "bg-dark-green";
   } else if (statusCategories.error.includes(props.type)) {
-    badgeColor = "bg-red-100 text-red-600";
-    dotColor = "bg-red-500";
-    dotPingColor = "bg-red-600";
+    badgeColor = "bg-light-red text-dark-red";
+    dotColor = "animate-ping bg-dark-red";
+    dotPingColor = "bg-dark-red";
     showDot = false;
   } else if (statusCategories.completed.includes(props.type)) {
-    badgeColor = "bg-green-100 text-green-600";
+    badgeColor = "bg-light-green text-dark-green";
     showDot = false;
   }
 

web/src/components/level-colors.tsx

@@ -1,6 +1,6 @@
 export const LevelColors = {
   DEFAULT: { text: "", bg: "" },
-  DEBUG: { text: "text-gray-500", bg: "bg-gray-50" },
-  WARNING: { text: "text-yellow-800", bg: "bg-yellow-50" },
-  ERROR: { text: "text-red-800", bg: "bg-red-50" },
+  DEBUG: { text: "text-muted-foreground", bg: "bg-primary-foreground" },
+  WARNING: { text: "text-dark-yellow", bg: "bg-light-yellow" },
+  ERROR: { text: "text-dark-red", bg: "bg-light-red" },
 };

web/src/components/projectNavigation.tsx

@@ -32,9 +32,9 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
         router.push(`/project/${value}`);
       }}
     >
-      <SelectTrigger className="text-gray-700 ring-transparent focus:ring-0 focus:ring-offset-0">
+      <SelectTrigger className="h-8 text-primary ring-transparent focus:ring-0 focus:ring-offset-0">
         <SelectValue
-          className="text-sm font-semibold text-gray-700"
+          className="text-sm font-semibold text-primary"
           placeholder={currentProjectId}
         />
       </SelectTrigger>
@@ -46,8 +46,8 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
                 className={cn(
                   "truncate",
                   currentProjectId === project.id
-                    ? "border-gray-700 text-gray-700"
-                    : "text-gray-400",
+                    ? "border-primary text-primary"
+                    : "text-muted-foreground",
                 )}
               >
                 {project.name}
@@ -57,8 +57,8 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
                   className={cn(
                     "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                     currentProjectId === project.id
-                      ? "border-gray-700 text-gray-700"
-                      : "border-gray-200 text-gray-400 group-hover:border-gray-700 group-hover:text-gray-700",
+                      ? "border-primary text-primary"
+                      : "border-border text-muted-foreground group-hover:border-primary group-hover:text-primary",
                   )}
                 >
                   view-only

web/src/components/publish-object-switch.tsx

@@ -93,7 +93,7 @@ const Base = (props: {
           "Loading.."
         ) : props.isPublic ? (
           <div
-            className="flex cursor-pointer items-center gap-1 text-green-800"
+            className="text-dark-green flex cursor-pointer items-center gap-1"
             onClick={() => copyUrl()}
           >
             {isCopied ? "Link copied ..." : "Public"}

web/src/components/session/index.tsx

@@ -171,7 +171,7 @@ const TraceCardList = ({
           }))
           .map(({ virtualItem, trace }) => (
             <Card
-              className="border-border-gray-150 group grid w-full gap-3 overflow-hidden p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
+              className="group grid w-full gap-3 overflow-hidden border-border p-2 shadow-none hover:border-ring   md:grid-cols-3"
               key={virtualItem.key}
               data-index={virtualItem.index}
               style={{
@@ -193,10 +193,12 @@ const TraceCardList = ({
                   >
                     Trace: {trace.name} ({trace.id})&nbsp;↗
                   </Link>
-                  <div className="text-xs text-gray-500">
+                  <div className="text-xs text-muted-foreground">
                     {trace.timestamp.toLocaleString()}
                   </div>
-                  <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
+                  <div className="mb-1 mt-2 text-xs text-muted-foreground">
+                    Scores
+                  </div>
                   <div className="flex flex-wrap content-start items-start gap-1">
                     <GroupedScoreBadges scores={trace.scores} />
                   </div>
@@ -244,7 +246,7 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
           hideIfNull
         />
       ) : (
-        <div className="p-2 text-xs text-gray-500">
+        <div className="p-2 text-xs text-muted-foreground">
           This trace has no input or output.
         </div>
       )}

web/src/components/star-toggle.tsx

@@ -7,6 +7,7 @@ import { cn } from "@/src/utils/tailwind";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import { useState } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 export function StarToggle({
   value,
@@ -32,7 +33,7 @@ export function StarToggle({
       <StarIcon
         className={cn(
           "h-4 w-4",
-          value ? "fill-current text-yellow-500" : "text-gray-500",
+          value ? "fill-current text-yellow-500" : "text-muted-foreground",
         )}
       />
     </Button>
@@ -76,7 +77,7 @@ export function StarTraceToggle({
     onError: (err, _newTodo, context) => {
       setIsLoading(false);
       // Rollback to the previous value if mutation fails
-      console.log("error", err);
+      trpcErrorToast(err);
       utils.traces.all.setData(tracesFilter, context?.prev);
     },
     onSettled: () => {
@@ -157,7 +158,7 @@ export function StarTraceDetailsToggle({
     },
     onError: (err, _newTodo, context) => {
       setIsLoading(false);
-      console.log("error", err);
+      trpcErrorToast(err);
       // Rollback to the previous value if mutation fails
       utils.traces.byId.setData({ traceId }, context?.prevData);
     },

web/src/components/stats-cards.tsx

@@ -6,19 +6,19 @@ interface Statistic {
 export default function StatsCards({ stats }: { stats: Statistic[] }) {
   return (
     <div>
-      <h3 className="text-base font-semibold leading-6 text-gray-900">
+      <h3 className="text-base font-semibold leading-6 text-primary">
         Model configuration
       </h3>
       <dl className="mt-5 grid grid-cols-1 gap-5 sm:grid-cols-4">
         {stats.map((item) => (
           <div
             key={item.name}
-            className="overflow-hidden rounded-lg bg-white px-4 py-5 shadow sm:p-6"
+            className="overflow-hidden rounded-lg bg-background px-4 py-5 shadow sm:p-6"
           >
-            <dt className="truncate text-sm font-medium text-gray-500">
+            <dt className="truncate text-sm font-medium text-muted-foreground">
               {item.name}
             </dt>
-            <dd className="mt-1 text-3xl font-semibold tracking-tight text-gray-900">
+            <dd className="mt-1 text-3xl font-semibold tracking-tight text-primary">
               {item.stat}
             </dd>
           </div>

web/src/components/table/data-table-column-visibility-filter.tsx

@@ -83,7 +83,7 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
       >
         <Button variant="outline" title="Show/hide columns">
           <Columns className="mr-2 h-4 w-4" />
-          <span className="text-xs text-gray-500">{`(${count}/${total})`}</span>
+          <span className="text-xs text-muted-foreground">{`(${count}/${total})`}</span>
           <ChevronDown className="ml-2 h-4 w-4" />
         </Button>
       </DropdownMenuTrigger>

web/src/components/table/data-table-row-height-switch.tsx

@@ -55,7 +55,7 @@ export const DataTableRowHeightSwitch = ({
           <TabsTrigger
             key={id}
             value={id}
-            className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
+            className="px-2 shadow-none data-[state=active]:bg-input data-[state=active]:ring-border"
           >
             <span role="img" aria-label={`${label} size`}>
               {icon}

web/src/components/table/data-table.tsx

@@ -12,7 +12,6 @@ import {
   Table,
   TableBody,
   TableCell,
-  TableFooter,
   TableHead,
   TableHeader,
   TableRow,
@@ -243,7 +242,7 @@ export function DataTable<TData extends object, TValue>({
         <div className="grow"></div>
       </div>
       {pagination !== undefined ? (
-        <div className="sticky bottom-0 z-10 flex w-full justify-end bg-white font-medium">
+        <div className="bg:background sticky bottom-0 z-10 flex w-full justify-end font-medium">
           <DataTablePagination
             table={table}
             paginationOptions={pagination.options}

web/src/components/table/table-link.tsx

@@ -19,7 +19,8 @@ export default function TableLink({
       : value;
   return (
     <Link
-      className="inline-block rounded bg-indigo-50 px-2 py-1 text-xs font-semibold text-blue-600 shadow-sm hover:bg-indigo-100"
+      className="inline-block rounded bg-primary-accent/20 px-2
+       py-1 text-xs font-semibold text-accent-dark-blue shadow-sm hover:bg-accent-light-blue/45"
       href={path}
       title={value}
     >

web/src/components/table/use-cases/generations.tsx

@@ -21,11 +21,7 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import {
-  formatIntervalSeconds,
-  intervalInSeconds,
-  utcDateOffsetByDays,
-} from "@/src/utils/dates";
+import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import {
@@ -59,6 +55,7 @@ export type GenerationsTableRow = {
   endTime?: string;
   completionStartTime?: Date;
   latency?: number;
+  timeToFirstToken?: number;
   name?: string;
   model?: string;
   // i/o not set explicitly, but fetched from the server from the cell
@@ -118,7 +115,7 @@ export default function GenerationsTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],
@@ -310,21 +307,12 @@ export default function GenerationsTable({
       enableHiding: true,
       enableSorting: true,
       cell: ({ row }) => {
-        const startTime: Date = row.getValue("startTime");
-        const completionStartTime: Date | undefined =
+        const timeToFirstToken: number | undefined =
           row.getValue("timeToFirstToken");
 
-        if (!completionStartTime) {
-          return undefined;
-        }
-
-        const latencyInSeconds =
-          intervalInSeconds(startTime, completionStartTime) || "-";
         return (
           <span>
-            {typeof latencyInSeconds === "number"
-              ? formatIntervalSeconds(latencyInSeconds)
-              : latencyInSeconds}
+            {timeToFirstToken ? formatIntervalSeconds(timeToFirstToken) : "-"}
           </span>
         );
       },
@@ -623,7 +611,7 @@ export default function GenerationsTable({
           traceName: generation.traceName ?? "",
           startTime: generation.startTime,
           endTime: generation.endTime?.toLocaleString() ?? undefined,
-          timeToFirstToken: generation.completionStartTime ?? undefined,
+          timeToFirstToken: generation.timeToFirstToken ?? undefined,
           latency: generation.latency ?? undefined,
           totalCost: generation.calculatedTotalCost ?? undefined,
           inputCost: generation.calculatedInputCost ?? undefined,
@@ -758,7 +746,7 @@ const GenerationsIOCell = ({
       data={
         io === "output" ? observation.data?.output : observation.data?.input
       }
-      className={cn(io === "output" && "bg-green-50")}
+      className={cn(io === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/table/use-cases/models.tsx

@@ -111,7 +111,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         return (
           <>
             Input Price{" "}
-            <span className="text-xs text-gray-400">/ 1k units</span>
+            <span className="text-xs text-muted-foreground">/ 1k units</span>
           </>
         );
       },
@@ -140,7 +140,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         return (
           <>
             Output Price{" "}
-            <span className="text-xs text-gray-400">/ 1k units</span>
+            <span className="text-xs text-muted-foreground">/ 1k units</span>
           </>
         );
       },
@@ -163,7 +163,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         return (
           <>
             Total Price{" "}
-            <span className="text-xs text-gray-400">/ 1k units</span>
+            <span className="text-xs text-muted-foreground">/ 1k units</span>
           </>
         );
       },

web/src/components/table/use-cases/scores.tsx

@@ -12,13 +12,17 @@ import {
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
+import { utcDateOffsetByDays } from "@/src/utils/dates";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
+import type { FilterState } from "@langfuse/shared";
+import { useSession } from "next-auth/react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
 export type ScoresTableRow = {
   id: string;
   traceId: string;
   timestamp: string;
+  source: string;
   name: string;
   value: number;
   comment?: string;
@@ -33,36 +37,65 @@ export type ScoreFilterInput = Omit<
   "projectId" | "userId"
 >;
 
+function createFilterState(
+  userFilterState: FilterState,
+  omittedFilters: Record<string, string>[],
+): FilterState {
+  return omittedFilters.reduce((filterState, { key, value }) => {
+    return filterState.concat([
+      {
+        column: `${key}`,
+        type: "string",
+        operator: "=",
+        value: value,
+      },
+    ]);
+  }, userFilterState);
+}
+
 export default function ScoresTable({
   projectId,
   userId,
+  traceId,
+  observationId,
   omittedFilter = [],
+  hiddenColumns = [],
+  tableColumnVisibilityName = "scoresColumnVisibility",
 }: {
   projectId: string;
   userId?: string;
+  traceId?: string;
+  observationId?: string;
   omittedFilter?: string[];
+  hiddenColumns?: string[];
+  tableColumnVisibilityName?: string;
 }) {
   const [paginationState, setPaginationState] = useQueryParams({
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
   });
-
+  const session = useSession();
   const [rowHeight, setRowHeight] = useRowHeightLocalStorage("scores", "s");
 
   const [userFilterState, setUserFilterState] = useQueryFilterState(
-    [],
+    [
+      {
+        column: "Timestamp",
+        type: "datetime",
+        operator: ">",
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
+        ),
+      },
+    ],
     "scores",
   );
-  const filterState = userId
-    ? userFilterState.concat([
-        {
-          column: "User ID",
-          type: "string",
-          operator: "=",
-          value: userId,
-        },
-      ])
-    : userFilterState;
+
+  const filterState = createFilterState(userFilterState, [
+    ...(userId ? [{ key: "User ID", value: userId }] : []),
+    ...(traceId ? [{ key: "Trace ID", value: traceId }] : []),
+    ...(observationId ? [{ key: "Observation ID", value: observationId }] : []),
+  ]);
 
   const [orderByState, setOrderByState] = useOrderByState({
     column: "timestamp",
@@ -82,7 +115,7 @@ export default function ScoresTable({
     projectId,
   });
 
-  const columns: LangfuseColumnDef<ScoresTableRow>[] = [
+  const rawColumns: LangfuseColumnDef<ScoresTableRow>[] = [
     {
       accessorKey: "traceId",
       id: "traceId",
@@ -146,6 +179,13 @@ export default function ScoresTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "source",
+      header: "Source",
+      id: "source",
+      enableHiding: true,
+      enableSorting: true,
+    },
     {
       accessorKey: "name",
       header: "Name",
@@ -218,16 +258,18 @@ export default function ScoresTable({
       cell: ({ row }) => {
         const value = row.getValue("comment") as ScoresTableRow["comment"];
         return (
-          value !== undefined && (
-            <IOTableCell data={value} singleLine={rowHeight === "s"} />
-          )
+          !!value && <IOTableCell data={value} singleLine={rowHeight === "s"} />
         );
       },
     },
   ];
 
+  const columns = rawColumns.filter(
+    (c) => !!c.id && !hiddenColumns.includes(c.id),
+  );
+
   const [columnVisibility, setColumnVisibility] =
-    useColumnVisibility<ScoresTableRow>("scoresColumnVisibility", columns);
+    useColumnVisibility<ScoresTableRow>(tableColumnVisibilityName, columns);
 
   const convertToTableRow = (
     score: RouterOutput["scores"]["all"]["scores"][0],
@@ -235,6 +277,7 @@ export default function ScoresTable({
     return {
       id: score.id,
       timestamp: score.timestamp.toLocaleString(),
+      source: score.source,
       name: score.name,
       value: score.value,
       comment: score.comment ?? undefined,
@@ -250,7 +293,7 @@ export default function ScoresTable({
     traceFilterOptions: ScoreOptions | undefined,
   ) => {
     return scoresTableColsWithOptions(traceFilterOptions).filter(
-      (c) => !omittedFilter?.includes(c.name),
+      (c) => !omittedFilter?.includes(c.name) && !hiddenColumns.includes(c.id),
     );
   };
 

web/src/components/table/use-cases/sessions.tsx

@@ -55,7 +55,7 @@ export default function SessionsTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],

web/src/components/table/use-cases/traces.tsx

@@ -96,7 +96,7 @@ export default function TracesTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],
@@ -547,6 +547,7 @@ export default function TracesTable({
       header: "Version",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "release",
@@ -554,6 +555,7 @@ export default function TracesTable({
       header: "Release",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "tags",
@@ -690,7 +692,7 @@ const TracesIOCell = ({
     <IOTableCell
       isLoading={trace.isLoading}
       data={io === "output" ? trace.data?.output : trace.data?.input}
-      className={cn(io === "output" && "bg-green-50")}
+      className={cn(io === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/trace/IOPreview.tsx

@@ -112,7 +112,7 @@ export const IOPreview: React.FC<{
               title="Output"
               json={outputClean}
               isLoading={isLoading}
-              className="flex-1 bg-green-50"
+              className="flex-1 bg-accent-light-green dark:border-accent-dark-green"
             />
           ) : null}
         </>
@@ -128,12 +128,11 @@ const ChatMlMessageSchema = z
       .optional(),
     name: z.string().optional(),
     content: z
-      .union([z.record(z.any()), z.record(z.any()).array(), z.string()])
+      .union([z.record(z.any()), z.string(), z.array(z.any())])
       .nullish(),
     additional_kwargs: z.record(z.any()).optional(),
   })
   .passthrough()
-
   .refine((value) => value.content !== null || value.role !== undefined)
   .transform(({ additional_kwargs, ...other }) => ({
     ...other,
@@ -188,10 +187,11 @@ export const OpenAiMessageView: React.FC<{
                     title={message.name ?? message.role}
                     json={message.content}
                     className={cn(
-                      "bg-gray-100",
-                      message.role === "system" && "bg-gray-100",
-                      message.role === "assistant" && "bg-green-50",
-                      message.role === "user" && "bg-white",
+                      "bg-muted",
+                      message.role === "system" && "bg-primary-foreground",
+                      message.role === "assistant" &&
+                        "bg-accent-light-green dark:border-accent-dark-green",
+                      message.role === "user" && "bg-background",
                       !!message.json && "rounded-b-none",
                     )}
                   />
@@ -203,10 +203,11 @@ export const OpenAiMessageView: React.FC<{
                     }
                     json={message.json}
                     className={cn(
-                      "bg-gray-100",
-                      message.role === "system" && "bg-gray-100",
-                      message.role === "assistant" && "bg-green-50",
-                      message.role === "user" && "bg-white",
+                      "bg-muted",
+                      message.role === "system" && "bg-primary-foreground",
+                      message.role === "assistant" &&
+                        "bg-accent-light-green dark:border-accent-dark-green",
+                      message.role === "user" && "bg-foreground",
                       !!message.content && "rounded-t-none border-t-0",
                     )}
                   />

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Score } from "@langfuse/shared";
+import { type ScoreSource, type Score } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -8,14 +8,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { Badge } from "@/src/components/ui/badge";
-import {
-  Table,
-  TableBody,
-  TableCell,
-  TableHead,
-  TableHeader,
-  TableRow,
-} from "@/src/components/ui/table";
 import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -25,6 +17,11 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import Link from "next/link";
 import { usdFormatter } from "@/src/utils/numbers";
 import { calculateDisplayTotalCost } from "@/src/components/trace";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import { withDefault, StringParam, useQueryParam } from "use-query-params";
+import ScoresTable from "@/src/components/table/use-cases/scores";
+import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
+import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
@@ -33,6 +30,11 @@ export const ObservationPreview = (props: {
   currentObservationId: string;
   traceId: string;
 }) => {
+  const [selectedTab, setSelectedTab] = useQueryParam(
+    "view",
+    withDefault(StringParam, "preview"),
+  );
+
   const observationWithInputAndOutput = api.observations.byId.useQuery({
     observationId: props.currentObservationId,
     traceId: props.traceId,
@@ -47,156 +49,181 @@ export const ObservationPreview = (props: {
   );
 
   if (!preloadedObservation) return <div className="flex-1">Not found</div>;
-  return (
-    <Card className="flex-1">
-      <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
-        <div className="flex flex-col gap-1">
-          <CardTitle>
-            <span className="mr-2 rounded-sm bg-gray-200 p-1 text-xs">
-              {preloadedObservation.type}
-            </span>
-            <span>{preloadedObservation.name}</span>
-          </CardTitle>
-          <CardDescription className="flex gap-2">
-            {preloadedObservation.startTime.toLocaleString()}
-          </CardDescription>
-          <div className="flex flex-wrap gap-2">
-            {preloadedObservation.promptId ? (
-              <PromptBadge
-                promptId={preloadedObservation.promptId}
-                projectId={preloadedObservation.projectId}
-              />
-            ) : undefined}
-            {preloadedObservation.completionStartTime ? (
-              <Badge variant="outline">
-                Time to first token:{" "}
-                {formatIntervalSeconds(
-                  (preloadedObservation.completionStartTime.getTime() -
-                    preloadedObservation.startTime.getTime()) /
-                    1000,
-                )}
-              </Badge>
-            ) : null}
-            {preloadedObservation.endTime ? (
-              <Badge variant="outline">
-                Latency:{" "}
-                {formatIntervalSeconds(
-                  (preloadedObservation.endTime.getTime() -
-                    preloadedObservation.startTime.getTime()) /
-                    1000,
-                )}
-              </Badge>
-            ) : null}
-            {preloadedObservation.type === "GENERATION" && (
-              <Badge variant="outline">
-                {preloadedObservation.promptTokens} prompt →{" "}
-                {preloadedObservation.completionTokens} completion (∑{" "}
-                {preloadedObservation.totalTokens})
-              </Badge>
-            )}
-            {preloadedObservation.version ? (
-              <Badge variant="outline">
-                Version: {preloadedObservation.version}
-              </Badge>
-            ) : undefined}
-            {preloadedObservation.model ? (
-              <Badge variant="outline">{preloadedObservation.model}</Badge>
-            ) : null}
-            {totalCost ? (
-              <Badge variant="outline">
-                {usdFormatter(totalCost.toNumber())}
-              </Badge>
-            ) : undefined}
 
-            {preloadedObservation.modelParameters &&
-            typeof preloadedObservation.modelParameters === "object"
-              ? Object.entries(preloadedObservation.modelParameters)
-                  .filter(Boolean)
-                  .map(([key, value]) => (
-                    <Badge variant="outline" key={key}>
-                      {key}: {value?.toString()}
-                    </Badge>
-                  ))
-              : null}
+  const observationScores = props.scores.filter(
+    (s) => s.observationId === preloadedObservation.id,
+  );
+  const observationScoresBySource = observationScores.reduce((acc, score) => {
+    if (!acc.get(score.source)) {
+      acc.set(score.source, []);
+    }
+    acc.get(score.source)?.push(score);
+    return acc;
+  }, new Map<ScoreSource, Score[]>());
+
+  return (
+    <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
+      <div className="flex flex-shrink-0 flex-row justify-end gap-2">
+        <Tabs
+          value={selectedTab}
+          onValueChange={setSelectedTab}
+          className="flex w-full justify-end border-b bg-background"
+        >
+          <TabsList className="bg-background py-0">
+            <TabsTrigger
+              value="preview"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            >
+              Preview
+            </TabsTrigger>
+            <TabsTrigger
+              value="scores"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            >
+              Scores
+            </TabsTrigger>
+          </TabsList>
+        </Tabs>
+      </div>
+      <div className="flex w-full flex-col overflow-y-auto">
+        <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
+          <div className="flex flex-col gap-1">
+            <CardTitle>
+              <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+                {preloadedObservation.type}
+              </span>
+              <span>{preloadedObservation.name}</span>
+            </CardTitle>
+            <CardDescription className="flex gap-2">
+              {preloadedObservation.startTime.toLocaleString()}
+            </CardDescription>
+            <div className="flex flex-wrap gap-2">
+              {preloadedObservation.promptId ? (
+                <PromptBadge
+                  promptId={preloadedObservation.promptId}
+                  projectId={preloadedObservation.projectId}
+                />
+              ) : undefined}
+              {preloadedObservation.timeToFirstToken ? (
+                <Badge variant="outline">
+                  Time to first token:{" "}
+                  {formatIntervalSeconds(preloadedObservation.timeToFirstToken)}
+                </Badge>
+              ) : null}
+              {preloadedObservation.endTime ? (
+                <Badge variant="outline">
+                  Latency:{" "}
+                  {formatIntervalSeconds(
+                    (preloadedObservation.endTime.getTime() -
+                      preloadedObservation.startTime.getTime()) /
+                      1000,
+                  )}
+                </Badge>
+              ) : null}
+              {preloadedObservation.type === "GENERATION" && (
+                <Badge variant="outline">
+                  {preloadedObservation.promptTokens} prompt →{" "}
+                  {preloadedObservation.completionTokens} completion (∑{" "}
+                  {preloadedObservation.totalTokens})
+                </Badge>
+              )}
+              {preloadedObservation.version ? (
+                <Badge variant="outline">
+                  Version: {preloadedObservation.version}
+                </Badge>
+              ) : undefined}
+              {preloadedObservation.model ? (
+                <Badge variant="outline">{preloadedObservation.model}</Badge>
+              ) : null}
+              {totalCost ? (
+                <Badge variant="outline">
+                  {usdFormatter(totalCost.toNumber())}
+                </Badge>
+              ) : undefined}
+
+              {preloadedObservation.modelParameters &&
+              typeof preloadedObservation.modelParameters === "object"
+                ? Object.entries(preloadedObservation.modelParameters)
+                    .filter(Boolean)
+                    .map(([key, value]) => (
+                      <Badge variant="outline" key={key}>
+                        {key}: {value?.toString()}
+                      </Badge>
+                    ))
+                : null}
+            </div>
           </div>
-        </div>
-        <div className="flex gap-2">
-          <ManualScoreButton
-            projectId={props.projectId}
-            traceId={preloadedObservation.traceId}
-            observationId={preloadedObservation.id}
-            scores={props.scores}
-          />
-          {observationWithInputAndOutput.data ? (
-            <NewDatasetItemFromTrace
+          <div className="flex flex-wrap gap-2">
+            <ManualScoreButton
+              projectId={props.projectId}
               traceId={preloadedObservation.traceId}
               observationId={preloadedObservation.id}
-              projectId={props.projectId}
-              input={observationWithInputAndOutput.data.input}
-              output={observationWithInputAndOutput.data.output}
-              metadata={preloadedObservation.metadata}
-              key={preloadedObservation.id}
+              scores={props.scores}
             />
-          ) : null}
-        </div>
-      </CardHeader>
-      <CardContent className="flex flex-col gap-4">
-        <IOPreview
-          key={preloadedObservation.id + "-input"}
-          input={observationWithInputAndOutput.data?.input ?? undefined}
-          output={observationWithInputAndOutput.data?.output ?? undefined}
-          isLoading={observationWithInputAndOutput.isLoading}
-        />
-        {preloadedObservation.statusMessage ? (
-          <JSONView
-            key={preloadedObservation.id + "-status"}
-            title="Status Message"
-            json={preloadedObservation.statusMessage}
-          />
-        ) : null}
-
-        {observationWithInputAndOutput.data?.metadata ? (
-          <JSONView
-            key={observationWithInputAndOutput.data.id + "-metadata"}
-            title="Metadata"
-            json={observationWithInputAndOutput.data.metadata}
-          />
-        ) : null}
-
-        {props.scores.find(
-          (s) => s.observationId === preloadedObservation.id,
-        ) ? (
-          <div className="flex flex-col gap-2">
-            <h3>Scores</h3>
-            <Table>
-              <TableHeader>
-                <TableRow>
-                  <TableHead className="w-[100px]">Timestamp</TableHead>
-                  <TableHead>Name</TableHead>
-                  <TableHead className="text-right">Value</TableHead>
-                  <TableHead>Comment</TableHead>
-                </TableRow>
-              </TableHeader>
-              <TableBody>
-                {props.scores
-                  .filter((s) => s.observationId === preloadedObservation.id)
-                  .map((s) => (
-                    <TableRow key={s.id}>
-                      <TableCell className="text-xs">
-                        {s.timestamp.toLocaleString()}
-                      </TableCell>
-                      <TableCell className="text-xs">{s.name}</TableCell>
-                      <TableCell className="text-right text-xs">
-                        {s.value}
-                      </TableCell>
-                      <TableCell className="text-xs">{s.comment}</TableCell>
-                    </TableRow>
-                  ))}
-              </TableBody>
-            </Table>
+            {observationWithInputAndOutput.data?.type === "GENERATION" && (
+              <JumpToPlaygroundButton
+                source="generation"
+                generation={observationWithInputAndOutput.data}
+                analyticsEventName="trace_detail:test_in_playground_button_click"
+                fullWidth
+              />
+            )}
+            {observationWithInputAndOutput.data ? (
+              <NewDatasetItemFromTrace
+                traceId={preloadedObservation.traceId}
+                observationId={preloadedObservation.id}
+                projectId={props.projectId}
+                input={observationWithInputAndOutput.data.input}
+                output={observationWithInputAndOutput.data.output}
+                metadata={preloadedObservation.metadata}
+                key={preloadedObservation.id}
+              />
+            ) : null}
           </div>
-        ) : null}
-      </CardContent>
+        </CardHeader>
+        <CardContent className="flex flex-col gap-4">
+          {selectedTab === "preview" && (
+            <>
+              <IOPreview
+                key={preloadedObservation.id + "-input"}
+                input={observationWithInputAndOutput.data?.input ?? undefined}
+                output={observationWithInputAndOutput.data?.output ?? undefined}
+                isLoading={observationWithInputAndOutput.isLoading}
+              />
+              {preloadedObservation.statusMessage ? (
+                <JSONView
+                  key={preloadedObservation.id + "-status"}
+                  title="Status Message"
+                  json={preloadedObservation.statusMessage}
+                />
+              ) : null}
+              {observationWithInputAndOutput.data?.metadata ? (
+                <JSONView
+                  key={observationWithInputAndOutput.data.id + "-metadata"}
+                  title="Metadata"
+                  json={observationWithInputAndOutput.data.metadata}
+                />
+              ) : null}
+              <ScoresPreview itemScoresBySource={observationScoresBySource} />
+            </>
+          )}
+          {selectedTab === "scores" && (
+            <ScoresTable
+              projectId={props.projectId}
+              omittedFilter={["Observation ID"]}
+              observationId={preloadedObservation.id}
+              hiddenColumns={[
+                "traceId",
+                "observationId",
+                "traceName",
+                "jobConfigurationId",
+                "userId",
+              ]}
+              tableColumnVisibilityName="scoresColumnVisibilityObservationPreview"
+            />
+          )}
+        </CardContent>
+      </div>
     </Card>
   );
 };

web/src/components/trace/ObservationTree.tsx

@@ -68,13 +68,13 @@ const ObservationTreeTraceNode = (props: {
       "group mb-0.5 flex cursor-pointer flex-col gap-1 rounded-sm p-1",
       props.currentObservationId === undefined ||
         props.currentObservationId === ""
-        ? "bg-gray-100"
-        : "hover:bg-gray-50",
+        ? "bg-muted"
+        : "hover:bg-primary-foreground",
     )}
     onClick={() => props.setCurrentObservationId(undefined)}
   >
     <div className="flex gap-2">
-      <span className={cn("rounded-sm bg-gray-200 p-1 text-xs")}>TRACE</span>
+      <span className={cn("rounded-sm bg-input p-1 text-xs")}>TRACE</span>
       <span className="flex-1 break-all text-sm">{props.trace.name}</span>
       <Button
         onClick={(ev) => (ev.stopPropagation(), props.expandAll())}
@@ -96,7 +96,7 @@ const ObservationTreeTraceNode = (props: {
 
     {props.showMetrics && props.trace.latency ? (
       <div className="flex gap-2">
-        <span className="text-xs text-gray-500">
+        <span className="text-xs text-muted-foreground">
           {formatIntervalSeconds(props.trace.latency)}
         </span>
       </div>
@@ -141,8 +141,8 @@ const ObservationTreeNode = (props: {
                   className={cn(
                     "group my-0.5 flex flex-1 cursor-pointer flex-col gap-1 rounded-sm p-1",
                     props.currentObservationId === observation.id
-                      ? "bg-gray-100"
-                      : "hover:bg-gray-50",
+                      ? "bg-muted"
+                      : "hover:bg-primary-foreground",
                   )}
                   onClick={() => props.setCurrentObservationId(observation.id)}
                 >
@@ -188,7 +188,7 @@ const ObservationTreeNode = (props: {
                       observation.endTime) && (
                       <div className="flex gap-2">
                         {observation.endTime ? (
-                          <span className="text-xs text-gray-500">
+                          <span className="text-xs text-muted-foreground">
                             {formatIntervalSeconds(
                               (observation.endTime.getTime() -
                                 observation.startTime.getTime()) /
@@ -199,7 +199,7 @@ const ObservationTreeNode = (props: {
                         {observation.promptTokens ||
                         observation.completionTokens ||
                         observation.totalTokens ? (
-                          <span className="text-xs text-gray-500">
+                          <span className="text-xs text-muted-foreground">
                             {observation.promptTokens} →{" "}
                             {observation.completionTokens} (∑{" "}
                             {observation.totalTokens})
@@ -258,9 +258,9 @@ const ColorCodedObservationType = (props: {
   observationType: $Enums.ObservationType;
 }) => {
   const colors: Record<$Enums.ObservationType, string> = {
-    [$Enums.ObservationType.SPAN]: "bg-blue-100",
-    [$Enums.ObservationType.GENERATION]: "bg-orange-100",
-    [$Enums.ObservationType.EVENT]: "bg-green-100",
+    [$Enums.ObservationType.SPAN]: "bg-muted-blue",
+    [$Enums.ObservationType.GENERATION]: "bg-muted-orange",
+    [$Enums.ObservationType.EVENT]: "bg-muted-green",
   };
 
   return (

web/src/components/trace/ScoresPreview.tsx

@@ -0,0 +1,29 @@
+import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
+import { type Score } from "@langfuse/shared";
+
+export const ScoresPreview = ({
+  itemScoresBySource,
+}: {
+  itemScoresBySource: Map<string, Score[]>;
+}) => {
+  if (!Boolean(itemScoresBySource.size)) return null;
+
+  return (
+    <div className="flex flex-col rounded-md border">
+      <span className="border-b px-3 py-1 text-xs font-medium">Scores</span>
+      <div
+        key={itemScoresBySource.size}
+        className="grid grid-flow-row gap-2 overflow-x-auto px-3 pb-3 pt-1"
+      >
+        {Array.from(itemScoresBySource).map(([source, scores]) => (
+          <div key={source} className="flex flex-col align-middle text-xs">
+            <span className="min-w-16 p-1 font-medium">{source}</span>
+            <div className="flex flex-col content-start items-start gap-1 text-nowrap">
+              <GroupedScoreBadges scores={scores} />
+            </div>
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+};

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Trace, type Score } from "@langfuse/shared";
+import { type Trace, type Score, type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -8,14 +8,6 @@ import {
   CardHeader,
   CardTitle,
 } from "@/src/components/ui/card";
-import {
-  Table,
-  TableBody,
-  TableCell,
-  TableHead,
-  TableHeader,
-  TableRow,
-} from "@/src/components/ui/table";
 import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
 import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { Badge } from "@/src/components/ui/badge";
@@ -23,6 +15,10 @@ import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
 import { formatIntervalSeconds } from "@/src/utils/dates";
 import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import { withDefault, StringParam, useQueryParam } from "use-query-params";
+import ScoresTable from "@/src/components/table/use-cases/scores";
+import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 
 export const TracePreview = ({
   trace,
@@ -33,92 +29,115 @@ export const TracePreview = ({
   observations: ObservationReturnType[];
   scores: Score[];
 }) => {
+  const [selectedTab, setSelectedTab] = useQueryParam(
+    "view",
+    withDefault(StringParam, "preview"),
+  );
+
+  const traceScores = scores.filter((s) => s.observationId === null);
+  const traceScoresBySource = traceScores.reduce((acc, score) => {
+    if (!acc.get(score.source)) {
+      acc.set(score.source, []);
+    }
+    acc.get(score.source)?.push(score);
+    return acc;
+  }, new Map<ScoreSource, Score[]>());
+
   return (
-    <Card className="flex-1">
-      <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
-        <div className="flex flex-col gap-1">
-          <CardTitle>
-            <span className="mr-2 rounded-sm bg-gray-200 p-1 text-xs">
-              TRACE
-            </span>
-            <span>{trace.name}</span>
-          </CardTitle>
-          <CardDescription>{trace.timestamp.toLocaleString()}</CardDescription>
+    <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
+      <div className="flex flex-shrink-0 flex-row justify-end gap-2">
+        <Tabs
+          value={selectedTab}
+          onValueChange={setSelectedTab}
+          className="flex w-full justify-end border-b bg-background"
+        >
+          <TabsList className="bg-background py-0">
+            <TabsTrigger
+              value="preview"
+              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+            >
+              Preview
+            </TabsTrigger>
+            <TabsTrigger
+              value="scores"
+              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+            >
+              Scores
+            </TabsTrigger>
+          </TabsList>
+        </Tabs>
+      </div>
+      <div className="flex w-full flex-col overflow-y-auto">
+        <CardHeader className="flex flex-row flex-wrap justify-between gap-2">
+          <div className="flex flex-col gap-1">
+            <CardTitle>
+              <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+                TRACE
+              </span>
+              <span>{trace.name}</span>
+            </CardTitle>
+            <CardDescription>
+              {trace.timestamp.toLocaleString()}
+            </CardDescription>
+            <div className="flex flex-wrap gap-2">
+              {!!trace.latency && (
+                <Badge variant="outline">
+                  {formatIntervalSeconds(trace.latency)}
+                </Badge>
+              )}
+              <TraceAggUsageBadge observations={observations} />
+              {!!trace.release && (
+                <Badge variant="outline">Release: {trace.release}</Badge>
+              )}
+              {!!trace.version && (
+                <Badge variant="outline">Version: {trace.version}</Badge>
+              )}
+            </div>
+          </div>
           <div className="flex flex-wrap gap-2">
-            {!!trace.latency && (
-              <Badge variant="outline">
-                {formatIntervalSeconds(trace.latency)}
-              </Badge>
-            )}
-            <TraceAggUsageBadge observations={observations} />
-            {!!trace.release && (
-              <Badge variant="outline">Release: {trace.release}</Badge>
-            )}
-            {!!trace.version && (
-              <Badge variant="outline">Version: {trace.version}</Badge>
-            )}
+            <ManualScoreButton
+              projectId={trace.projectId}
+              traceId={trace.id}
+              scores={scores}
+            />
+            <NewDatasetItemFromTrace
+              traceId={trace.id}
+              projectId={trace.projectId}
+              input={trace.input}
+              output={trace.output}
+              metadata={trace.metadata}
+              key={trace.id}
+            />
           </div>
-        </div>
-        <div className="flex gap-2">
-          <ManualScoreButton
-            projectId={trace.projectId}
-            traceId={trace.id}
-            scores={scores}
-          />
-          <NewDatasetItemFromTrace
-            traceId={trace.id}
-            projectId={trace.projectId}
-            input={trace.input}
-            output={trace.output}
-            metadata={trace.metadata}
-            key={trace.id}
-          />
-        </div>
-      </CardHeader>
-      <CardContent className="flex flex-col gap-4">
-        <IOPreview
-          key={trace.id + "-io"}
-          input={trace.input ?? undefined}
-          output={trace.output ?? undefined}
-        />
-        <JSONView
-          key={trace.id + "-metadata"}
-          title="Metadata"
-          json={trace.metadata}
-        />
-        {scores.find((s) => s.observationId === null) ? (
-          <div className="mt-5 flex flex-col gap-2">
-            <h3>Scores</h3>
-            <Table>
-              <TableHeader>
-                <TableRow>
-                  <TableHead className="w-[100px]">Timestamp</TableHead>
-                  <TableHead>Name</TableHead>
-                  <TableHead className="text-right">Value</TableHead>
-                  <TableHead>Comment</TableHead>
-                </TableRow>
-              </TableHeader>
-              <TableBody>
-                {scores
-                  .filter((s) => s.observationId === null)
-                  .map((s) => (
-                    <TableRow key={s.id}>
-                      <TableCell className="text-xs">
-                        {s.timestamp.toLocaleString()}
-                      </TableCell>
-                      <TableCell className="text-xs">{s.name}</TableCell>
-                      <TableCell className="text-right text-xs">
-                        {s.value}
-                      </TableCell>
-                      <TableCell className="text-xs">{s.comment}</TableCell>
-                    </TableRow>
-                  ))}
-              </TableBody>
-            </Table>
-          </div>
-        ) : null}
-      </CardContent>
-      <CardFooter></CardFooter>
+        </CardHeader>
+        <CardContent className="flex flex-col gap-4">
+          {selectedTab === "preview" && (
+            <>
+              <IOPreview
+                key={trace.id + "-io"}
+                input={trace.input ?? undefined}
+                output={trace.output ?? undefined}
+              />
+              <JSONView
+                key={trace.id + "-metadata"}
+                title="Metadata"
+                json={trace.metadata}
+              />
+              <ScoresPreview itemScoresBySource={traceScoresBySource} />
+            </>
+          )}
+          {selectedTab === "scores" && (
+            <ScoresTable
+              projectId={trace.projectId}
+              omittedFilter={["Trace ID"]}
+              traceId={trace.id}
+              hiddenColumns={["traceName", "jobConfigurationId", "userId"]}
+              tableColumnVisibilityName="scoresColumnVisibilityTracePreview"
+            />
+          )}
+        </CardContent>
+        <CardFooter></CardFooter>
+      </div>
     </Card>
   );
 };

web/src/components/trace/index.tsx

@@ -277,6 +277,7 @@ export function TracePage({ traceId }: { traceId: string }) {
             availableTags={allTags}
             traceId={trace.data.id}
             projectId={trace.data.projectId}
+            className="flex-wrap"
           />
         </div>
       </div>

web/src/components/ui/CodeJsonViewer.tsx

@@ -3,8 +3,10 @@ import { Button } from "@/src/components/ui/button";
 import { Check, ChevronsDownUp, ChevronsUpDown, Copy } from "lucide-react";
 import { cn } from "@/src/utils/tailwind";
 import { default as React18JsonView } from "react18-json-view";
+import "react18-json-view/src/dark.css";
 import { deepParseJson } from "@/src/utils/json";
 import { Skeleton } from "@/src/components/ui/skeleton";
+import { useTheme } from "next-themes";
 
 export function JSONView(props: {
   json?: unknown;
@@ -15,11 +17,18 @@ export function JSONView(props: {
 }) {
   // some users ingest stringified json nested in json, parse it
   const parsedJson = deepParseJson(props.json);
-
+  const { resolvedTheme } = useTheme();
   return (
     <div className={cn("rounded-md border", props.className)}>
       {props.title ? (
-        <div className="border-b px-3 py-1 text-xs font-medium">
+        <div
+          className={cn(
+            props.title === "assistant" || props.title === "Output"
+              ? "dark:border-accent-dark-green"
+              : "",
+            "border-b px-3 py-1 text-xs font-medium",
+          )}
+        >
           {props.title}
         </div>
       ) : undefined}
@@ -35,6 +44,7 @@ export function JSONView(props: {
           <React18JsonView
             src={parsedJson}
             theme="github"
+            dark={resolvedTheme === "dark"}
             collapseObjectsAfterLength={20}
             collapseStringsAfterLength={500}
             displaySize={"collapsed"}

web/src/components/ui/button.tsx

@@ -19,6 +19,8 @@ const buttonVariants = cva(
           "bg-secondary text-secondary-foreground hover:bg-secondary/80",
         ghost: "hover:bg-accent hover:text-accent-foreground",
         link: "text-primary underline-offset-4 hover:underline",
+        errorNotification:
+          "bg-destructive-foreground/90 text-destructive hover:bg-destructive-foreground/80",
       },
       size: {
         default: "h-10 px-4 py-2",

web/src/components/ui/command.tsx

@@ -42,7 +42,7 @@ const CommandInput = React.forwardRef<
   React.ComponentPropsWithoutRef<typeof CommandPrimitive.Input>
 >(({ className, ...props }, ref) => (
   <div
-    className="flex items-center rounded-lg border px-3"
+    className="flex items-center rounded-md border px-3"
     cmdk-input-wrapper=""
   >
     <Search className="mr-2 h-4 w-4 shrink-0 opacity-50" />

web/src/components/ui/description-lists.tsx

@@ -1,50 +0,0 @@
-// https://tailwindui.com/components/application-ui/data-display/description-lists
-
-import clsx from "clsx";
-import { type ReactNode } from "react";
-
-export default function DescriptionList(props: {
-  header?: {
-    title: string;
-    description: string;
-  };
-  items: { label: string; value: string | ReactNode }[];
-  descriptionColumns?: number;
-  valueColumns?: number;
-}) {
-  const { descriptionColumns = 1, valueColumns = 2 } = props;
-  const totalColumns = descriptionColumns + valueColumns;
-  return (
-    <div>
-      {props.header ? (
-        <div className="px-4 sm:px-0">
-          <h3 className="text-base font-semibold leading-7 text-gray-900">
-            {props.header.title}
-          </h3>
-          <p className="mt-1 max-w-2xl text-sm leading-6 text-gray-500">
-            {props.header.description}
-          </p>
-        </div>
-      ) : null}
-      <div className={clsx(props.header && "mt-6 border-t border-gray-100")}>
-        <dl className="divide-y divide-gray-100">
-          {props.items.map((item) => (
-            <div
-              key={item.label}
-              className={`sm:grid-cols-${totalColumns} px-4 py-3 sm:grid sm:gap-4 sm:px-0`}
-            >
-              <dt className="text-sm font-medium leading-6 text-gray-900">
-                {item.label}
-              </dt>
-              <dd
-                className={`mt-1 text-sm leading-6 text-gray-700 sm:col-span-${valueColumns} sm:mt-0`}
-              >
-                {item.value}
-              </dd>
-            </div>
-          ))}
-        </dl>
-      </div>
-    </div>
-  );
-}

web/src/components/ui/dialog.tsx

@@ -21,7 +21,7 @@ const DialogOverlay = React.forwardRef<
   <DialogPrimitive.Overlay
     ref={ref}
     className={cn(
-      "fixed inset-0 z-50 bg-black/80  data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
+      "fixed inset-0 z-50 bg-foreground/40 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
       className,
     )}
     {...props}

web/src/components/ui/input.tsx

@@ -11,7 +11,7 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
       <input
         type={type}
         className={cn(
-          "flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+          "flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus:ring-0 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50",
           className,
         )}
         ref={ref}

web/src/components/ui/password-input.tsx

@@ -27,9 +27,9 @@ const PasswordInput = React.forwardRef<HTMLInputElement, PasswordInputProps>(
           title={showPassword ? "Hide password" : "Show password"}
         >
           {showPassword ? (
-            <EyeOff className="h-5 w-5 text-gray-400" />
+            <EyeOff className="h-5 w-5 text-muted-foreground" />
           ) : (
-            <Eye className="h-5 w-5 text-gray-400" />
+            <Eye className="h-5 w-5 text-muted-foreground" />
           )}
         </button>
       </div>