chore: release v2.49.0

Co-authored-by: Marc Klingen <git@marcklingen.com>

CONTRIBUTING.md

@@ -191,6 +191,16 @@ Requirements
     pnpm run dev
    ```
 
+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -358,7 +368,9 @@ The background color of the following component will be `hsl(var(--primary))` an
 
 By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
 
-## Using secrets stored in 1Password
+## Maintainers
+
+### Using secrets stored in 1Password
 
 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
 

ee/package.json

@@ -33,7 +33,7 @@
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

fern/apis/server/definition/metrics.yml

@@ -48,12 +48,25 @@ types:
       date: date
       countTraces: integer
       countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
       model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD

fern/apis/server/definition/prompts.yml

@@ -70,14 +70,24 @@ types:
       name: string
       prompt: list<ChatMessage>
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   CreateTextPromptRequest:
     properties:
       name: string
       prompt: string
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   Prompt:
     union:
@@ -89,7 +99,12 @@ types:
       name: string
       version: integer
       config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
 
   ChatMessage:
     properties:

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
       path: /sessions/{sessionId}
       path-parameters:

fern/apis/server/definition/trace.yml

@@ -30,6 +30,7 @@ service:
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only traces newer than this datetime (ISO 8601).

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.47.2",
+  "version": "2.49.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/config-eslint/package.json

@@ -9,7 +9,7 @@
   ],
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",

packages/shared/package.json

@@ -63,7 +63,7 @@
     "@types/node": "^20.11.29",
     "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

packages/shared/prisma/generated/types.ts

@@ -32,20 +32,10 @@ export const ScoreSource = {
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const ScoreDataType = {
     CATEGORICAL: "CATEGORICAL",
-    NUMERIC: "NUMERIC"
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
-} as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -309,14 +299,6 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -470,7 +452,6 @@ export type DB = {
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;

packages/shared/prisma/migrations/20240603212024_dataset_items_add_index_source_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");

packages/shared/prisma/migrations/20240604133338_scores_add_index_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");

packages/shared/prisma/migrations/20240604133339_score_data_type_add_boolean/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';

packages/shared/prisma/migrations/20240604133340_backfill_manual_scores/migration.sql

@@ -0,0 +1,50 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	gen_random_uuid () AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,5 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240606093356_drop_unused_pricings_table/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";

packages/shared/prisma/migrations/20240606133011_remove_trace_fkey_datasetrunitems/migration.sql

@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";

packages/shared/prisma/schema.prisma

@@ -233,9 +233,8 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetRunItems DatasetRunItems[]
-    DatasetItem     DatasetItem[]
-    JobExecution    JobExecution[]
+    DatasetItem  DatasetItem[]
+    JobExecution JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -298,21 +297,20 @@ model Observation {
 
     // GENERATION ONLY
     model               String?
-    internalModel       String?           @map("internal_model")
+    internalModel       String?       @map("internal_model")
     modelParameters     Json?
     input               Json?
     output              Json?
-    promptTokens        Int               @default(0) @map("prompt_tokens")
-    completionTokens    Int               @default(0) @map("completion_tokens")
-    totalTokens         Int               @default(0) @map("total_tokens")
+    promptTokens        Int           @default(0) @map("prompt_tokens")
+    completionTokens    Int           @default(0) @map("completion_tokens")
+    totalTokens         Int           @default(0) @map("total_tokens")
     unit                String?
-    inputCost           Decimal?          @map("input_cost")
-    outputCost          Decimal?          @map("output_cost")
-    totalCost           Decimal?          @map("total_cost")
-    completionStartTime DateTime?         @map("completion_start_time")
-    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    inputCost           Decimal?      @map("input_cost")
+    outputCost          Decimal?      @map("output_cost")
+    totalCost           Decimal?      @map("total_cost")
+    completionStartTime DateTime?     @map("completion_start_time")
+    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
-    datasetRunItems     DatasetRunItems[]
 
     promptId String? @map("prompt_id")
     prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@ -419,6 +417,7 @@ model Score {
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
+    @@index([projectId, name])
     @@index([authorUserId])
     @@index([configId])
     @@index([traceId], type: Hash)
@@ -463,29 +462,7 @@ model ScoreConfig {
 enum ScoreDataType {
     CATEGORICAL
     NUMERIC
-}
-
-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
-}
-
-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
+    BOOLEAN
 }
 
 model CronJobs {
@@ -532,6 +509,7 @@ model DatasetItem {
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
     @@index([createdAt])
@@ -563,17 +541,15 @@ model DatasetRuns {
 }
 
 model DatasetRunItems {
-    id            String       @id @default(cuid())
-    datasetRunId  String       @map("dataset_run_id")
-    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String       @map("dataset_item_id")
-    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime     @default(now()) @map("created_at")
-    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String      @map("trace_id")
+    observationId String?     @map("observation_id")
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)

packages/shared/prisma/seed.ts

@@ -4,6 +4,7 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
+  ScoreDataType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -39,6 +40,7 @@ async function main() {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
 
@@ -144,6 +146,11 @@ async function main() {
       });
     }
 
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -157,7 +164,8 @@ async function main() {
       colorTags,
       project1,
       project2,
-      promptIds
+      promptIds,
+      configIdsAndNames
     );
 
     console.log(
@@ -453,13 +461,15 @@ function createObjects(
   colorTags: (string | null)[],
   project1: Project,
   project2: Project,
-  promptIds: Map<string, string[]>
+  promptIds: Map<string, string[]>,
+  configIdsAndNames: Map<string, { name: string; id: string }[]>
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
   const scores: Prisma.ScoreCreateManyInput[] = [];
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
+  const configs: Prisma.ScoreConfigCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -511,17 +521,28 @@ function createObjects(
 
     traces.push(trace);
 
+    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const randomIndex = Math.floor(Math.random() * 3);
+    const config =
+      configArray.length >= randomIndex - 1 && configArray[randomIndex];
+    const { name: annotationScoreName, id: configId } = config || {
+      name: "manual-score",
+      id: undefined,
+    };
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
             {
               traceId: trace.id,
-              name: "manual-score",
+              name: annotationScoreName,
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
+              dataType: ScoreDataType.NUMERIC,
+              ...(configId ? { configId } : {}),
             },
           ]
         : []),
@@ -534,6 +555,7 @@ function createObjects(
               timestamp: traceTs,
               source: ScoreSource.API,
               projectId,
+              dataType: ScoreDataType.NUMERIC,
             },
           ]
         : []),
@@ -772,6 +794,7 @@ function createObjects(
     traces,
     observations,
     scores,
+    configs,
     sessions: uniqueSessions,
     events,
   };
@@ -954,3 +977,80 @@ async function generatePrompts(project: Project) {
   }
   return promptIds;
 }
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
+    new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    })
+  );
+  return projectIdsToConfigs;
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: { name: string; id: string }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({ name: config.name, id: config.id });
+  }
+
+  return configNameAndId;
+}

packages/shared/src/filterToPrisma.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import {
   ColumnDefinition,
-  type TableNames as TableName,
+  type TableNames,
 } from "./interfaces/tableDefinition";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,7 +43,7 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName
+  table: TableNames
 ) => {
   return column.name === "type" &&
     (table === "observations" ||

packages/shared/src/interfaces/tableDefinition.ts

@@ -42,6 +42,7 @@ export const tableNames = [
   "traces_parent_observation_scores",
   "sessions",
   "prompts",
+  "users",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/types.ts

@@ -30,4 +30,5 @@ export type TableName =
   | "sessions"
   | "scores"
   | "prompts"
-  | "dashboard";
+  | "dashboard"
+  | "users";

pnpm-lock.yaml

@@ -49,8 +49,8 @@ importers:
         specifier: ^20.11.29
         version: 20.11.29
       '@typescript-eslint/parser':
-        specifier: ^7.7.0
-        version: 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+        specifier: ^7.12.0
+        version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint:
         specifier: ^8.57.0
         version: 8.57.0
@@ -80,10 +80,10 @@ importers:
     devDependencies:
       '@typescript-eslint/eslint-plugin':
         specifier: ^7.1.0
-        version: 7.3.1(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5)
+        version: 7.3.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/parser':
-        specifier: ^7.7.0
-        version: 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+        specifier: ^7.12.0
+        version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       '@vercel/style-guide':
         specifier: ^6.0.0
         version: 6.0.0(eslint@8.57.0)(prettier@3.2.5)(typescript@5.4.5)
@@ -160,8 +160,8 @@ importers:
         specifier: ^9.0.8
         version: 9.0.8
       '@typescript-eslint/parser':
-        specifier: ^7.7.0
-        version: 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+        specifier: ^7.12.0
+        version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint:
         specifier: ^8.57.0
         version: 8.57.0
@@ -315,6 +315,9 @@ importers:
       '@radix-ui/react-toggle':
         specifier: ^1.0.3
         version: 1.0.3(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
+      '@radix-ui/react-toggle-group':
+        specifier: ^1.0.4
+        version: 1.0.4(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
       '@radix-ui/react-tooltip':
         specifier: ^1.0.7
         version: 1.0.7(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
@@ -357,9 +360,6 @@ importers:
       '@tanstack/react-table':
         specifier: ^8.11.8
         version: 8.13.2(react-dom@18.2.0)(react@18.2.0)
-      '@tanstack/react-virtual':
-        specifier: ^3.5.0
-        version: 3.5.0(react-dom@18.2.0)(react@18.2.0)
       '@tremor/react':
         specifier: 3.16.2
         version: 3.16.2(react-dom@18.2.0)(react@18.2.0)(tailwindcss@3.4.3)
@@ -434,7 +434,7 @@ importers:
         version: 0.364.0(react@18.2.0)
       next:
         specifier: ^14.2.3
-        version: 14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
+        version: 14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
       next-auth:
         specifier: ^4.24.7
         version: 4.24.7(next@14.2.3)(nodemailer@6.9.13)(react-dom@18.2.0)(react@18.2.0)
@@ -495,6 +495,9 @@ importers:
       uuid:
         specifier: ^9.0.1
         version: 9.0.1
+      vaul:
+        specifier: ^0.9.1
+        version: 0.9.1(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
       zod:
         specifier: ^3.22.4
         version: 3.22.4
@@ -556,10 +559,10 @@ importers:
         version: 9.0.8
       '@typescript-eslint/eslint-plugin':
         specifier: ^6.21.0
-        version: 6.21.0(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5)
+        version: 6.21.0(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/parser':
-        specifier: ^7.7.0
-        version: 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+        specifier: ^7.12.0
+        version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       autoprefixer:
         specifier: ^10.4.19
         version: 10.4.19(postcss@8.4.38)
@@ -697,8 +700,8 @@ importers:
         specifier: ^8.11.6
         version: 8.11.6
       '@typescript-eslint/parser':
-        specifier: ^7.7.0
-        version: 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+        specifier: ^7.12.0
+        version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint:
         specifier: ^8.57.0
         version: 8.57.0
@@ -1550,13 +1553,12 @@ packages:
       '@babel/traverse': 7.24.1
       '@babel/types': 7.24.0
       convert-source-map: 2.0.0
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       gensync: 1.0.0-beta.2
       json5: 2.2.3
       semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
-    dev: true
 
   /@babel/core@7.24.5:
     resolution: {integrity: sha512-tVQRucExLQ02Boi4vdPp49svNGcfL2GhdTCT9aldhXgCJVAI21EtRfBettiuLUwce/7r6bFdgs6JFkcdTiFttA==}
@@ -1603,7 +1605,6 @@ packages:
       '@jridgewell/gen-mapping': 0.3.5
       '@jridgewell/trace-mapping': 0.3.25
       jsesc: 2.5.2
-    dev: true
 
   /@babel/generator@7.24.5:
     resolution: {integrity: sha512-x32i4hEXvr+iI0NEoEfDKzlemF8AmtOP8CcrRaEcpzysWuoEb1KknpcvMsHKPONoKZiDuItklgWhB18xEhr9PA==}
@@ -1647,7 +1648,6 @@ packages:
     engines: {node: '>=6.9.0'}
     dependencies:
       '@babel/types': 7.24.0
-    dev: true
 
   /@babel/helper-module-imports@7.24.3:
     resolution: {integrity: sha512-viKb0F9f2s0BCS22QSF308z/+1YWKV/76mwt61NBzS5izMzDPwdq1pTrzf+Li3npBWX9KdQbkeCt1jSAM7lZqg==}
@@ -1668,7 +1668,6 @@ packages:
       '@babel/helper-simple-access': 7.22.5
       '@babel/helper-split-export-declaration': 7.22.6
       '@babel/helper-validator-identifier': 7.24.5
-    dev: true
 
   /@babel/helper-module-transforms@7.24.5(@babel/core@7.24.5):
     resolution: {integrity: sha512-9GxeY8c2d2mdQUP1Dye0ks3VDyIMS98kt/llQ2nUId8IsWqTF0l1LkSX0/uP7l7MCDrzXS009Hyhe2gzTiGW8A==}
@@ -1694,7 +1693,6 @@ packages:
     engines: {node: '>=6.9.0'}
     dependencies:
       '@babel/types': 7.24.0
-    dev: true
 
   /@babel/helper-simple-access@7.24.5:
     resolution: {integrity: sha512-uH3Hmf5q5n7n8mz7arjUlDOCbttY/DW4DYhE6FUsjKJ/oYC1kQQUvwEQWxRwUpX9qQKRXeqLwWxrqilMrf32sQ==}
@@ -1708,7 +1706,6 @@ packages:
     engines: {node: '>=6.9.0'}
     dependencies:
       '@babel/types': 7.24.0
-    dev: true
 
   /@babel/helper-split-export-declaration@7.24.5:
     resolution: {integrity: sha512-5CHncttXohrHk8GWOFCcCl4oRD9fKosWlIRgWm4ql9VYioKm52Mk2xsmoohvm7f3JoiLSM5ZgJuRaf5QZZYd3Q==}
@@ -1743,7 +1740,6 @@ packages:
       '@babel/types': 7.24.0
     transitivePeerDependencies:
       - supports-color
-    dev: true
 
   /@babel/helpers@7.24.5:
     resolution: {integrity: sha512-CiQmBMMpMQHwM5m01YnrM6imUG1ebgYJ+fAIW4FZe6m4qHTPaRHti+R8cggAwkdz4oXhtO4/K9JWlh+8hIfR2Q==}
@@ -1949,11 +1945,10 @@ packages:
       '@babel/helper-split-export-declaration': 7.22.6
       '@babel/parser': 7.24.1
       '@babel/types': 7.24.0
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       globals: 11.12.0
     transitivePeerDependencies:
       - supports-color
-    dev: true
 
   /@babel/traverse@7.24.5:
     resolution: {integrity: sha512-7aaBLeDQ4zYcUFDUD41lJc1fG8+5IU9DaNSJAgal866FGvmD5EbWQgnEC6kO1gGLsX0esNkfnJSndbTXA3r7UA==}
@@ -2079,14 +2074,6 @@ packages:
       style-mod: 4.1.2
     dev: false
 
-  /@codemirror/lint@6.7.1:
-    resolution: {integrity: sha512-rELba6QJD20/bNXWP/cKTGLrwVEcpa2ViwULCV03ONcY1Je85++7sczVRUlnE4TJMjatx3IJTz6HX4NXi+moXw==}
-    dependencies:
-      '@codemirror/state': 6.4.1
-      '@codemirror/view': 6.26.3
-      crelt: 1.0.6
-    dev: false
-
   /@codemirror/lint@6.8.0:
     resolution: {integrity: sha512-lsFofvaw0lnPRJlQylNsC4IRt/1lI4OD/yYslrSGVndOJfStc58v+8p9dgGiD90ktOfL7OhBWns1ZETYgz0EJA==}
     dependencies:
@@ -2572,7 +2559,7 @@ packages:
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       espree: 9.6.1
       globals: 13.24.0
       ignore: 5.3.1
@@ -2697,7 +2684,7 @@ packages:
     engines: {node: '>=10.10.0'}
     dependencies:
       '@humanwhocodes/object-schema': 2.0.2
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -4010,7 +3997,7 @@ packages:
       typescript:
         optional: true
     dependencies:
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       extract-zip: 2.0.1
       https-proxy-agent: 5.0.1
       progress: 2.0.3
@@ -4256,7 +4243,7 @@ packages:
       '@types/react-dom':
         optional: true
     dependencies:
-      '@babel/runtime': 7.24.0
+      '@babel/runtime': 7.24.5
       '@radix-ui/primitive': 1.0.1
       '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.79)(react@18.2.0)
       '@radix-ui/react-context': 1.0.1(@types/react@18.2.79)(react@18.2.0)
@@ -4841,6 +4828,33 @@ packages:
       react-dom: 18.2.0(react@18.2.0)
     dev: false
 
+  /@radix-ui/react-toggle-group@1.0.4(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0):
+    resolution: {integrity: sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==}
+    peerDependencies:
+      '@types/react': '*'
+      '@types/react-dom': '*'
+      react: ^16.8 || ^17.0 || ^18.0
+      react-dom: ^16.8 || ^17.0 || ^18.0
+    peerDependenciesMeta:
+      '@types/react':
+        optional: true
+      '@types/react-dom':
+        optional: true
+    dependencies:
+      '@babel/runtime': 7.24.5
+      '@radix-ui/primitive': 1.0.1
+      '@radix-ui/react-context': 1.0.1(@types/react@18.2.79)(react@18.2.0)
+      '@radix-ui/react-direction': 1.0.1(@types/react@18.2.79)(react@18.2.0)
+      '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
+      '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
+      '@radix-ui/react-toggle': 1.0.3(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
+      '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.79)(react@18.2.0)
+      '@types/react': 18.2.79
+      '@types/react-dom': 18.2.25
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+    dev: false
+
   /@radix-ui/react-toggle@1.0.3(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==}
     peerDependencies:
@@ -5511,7 +5525,7 @@ packages:
       '@sentry/vercel-edge': 7.113.0
       '@sentry/webpack-plugin': 1.21.0
       chalk: 3.0.0
-      next: 14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
+      next: 14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       resolve: 1.22.8
       rollup: 2.78.0
@@ -6315,7 +6329,7 @@ packages:
       '@trpc/client': 10.45.2(@trpc/server@10.45.2)
       '@trpc/react-query': 10.45.2(@tanstack/react-query@4.36.1)(@trpc/client@10.45.2)(@trpc/server@10.45.2)(react-dom@18.2.0)(react@18.2.0)
       '@trpc/server': 10.45.2
-      next: 14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
+      next: 14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
     dev: false
@@ -6710,7 +6724,7 @@ packages:
     dev: true
     optional: true
 
-  /@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5):
+  /@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==}
     engines: {node: ^16.0.0 || >=18.0.0}
     peerDependencies:
@@ -6722,11 +6736,40 @@ packages:
         optional: true
     dependencies:
       '@eslint-community/regexpp': 4.10.0
-      '@typescript-eslint/parser': 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/scope-manager': 6.21.0
       '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/visitor-keys': 6.21.0
+      debug: 4.3.4
+      eslint: 8.57.0
+      graphemer: 1.4.0
+      ignore: 5.3.1
+      natural-compare: 1.4.0
+      semver: 7.6.0
+      ts-api-utils: 1.3.0(typescript@5.4.5)
+      typescript: 5.4.5
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
+  /@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5):
+    resolution: {integrity: sha512-STEDMVQGww5lhCuNXVSQfbfuNII5E08QWkvAw5Qwf+bj2WT+JkG1uc+5/vXA3AOYMDHVOSpL+9rcbEUiHIm2dw==}
+    engines: {node: ^18.18.0 || >=20.0.0}
+    peerDependencies:
+      '@typescript-eslint/parser': ^7.0.0
+      eslint: ^8.56.0
+      typescript: '*'
+    peerDependenciesMeta:
+      typescript:
+        optional: true
+    dependencies:
+      '@eslint-community/regexpp': 4.10.0
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/scope-manager': 7.3.1
+      '@typescript-eslint/type-utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/visitor-keys': 7.3.1
       debug: 4.3.4(supports-color@5.5.0)
       eslint: 8.57.0
       graphemer: 1.4.0
@@ -6739,30 +6782,22 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5):
-    resolution: {integrity: sha512-STEDMVQGww5lhCuNXVSQfbfuNII5E08QWkvAw5Qwf+bj2WT+JkG1uc+5/vXA3AOYMDHVOSpL+9rcbEUiHIm2dw==}
+  /@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5):
+    resolution: {integrity: sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
-      '@typescript-eslint/parser': ^7.0.0
       eslint: ^8.56.0
       typescript: '*'
     peerDependenciesMeta:
       typescript:
         optional: true
     dependencies:
-      '@eslint-community/regexpp': 4.10.0
-      '@typescript-eslint/parser': 7.7.0(eslint@8.57.0)(typescript@5.4.5)
-      '@typescript-eslint/scope-manager': 7.3.1
-      '@typescript-eslint/type-utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
-      '@typescript-eslint/utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
-      '@typescript-eslint/visitor-keys': 7.3.1
-      debug: 4.3.4(supports-color@5.5.0)
+      '@typescript-eslint/scope-manager': 7.12.0
+      '@typescript-eslint/types': 7.12.0
+      '@typescript-eslint/typescript-estree': 7.12.0(typescript@5.4.5)
+      '@typescript-eslint/visitor-keys': 7.12.0
+      debug: 4.3.4
       eslint: 8.57.0
-      graphemer: 1.4.0
-      ignore: 5.3.1
-      natural-compare: 1.4.0
-      semver: 7.6.0
-      ts-api-utils: 1.3.0(typescript@5.4.5)
       typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
@@ -6782,28 +6817,7 @@ packages:
       '@typescript-eslint/types': 7.2.0
       '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.4.5)
       '@typescript-eslint/visitor-keys': 7.2.0
-      debug: 4.3.4(supports-color@5.5.0)
-      eslint: 8.57.0
-      typescript: 5.4.5
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
-
-  /@typescript-eslint/parser@7.7.0(eslint@8.57.0)(typescript@5.4.5):
-    resolution: {integrity: sha512-fNcDm3wSwVM8QYL4HKVBggdIPAy9Q41vcvC/GtDobw3c4ndVT3K6cqudUmjHPw8EAp4ufax0o58/xvWaP2FmTg==}
-    engines: {node: ^18.18.0 || >=20.0.0}
-    peerDependencies:
-      eslint: ^8.56.0
-      typescript: '*'
-    peerDependenciesMeta:
-      typescript:
-        optional: true
-    dependencies:
-      '@typescript-eslint/scope-manager': 7.7.0
-      '@typescript-eslint/types': 7.7.0
-      '@typescript-eslint/typescript-estree': 7.7.0(typescript@5.4.5)
-      '@typescript-eslint/visitor-keys': 7.7.0
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       eslint: 8.57.0
       typescript: 5.4.5
     transitivePeerDependencies:
@@ -6826,6 +6840,14 @@ packages:
       '@typescript-eslint/visitor-keys': 6.21.0
     dev: true
 
+  /@typescript-eslint/scope-manager@7.12.0:
+    resolution: {integrity: sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg==}
+    engines: {node: ^18.18.0 || >=20.0.0}
+    dependencies:
+      '@typescript-eslint/types': 7.12.0
+      '@typescript-eslint/visitor-keys': 7.12.0
+    dev: true
+
   /@typescript-eslint/scope-manager@7.2.0:
     resolution: {integrity: sha512-Qh976RbQM/fYtjx9hs4XkayYujB/aPwglw2choHmf3zBjB4qOywWSdt9+KLRdHubGcoSwBnXUH2sR3hkyaERRg==}
     engines: {node: ^16.0.0 || >=18.0.0}
@@ -6842,14 +6864,6 @@ packages:
       '@typescript-eslint/visitor-keys': 7.3.1
     dev: true
 
-  /@typescript-eslint/scope-manager@7.7.0:
-    resolution: {integrity: sha512-/8INDn0YLInbe9Wt7dK4cXLDYp0fNHP5xKLHvZl3mOT5X17rK/YShXaiNmorl+/U4VKCVIjJnx4Ri5b0y+HClw==}
-    engines: {node: ^18.18.0 || >=20.0.0}
-    dependencies:
-      '@typescript-eslint/types': 7.7.0
-      '@typescript-eslint/visitor-keys': 7.7.0
-    dev: true
-
   /@typescript-eslint/type-utils@6.21.0(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag==}
     engines: {node: ^16.0.0 || >=18.0.0}
@@ -6862,7 +6876,7 @@ packages:
     dependencies:
       '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.4.5)
       '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.4.5)
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       eslint: 8.57.0
       ts-api-utils: 1.3.0(typescript@5.4.5)
       typescript: 5.4.5
@@ -6900,6 +6914,11 @@ packages:
     engines: {node: ^16.0.0 || >=18.0.0}
     dev: true
 
+  /@typescript-eslint/types@7.12.0:
+    resolution: {integrity: sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg==}
+    engines: {node: ^18.18.0 || >=20.0.0}
+    dev: true
+
   /@typescript-eslint/types@7.2.0:
     resolution: {integrity: sha512-XFtUHPI/abFhm4cbCDc5Ykc8npOKBSJePY3a3s+lwumt7XWJuzP5cZcfZ610MIPHjQjNsOLlYK8ASPaNG8UiyA==}
     engines: {node: ^16.0.0 || >=18.0.0}
@@ -6910,11 +6929,6 @@ packages:
     engines: {node: ^18.18.0 || >=20.0.0}
     dev: true
 
-  /@typescript-eslint/types@7.7.0:
-    resolution: {integrity: sha512-G01YPZ1Bd2hn+KPpIbrAhEWOn5lQBrjxkzHkWvP6NucMXFtfXoevK82hzQdpfuQYuhkvFDeQYbzXCjR1z9Z03w==}
-    engines: {node: ^18.18.0 || >=20.0.0}
-    dev: true
-
   /@typescript-eslint/typescript-estree@5.62.0(typescript@5.4.5):
     resolution: {integrity: sha512-CmcQ6uY7b9y694lKdRB8FEel7JbU/40iSAPomu++SjLMntB+2Leay2LO6i8VnJk58MtE9/nQSFIH6jpyRWyYzA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -6947,7 +6961,7 @@ packages:
     dependencies:
       '@typescript-eslint/types': 6.21.0
       '@typescript-eslint/visitor-keys': 6.21.0
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       globby: 11.1.0
       is-glob: 4.0.3
       minimatch: 9.0.3
@@ -6958,6 +6972,28 @@ packages:
       - supports-color
     dev: true
 
+  /@typescript-eslint/typescript-estree@7.12.0(typescript@5.4.5):
+    resolution: {integrity: sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ==}
+    engines: {node: ^18.18.0 || >=20.0.0}
+    peerDependencies:
+      typescript: '*'
+    peerDependenciesMeta:
+      typescript:
+        optional: true
+    dependencies:
+      '@typescript-eslint/types': 7.12.0
+      '@typescript-eslint/visitor-keys': 7.12.0
+      debug: 4.3.4
+      globby: 11.1.0
+      is-glob: 4.0.3
+      minimatch: 9.0.4
+      semver: 7.6.2
+      ts-api-utils: 1.3.0(typescript@5.4.5)
+      typescript: 5.4.5
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
   /@typescript-eslint/typescript-estree@7.2.0(typescript@5.4.5):
     resolution: {integrity: sha512-cyxS5WQQCoBwSakpMrvMXuMDEbhOo9bNHHrNcEWis6XHx6KF518tkF1wBvKIn/tpq5ZpUYK7Bdklu8qY0MsFIA==}
     engines: {node: ^16.0.0 || >=18.0.0}
@@ -6969,7 +7005,7 @@ packages:
     dependencies:
       '@typescript-eslint/types': 7.2.0
       '@typescript-eslint/visitor-keys': 7.2.0
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       globby: 11.1.0
       is-glob: 4.0.3
       minimatch: 9.0.3
@@ -7002,28 +7038,6 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/typescript-estree@7.7.0(typescript@5.4.5):
-    resolution: {integrity: sha512-8p71HQPE6CbxIBy2kWHqM1KGrC07pk6RJn40n0DSc6bMOBBREZxSDJ+BmRzc8B5OdaMh1ty3mkuWRg4sCFiDQQ==}
-    engines: {node: ^18.18.0 || >=20.0.0}
-    peerDependencies:
-      typescript: '*'
-    peerDependenciesMeta:
-      typescript:
-        optional: true
-    dependencies:
-      '@typescript-eslint/types': 7.7.0
-      '@typescript-eslint/visitor-keys': 7.7.0
-      debug: 4.3.4(supports-color@5.5.0)
-      globby: 11.1.0
-      is-glob: 4.0.3
-      minimatch: 9.0.4
-      semver: 7.6.0
-      ts-api-utils: 1.3.0(typescript@5.4.5)
-      typescript: 5.4.5
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
-
   /@typescript-eslint/utils@5.62.0(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-n8oxjeb5aIbPFEtmQxQYOLI0i9n5ySBEY/ZEHHZqKQSFnxio1rv6dthascc9dLuwrL0RC5mPCxB7vnAVGAYWAQ==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -7098,6 +7112,14 @@ packages:
       eslint-visitor-keys: 3.4.3
     dev: true
 
+  /@typescript-eslint/visitor-keys@7.12.0:
+    resolution: {integrity: sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ==}
+    engines: {node: ^18.18.0 || >=20.0.0}
+    dependencies:
+      '@typescript-eslint/types': 7.12.0
+      eslint-visitor-keys: 3.4.3
+    dev: true
+
   /@typescript-eslint/visitor-keys@7.2.0:
     resolution: {integrity: sha512-c6EIQRHhcpl6+tO8EMR+kjkkV+ugUNXOmeASA1rlzkd8EPIriavpWoiEz1HR/VLhbVIdhqnV6E7JZm00cBDx2A==}
     engines: {node: ^16.0.0 || >=18.0.0}
@@ -7114,14 +7136,6 @@ packages:
       eslint-visitor-keys: 3.4.3
     dev: true
 
-  /@typescript-eslint/visitor-keys@7.7.0:
-    resolution: {integrity: sha512-h0WHOj8MhdhY8YWkzIF30R379y0NqyOHExI9N9KCzvmu05EgG4FumeYa3ccfKUSphyWkWQE1ybVrgz/Pbam6YA==}
-    engines: {node: ^18.18.0 || >=20.0.0}
-    dependencies:
-      '@typescript-eslint/types': 7.7.0
-      eslint-visitor-keys: 3.4.3
-    dev: true
-
   /@uiw/codemirror-extensions-basic-setup@4.21.25(@codemirror/autocomplete@6.16.0)(@codemirror/commands@6.3.3)(@codemirror/language@6.10.1)(@codemirror/lint@6.8.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3):
     resolution: {integrity: sha512-eeUKlmEE8aSoSgelS8OR2elcPGntpRo669XinAqPCLa0eKorT2B0d3ts+AE+njAeGk744tiyAEbHb2n+6OQmJw==}
     peerDependencies:
@@ -7238,14 +7252,14 @@ packages:
       '@babel/core': 7.24.3
       '@babel/eslint-parser': 7.24.1(@babel/core@7.24.3)(eslint@8.57.0)
       '@rushstack/eslint-patch': 1.7.2
-      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5)
-      '@typescript-eslint/parser': 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
       eslint-config-prettier: 9.1.0(eslint@8.57.0)
       eslint-import-resolver-alias: 1.1.2(eslint-plugin-import@2.29.1)
-      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.7.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
       eslint-plugin-eslint-comments: 3.2.0(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
       eslint-plugin-jest: 27.9.0(@typescript-eslint/eslint-plugin@7.3.1)(eslint@8.57.0)(typescript@5.4.5)
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
       eslint-plugin-playwright: 1.5.4(eslint-plugin-jest@27.9.0)(eslint@8.57.0)
@@ -7434,7 +7448,7 @@ packages:
     resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
     engines: {node: '>= 6.0.0'}
     dependencies:
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
     transitivePeerDependencies:
       - supports-color
 
@@ -8420,7 +8434,7 @@ packages:
       '@codemirror/autocomplete': 6.16.0(@codemirror/language@6.10.1)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3)(@lezer/common@1.2.1)
       '@codemirror/commands': 6.5.0
       '@codemirror/language': 6.10.1
-      '@codemirror/lint': 6.7.1
+      '@codemirror/lint': 6.8.0
       '@codemirror/search': 6.5.6
       '@codemirror/state': 6.4.1
       '@codemirror/view': 6.26.3
@@ -8865,6 +8879,17 @@ packages:
       ms: 2.1.3
     dev: true
 
+  /debug@4.3.4:
+    resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+    dependencies:
+      ms: 2.1.2
+
   /debug@4.3.4(supports-color@5.5.0):
     resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
     engines: {node: '>=6.0'}
@@ -9521,7 +9546,7 @@ packages:
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
       eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
       eslint-plugin-react: 7.34.1(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.0(eslint@8.57.0)
@@ -9550,7 +9575,7 @@ packages:
       eslint-plugin-promise: ^6.0.0
     dependencies:
       eslint: 8.57.0
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
       eslint-plugin-n: 16.6.2(eslint@8.57.0)
       eslint-plugin-promise: 6.1.1(eslint@8.57.0)
     dev: true
@@ -9570,7 +9595,7 @@ packages:
     peerDependencies:
       eslint-plugin-import: '>=1.4.0'
     dependencies:
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
     dev: true
 
   /eslint-import-resolver-node@0.3.9:
@@ -9583,7 +9608,7 @@ packages:
       - supports-color
     dev: true
 
-  /eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
+  /eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
     resolution: {integrity: sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==}
     engines: {node: ^14.18.0 || >=16.0.0}
     peerDependencies:
@@ -9593,8 +9618,8 @@ packages:
       debug: 4.3.4(supports-color@5.5.0)
       enhanced-resolve: 5.16.0
       eslint: 8.57.0
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
       fast-glob: 3.3.2
       get-tsconfig: 4.7.3
       is-core-module: 2.13.1
@@ -9606,18 +9631,18 @@ packages:
       - supports-color
     dev: true
 
-  /eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.7.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
+  /eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
     resolution: {integrity: sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==}
     engines: {node: ^14.18.0 || >=16.0.0}
     peerDependencies:
       eslint: '*'
       eslint-plugin-import: '*'
     dependencies:
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       enhanced-resolve: 5.16.0
       eslint: 8.57.0
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)
       fast-glob: 3.3.2
       get-tsconfig: 4.7.3
       is-core-module: 2.13.1
@@ -9629,6 +9654,65 @@ packages:
       - supports-color
     dev: true
 
+  /eslint-module-utils@2.8.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
+    resolution: {integrity: sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==}
+    engines: {node: '>=4'}
+    peerDependencies:
+      '@typescript-eslint/parser': '*'
+      eslint: '*'
+      eslint-import-resolver-node: '*'
+      eslint-import-resolver-typescript: '*'
+      eslint-import-resolver-webpack: '*'
+    peerDependenciesMeta:
+      '@typescript-eslint/parser':
+        optional: true
+      eslint:
+        optional: true
+      eslint-import-resolver-node:
+        optional: true
+      eslint-import-resolver-typescript:
+        optional: true
+      eslint-import-resolver-webpack:
+        optional: true
+    dependencies:
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
+      debug: 3.2.7
+      eslint: 8.57.0
+      eslint-import-resolver-node: 0.3.9
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
+  /eslint-module-utils@2.8.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-node@0.3.9)(eslint@8.57.0):
+    resolution: {integrity: sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==}
+    engines: {node: '>=4'}
+    peerDependencies:
+      '@typescript-eslint/parser': '*'
+      eslint: '*'
+      eslint-import-resolver-node: '*'
+      eslint-import-resolver-typescript: '*'
+      eslint-import-resolver-webpack: '*'
+    peerDependenciesMeta:
+      '@typescript-eslint/parser':
+        optional: true
+      eslint:
+        optional: true
+      eslint-import-resolver-node:
+        optional: true
+      eslint-import-resolver-typescript:
+        optional: true
+      eslint-import-resolver-webpack:
+        optional: true
+    dependencies:
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
+      debug: 3.2.7
+      eslint: 8.57.0
+      eslint-import-resolver-node: 0.3.9
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
   /eslint-module-utils@2.8.1(@typescript-eslint/parser@7.2.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
     resolution: {integrity: sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==}
     engines: {node: '>=4'}
@@ -9659,36 +9743,6 @@ packages:
       - supports-color
     dev: true
 
-  /eslint-module-utils@2.8.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
-    resolution: {integrity: sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==}
-    engines: {node: '>=4'}
-    peerDependencies:
-      '@typescript-eslint/parser': '*'
-      eslint: '*'
-      eslint-import-resolver-node: '*'
-      eslint-import-resolver-typescript: '*'
-      eslint-import-resolver-webpack: '*'
-    peerDependenciesMeta:
-      '@typescript-eslint/parser':
-        optional: true
-      eslint:
-        optional: true
-      eslint-import-resolver-node:
-        optional: true
-      eslint-import-resolver-typescript:
-        optional: true
-      eslint-import-resolver-webpack:
-        optional: true
-    dependencies:
-      '@typescript-eslint/parser': 7.7.0(eslint@8.57.0)(typescript@5.4.5)
-      debug: 3.2.7
-      eslint: 8.57.0
-      eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.7.0)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
-
   /eslint-plugin-es-x@7.6.0(eslint@8.57.0):
     resolution: {integrity: sha512-I0AmeNgevgaTR7y2lrVCJmGYF0rjoznpDvqV/kIkZSZbZ8Rw3eu4cGlvBBULScfkSOCzqKbff5LR4CNrV7mZHA==}
     engines: {node: ^14.18.0 || >=16.0.0}
@@ -9712,7 +9766,7 @@ packages:
       ignore: 5.3.1
     dev: true
 
-  /eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
+  /eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
     resolution: {integrity: sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==}
     engines: {node: '>=4'}
     peerDependencies:
@@ -9722,7 +9776,7 @@ packages:
       '@typescript-eslint/parser':
         optional: true
     dependencies:
-      '@typescript-eslint/parser': 7.7.0(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       array-includes: 3.1.7
       array.prototype.findlastindex: 1.2.4
       array.prototype.flat: 1.3.2
@@ -9731,7 +9785,42 @@ packages:
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.7.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      hasown: 2.0.2
+      is-core-module: 2.13.1
+      is-glob: 4.0.3
+      minimatch: 3.1.2
+      object.fromentries: 2.0.8
+      object.groupby: 1.0.3
+      object.values: 1.2.0
+      semver: 6.3.1
+      tsconfig-paths: 3.15.0
+    transitivePeerDependencies:
+      - eslint-import-resolver-typescript
+      - eslint-import-resolver-webpack
+      - supports-color
+    dev: true
+
+  /eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0):
+    resolution: {integrity: sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==}
+    engines: {node: '>=4'}
+    peerDependencies:
+      '@typescript-eslint/parser': '*'
+      eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8
+    peerDependenciesMeta:
+      '@typescript-eslint/parser':
+        optional: true
+    dependencies:
+      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
+      array-includes: 3.1.7
+      array.prototype.findlastindex: 1.2.4
+      array.prototype.flat: 1.3.2
+      array.prototype.flatmap: 1.3.2
+      debug: 3.2.7
+      doctrine: 2.1.0
+      eslint: 8.57.0
+      eslint-import-resolver-node: 0.3.9
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0)(eslint-import-resolver-node@0.3.9)(eslint@8.57.0)
       hasown: 2.0.2
       is-core-module: 2.13.1
       is-glob: 4.0.3
@@ -9760,7 +9849,7 @@ packages:
       jest:
         optional: true
     dependencies:
-      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
     transitivePeerDependencies:
@@ -9968,7 +10057,7 @@ packages:
       vitest:
         optional: true
     dependencies:
-      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.7.0)(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.12.0)(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
     transitivePeerDependencies:
@@ -10018,7 +10107,7 @@ packages:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.3
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       doctrine: 3.0.0
       escape-string-regexp: 4.0.0
       eslint-scope: 7.2.2
@@ -10251,7 +10340,7 @@ packages:
     engines: {node: '>= 10.17.0'}
     hasBin: true
     dependencies:
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       get-stream: 5.2.0
       yauzl: 2.10.0
     optionalDependencies:
@@ -10986,7 +11075,7 @@ packages:
     dependencies:
       '@tootallnate/once': 2.0.0
       agent-base: 6.0.2
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
     transitivePeerDependencies:
       - supports-color
     dev: true
@@ -11024,7 +11113,7 @@ packages:
     engines: {node: '>= 6'}
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
     transitivePeerDependencies:
       - supports-color
 
@@ -11654,7 +11743,7 @@ packages:
     resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==}
     engines: {node: '>=10'}
     dependencies:
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       istanbul-lib-coverage: 3.2.2
       source-map: 0.6.1
     transitivePeerDependencies:
@@ -13533,7 +13622,7 @@ packages:
       '@panva/hkdf': 1.1.1
       cookie: 0.5.0
       jose: 4.15.5
-      next: 14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
+      next: 14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
       nodemailer: 6.9.13
       oauth: 0.9.15
       openid-client: 5.6.5
@@ -13551,7 +13640,7 @@ packages:
       react: ^16.8.0 || ^17.0.0 || ^18.0.0
       use-query-params: ^2.0.0
     dependencies:
-      next: 14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
+      next: 14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       tslib: 2.6.2
       use-query-params: 2.2.1(react-dom@18.2.0)(react@18.2.0)
@@ -13567,6 +13656,49 @@ packages:
       react-dom: 18.2.0(react@18.2.0)
     dev: false
 
+  /next@14.2.3(@babel/core@7.24.3)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0):
+    resolution: {integrity: sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==}
+    engines: {node: '>=18.17.0'}
+    hasBin: true
+    peerDependencies:
+      '@opentelemetry/api': ^1.1.0
+      '@playwright/test': ^1.41.2
+      react: ^18.2.0
+      react-dom: ^18.2.0
+      sass: ^1.3.0
+    peerDependenciesMeta:
+      '@opentelemetry/api':
+        optional: true
+      '@playwright/test':
+        optional: true
+      sass:
+        optional: true
+    dependencies:
+      '@next/env': 14.2.3
+      '@playwright/test': 1.43.1
+      '@swc/helpers': 0.5.5
+      busboy: 1.6.0
+      caniuse-lite: 1.0.30001599
+      graceful-fs: 4.2.11
+      postcss: 8.4.31
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+      styled-jsx: 5.1.1(@babel/core@7.24.3)(react@18.2.0)
+    optionalDependencies:
+      '@next/swc-darwin-arm64': 14.2.3
+      '@next/swc-darwin-x64': 14.2.3
+      '@next/swc-linux-arm64-gnu': 14.2.3
+      '@next/swc-linux-arm64-musl': 14.2.3
+      '@next/swc-linux-x64-gnu': 14.2.3
+      '@next/swc-linux-x64-musl': 14.2.3
+      '@next/swc-win32-arm64-msvc': 14.2.3
+      '@next/swc-win32-ia32-msvc': 14.2.3
+      '@next/swc-win32-x64-msvc': 14.2.3
+    transitivePeerDependencies:
+      - '@babel/core'
+      - babel-plugin-macros
+    dev: false
+
   /next@14.2.3(@babel/core@7.24.5)(@playwright/test@1.43.1)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==}
     engines: {node: '>=18.17.0'}
@@ -14905,7 +15037,7 @@ packages:
       '@puppeteer/browsers': 0.5.0(typescript@5.4.5)
       chromium-bidi: 0.4.7(devtools-protocol@0.0.1107588)
       cross-fetch: 3.1.5
-      debug: 4.3.4(supports-color@5.5.0)
+      debug: 4.3.4
       devtools-protocol: 0.0.1107588
       extract-zip: 2.0.1
       https-proxy-agent: 5.0.1
@@ -16158,6 +16290,24 @@ packages:
     resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==}
     dev: false
 
+  /styled-jsx@5.1.1(@babel/core@7.24.3)(react@18.2.0):
+    resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==}
+    engines: {node: '>= 12.0.0'}
+    peerDependencies:
+      '@babel/core': '*'
+      babel-plugin-macros: '*'
+      react: '>= 16.8.0 || 17.x.x || ^18.0.0-0'
+    peerDependenciesMeta:
+      '@babel/core':
+        optional: true
+      babel-plugin-macros:
+        optional: true
+    dependencies:
+      '@babel/core': 7.24.3
+      client-only: 0.0.1
+      react: 18.2.0
+    dev: false
+
   /styled-jsx@5.1.1(@babel/core@7.24.5)(react@18.2.0):
     resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==}
     engines: {node: '>= 12.0.0'}
@@ -17050,6 +17200,20 @@ packages:
     engines: {node: '>= 0.8'}
     dev: false
 
+  /vaul@0.9.1(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0):
+    resolution: {integrity: sha512-fAhd7i4RNMinx+WEm6pF3nOl78DFkAazcN04ElLPFF9BMCNGbY/kou8UMhIcicm0rJCNePJP0Yyza60gGOD0Jw==}
+    peerDependencies:
+      react: ^16.8 || ^17.0 || ^18.0
+      react-dom: ^16.8 || ^17.0 || ^18.0
+    dependencies:
+      '@radix-ui/react-dialog': 1.0.5(@types/react-dom@18.2.25)(@types/react@18.2.79)(react-dom@18.2.0)(react@18.2.0)
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+    transitivePeerDependencies:
+      - '@types/react'
+      - '@types/react-dom'
+    dev: false
+
   /victory-vendor@36.9.2:
     resolution: {integrity: sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==}
     dependencies:

web/Dockerfile

@@ -5,7 +5,7 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.47.2",
+  "version": "2.49.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -56,6 +56,7 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
+    "@radix-ui/react-toggle-group": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
     "@react-email/components": "^0.0.18",
     "@react-email/render": "^0.0.14",
@@ -70,7 +71,6 @@
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
-    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
@@ -116,6 +116,7 @@
     "tailwindcss-animate": "^1.0.7",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
@@ -137,7 +138,7 @@
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",

web/public/generated/api/openapi.yml

@@ -1137,7 +1137,10 @@ paths:
       security: *ref_0
   /api/public/sessions/{sessionId}:
     get:
-      description: Get a session
+      description: >-
+        Get a session. Please note that `traces` on this endpoint are not
+        paginated, if you plan to fetch large sessions, consider `GET
+        /api/public/traces?sessionId=<sessionId>`
       operationId: sessions_get
       tags:
         - Sessions
@@ -1262,6 +1265,12 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: sessionId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
         - name: fromTimestamp
           in: query
           description: Retrieve only traces newer than this datetime (ISO 8601).
@@ -2499,6 +2508,7 @@ components:
         totalCost:
           type: number
           format: double
+          description: Total model cost in USD
         usage:
           type: array
           items:
@@ -2521,14 +2531,28 @@ components:
           nullable: true
         inputUsage:
           type: integer
+          description: Total number of generation input units (e.g. tokens)
         outputUsage:
           type: integer
+          description: Total number of generation output units (e.g. tokens)
         totalUsage:
           type: integer
+          description: Total number of generation total units (e.g. tokens)
+        countTraces:
+          type: integer
+        countObservations:
+          type: integer
+        totalCost:
+          type: number
+          format: double
+          description: Total model cost in USD
       required:
         - inputUsage
         - outputUsage
         - totalUsage
+        - countTraces
+        - countObservations
+        - totalCost
     Observations:
       title: Observations
       type: object
@@ -2654,6 +2678,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2672,6 +2703,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2713,11 +2751,20 @@ components:
           type: array
           items:
             type: string
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          description: >-
+            List of tags. Used to filter via UI and API. The same across
+            versions of a prompt.
       required:
         - name
         - version
         - config
         - labels
+        - tags
     ChatMessage:
       title: ChatMessage
       type: object

web/public/generated/postman/collection.json

@@ -699,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -890,7 +890,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a session",
+            "description": "Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`",
             "url": {
               "raw": "{{baseUrl}}/api/public/sessions/:sessionId",
               "host": [
@@ -963,7 +963,7 @@
           "request": {
             "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -993,6 +993,11 @@
                   "value": "",
                   "description": null
                 },
+                {
+                  "key": "sessionId",
+                  "value": "",
+                  "description": null
+                },
                 {
                   "key": "fromTimestamp",
                   "value": "",

web/sentry.server.config.ts

@@ -18,7 +18,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.3;
+        return 0.1;
       }
       if (
         samplingContext.request &&
@@ -27,12 +27,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.transactionContext.status !== "ok" &&
         samplingContext.transactionContext.status !== "unauthenticated"
       ) {
-        return 1;
+        return 0.1;
       }
-      return 0.1;
+      return 0.01;
     },
 
-    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

web/src/__tests__/metrics-api.servertest.ts

@@ -68,12 +68,18 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 333,
         outputUsage: 0,
         totalUsage: 333,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
       {
         model: "modelC",
         inputUsage: 666,
         outputUsage: 777,
         totalUsage: 1443,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 1024.22,
       },
     ]);
 
@@ -88,6 +94,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         inputUsage: 100,
         outputUsage: 200,
         totalUsage: 300,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
       },
     ]);
   });

web/src/__tests__/prompts.v2.servertest.ts

@@ -680,6 +680,71 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(fetchedPrompt.body.createdBy).toBe("API");
       expect(fetchedPrompt.body.config).toEqual({});
     });
+
+    it("should update tags across versions", async () => {
+      const promptName = "prompt-name" + nanoid();
+
+      const createPromptVersion = async (tags?: string[]) => {
+        await makeAPICall("POST", baseURI, {
+          name: promptName,
+          prompt: "This is a test prompt",
+          type: PromptType.Text,
+          ...(tags !== undefined && { tags: tags }),
+        });
+      };
+
+      const fetchPromptVersion = async (version: number) => {
+        const fetchedPrompt = await makeAPICall(
+          "GET",
+          `${baseURI}/${promptName}?version=${version}`,
+          undefined,
+        );
+        expect(fetchedPrompt.status).toBe(200);
+        if (!isPrompt(fetchedPrompt.body)) {
+          throw new Error("Expected body to be a prompt");
+        }
+        return fetchedPrompt.body;
+      };
+
+      // Create version 1 with ["tag"]
+      await createPromptVersion(["tag"]);
+      let fetchedPrompt1 = await fetchPromptVersion(1);
+      expect(fetchedPrompt1.tags).toEqual(["tag"]);
+      expect(fetchedPrompt1.version).toBe(1);
+
+      // Create version 2 with no tags provided (should use tags from version 1)
+      await createPromptVersion();
+      let fetchedPrompt2 = await fetchPromptVersion(2);
+      expect(fetchedPrompt2.tags).toEqual(["tag"]);
+      expect(fetchedPrompt2.version).toBe(2);
+
+      // Create version 3 with ["tag1", "tag2", "tag3"] (should update tags across versions)
+      await createPromptVersion(["tag1", "tag2", "tag3"]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      let fetchedPrompt3 = await fetchPromptVersion(3);
+      expect(fetchedPrompt1.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt3.version).toBe(3);
+
+      // remove tags
+      await createPromptVersion([]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      fetchedPrompt3 = await fetchPromptVersion(3);
+      let fetchedPrompt4 = await fetchPromptVersion(4);
+      expect(fetchedPrompt1.tags).toEqual([]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual([]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual([]);
+      expect(fetchedPrompt3.version).toBe(3);
+      expect(fetchedPrompt4.tags).toEqual([]);
+      expect(fetchedPrompt4.version).toBe(4);
+    });
   });
 
   describe("when fetching a prompt list", () => {

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
+          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id  WHERE  t."project_id" = `,
           ";",
         ],
       } as const,

web/src/__tests__/traces.servertest.ts

@@ -238,4 +238,62 @@ describe("/api/public/traces API Endpoint", () => {
     expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
     expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
   });
+
+  it("should filter traces by session ID", async () => {
+    const sessionId = "test-session-id";
+    const anotherSessionId = "another-session-id";
+
+    // Create traces with different session IDs
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-1",
+      name: "test-trace-1",
+      sessionId,
+      userId: "user-1",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-2",
+      name: "test-trace-2",
+      sessionId: anotherSessionId,
+      userId: "user-2",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    // Filter by session ID
+    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${sessionId}`,
+    );
+
+    expect(tracesBySessionId.status).toBe(200);
+    expect(tracesBySessionId.body.data).toHaveLength(1);
+    expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
+
+    // Filter by another session ID
+    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${anotherSessionId}`,
+    );
+
+    expect(tracesByAnotherSessionId.status).toBe(200);
+    expect(tracesByAnotherSessionId.body.data).toHaveLength(1);
+    expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
+
+    // Filter by non-existent session ID
+    const tracesByNonExistentSessionId =
+      await makeAPICall<GetTracesAPIResponse>(
+        "GET",
+        `/api/public/traces?sessionId=non-existent-session-id`,
+      );
+
+    expect(tracesByNonExistentSessionId.status).toBe(200);
+    expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);
+  });
 });

web/src/components/LangfuseLogo.tsx

@@ -1,8 +1,105 @@
-import { AlertTriangle, Check } from "lucide-react";
+import {
+  AlertTriangle,
+  Check,
+  Github,
+  HardDriveDownload,
+  Newspaper,
+} from "lucide-react";
 
 import { VERSION } from "@/src/constants";
 import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
+import {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+} from "@/src/components/ui/dropdown-menu";
+import { ArrowUp } from "lucide-react";
+import { api } from "@/src/utils/api";
+import { Button } from "@/src/components/ui/button";
+import Link from "next/link";
+
+const VersionLabel = ({ className }: { className?: string }) => {
+  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
+    refetchOnMount: false,
+    refetchOnWindowFocus: false,
+    refetchOnReconnect: false,
+    enabled: !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION, // do not check for updates on Langfuse Cloud
+    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
+  });
+
+  const hasUpdate =
+    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+    checkUpdate.data &&
+    checkUpdate.data.updateType;
+
+  const color =
+    checkUpdate.data?.updateType === "major"
+      ? "text-dark-red"
+      : checkUpdate.data?.updateType === "minor"
+        ? "text-dark-yellow"
+        : undefined;
+
+  return (
+    <DropdownMenu>
+      <DropdownMenuTrigger asChild>
+        <Button variant="ghost" size="xs" className={className}>
+          {VERSION}
+          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
+        </Button>
+      </DropdownMenuTrigger>
+      <DropdownMenuContent>
+        {hasUpdate ? (
+          <>
+            <DropdownMenuLabel>
+              New {checkUpdate.data?.updateType} version:{" "}
+              {checkUpdate.data?.latestRelease}
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ? (
+          <>
+            <DropdownMenuLabel>This is the latest release</DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : null}
+
+        <DropdownMenuItem asChild>
+          <Link
+            href="https://github.com/langfuse/langfuse/releases"
+            target="_blank"
+          >
+            <Github size={16} className="mr-2" />
+            Releases
+          </Link>
+        </DropdownMenuItem>
+        <DropdownMenuItem asChild>
+          <Link href="https://langfuse.com/changelog" target="_blank">
+            <Newspaper size={16} className="mr-2" />
+            Changelog
+          </Link>
+        </DropdownMenuItem>
+        {hasUpdate && (
+          <>
+            <DropdownMenuSeparator />
+            <DropdownMenuItem asChild>
+              <Link
+                href="https://langfuse.com/docs/deployment/self-host#update"
+                target="_blank"
+              >
+                <HardDriveDownload size={16} className="mr-2" />
+                Update
+              </Link>
+            </DropdownMenuItem>
+          </>
+        )}
+      </DropdownMenuContent>
+    </DropdownMenu>
+  );
+};
 
 export const LangfuseIcon = ({
   size = 32,
@@ -65,22 +162,12 @@ export const LangfuseLogo = ({
       <span
         className={cn(
           "ml-2 font-mono font-semibold",
-          size === "sm" ? "text-sm" : "text-xl",
+          size === "sm" ? "text-sm" : "text-lg",
         )}
       >
         Langfuse
       </span>
-      {version && (
-        <a
-          href="https://github.com/langfuse/langfuse/releases"
-          target="_blank"
-          rel="noopener"
-          title="View releases on GitHub"
-          className="ml-2 text-xs text-muted-foreground"
-        >
-          {VERSION}
-        </a>
-      )}
+      {version && <VersionLabel className="ml-2" />}
     </div>
   </div>
 );

web/src/components/Support.tsx

@@ -67,7 +67,7 @@ const supportChannels = [
     title: "Slack Connect",
     description: "Get a dedicated support channel for you and your team.",
     href: "mailto:help@langfuse.com?subject=Slack%20Connect%20Request&body=I'd%20like%20to%20request%20a%20dedicated%20Slack%20Connect%20channel%20for%20me%20and%20my%20team.%0D%0A%0D%0AUsers%20(emails)%20to%20include%20besides%20mine%3A%0D%0A%0D%0A",
-    buttonText: "Request Slack Connect (via Email)",
+    buttonText: "Request via Email",
   },
 ];
 

web/src/components/grouped-score-badge.tsx

@@ -4,9 +4,13 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import {
+  isBooleanDataType,
+  isCategoricalDataType,
+} from "@/src/features/manual-scoring/lib/helpers";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
 import { cn } from "@/src/utils/tailwind";
-import { MessageCircle } from "lucide-react";
+import { MessageCircleMore } from "lucide-react";
 
 export const GroupedScoreBadges = ({
   scores,
@@ -34,11 +38,13 @@ export const GroupedScoreBadges = ({
     <div className={cn("text-xs", props.className)}>
       {props.scores.map((s, i) => (
         <span key={i} className="group/score ml-1 first:ml-0">
-          {s.value.toFixed(2)}
+          {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
+            ? s.stringValue
+            : s.value.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
-                <MessageCircle size={12} />
+                <MessageCircleMore size={12} />
               </HoverCardTrigger>
               <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                 <p>{s.comment}</p>
@@ -73,9 +79,12 @@ export const GroupedScoreBadges = ({
             <Badge
               variant="outline"
               key={name}
-              className="break-all font-normal"
+              className="grid grid-cols-[1fr,auto] gap-1 font-normal"
             >
-              {name}: <ScoresOfGroup scores={scores} className="ml-2" />
+              <p className="truncate" title={name}>
+                {name}:
+              </p>
+              <ScoresOfGroup scores={scores} />
             </Badge>
           ))}
       </>

web/src/components/session/index.tsx

@@ -7,38 +7,24 @@ import { IOPreview } from "@/src/components/trace/IOPreview";
 import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 import { Badge } from "@/src/components/ui/badge";
 import { Card } from "@/src/components/ui/card";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect, useRef } from "react";
-import { useWindowVirtualizer } from "@tanstack/react-virtual";
-import { type RouterOutput } from "@/src/utils/types";
-import {
-  DataTableRowHeightSwitch,
-  type RowHeight,
-  useRowHeightLocalStorage,
-} from "@/src/components/table/data-table-row-height-switch";
-import { ScrollArea } from "@/src/components/ui/scroll-area";
+import { useEffect, useState } from "react";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
+import { Button } from "@/src/components/ui/button";
 
-// do not use the usual table row heights here
-const rowHeightMapping: Record<RowHeight, number> = {
-  s: 200,
-  m: 350,
-  l: 700,
-};
+// some projects have thousands of traces in a sessions, paginate to avoid rendering all at once
+const PAGE_SIZE = 50;
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
-  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
-    "single-session",
-    "m",
-  );
+  const [visibleTraces, setVisibleTraces] = useState(PAGE_SIZE);
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -96,11 +82,6 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
-          <DataTableRowHeightSwitch
-            rowHeight={rowHeight}
-            setRowHeight={setRowHeight}
-            key="height"
-          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -121,97 +102,49 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      {session.data && (
-        <TraceCardList
-          session={session.data}
-          projectId={projectId}
-          rowHeight={rowHeightMapping[rowHeight]}
-        />
-      )}
-    </div>
-  );
-};
-
-const TraceCardList = ({
-  session,
-  projectId,
-  rowHeight,
-}: {
-  session: RouterOutput["sessions"]["byId"];
-  projectId: string;
-  rowHeight: number;
-}) => {
-  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
-
-  const virtualizer = useWindowVirtualizer({
-    count: session.traces.length,
-    estimateSize: () => rowHeight,
-    overscan: 5,
-    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
-    gap: 10,
-  });
-  useEffect(() => {
-    // re-measure when rowHeight changes to update the virtualizer
-    virtualizer.measure();
-  }, [rowHeight, virtualizer]);
-  return (
-    <div className="mt-5 border-t pt-5">
-      <div
-        ref={listVirtualizationRef}
-        style={{
-          height: `${virtualizer.getTotalSize()}px`,
-          position: "relative",
-        }}
-      >
-        {virtualizer
-          .getVirtualItems()
-          .map((virtualItem) => ({
-            virtualItem,
-            trace: session.traces[virtualItem.index],
-          }))
-          .map(({ virtualItem, trace }) => (
-            <Card
-              className="group grid w-full gap-3 overflow-hidden border-border p-2 shadow-none hover:border-ring   md:grid-cols-3"
-              key={virtualItem.key}
-              data-index={virtualItem.index}
-              style={{
-                position: "absolute",
-                top: 0,
-                left: 0,
-                height: `${rowHeight}px`,
-                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
-              }}
-            >
-              <ScrollArea className="col-span-2 pr-3">
-                <SessionIO traceId={trace.id} />
-              </ScrollArea>
-              <ScrollArea>
-                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
-                  <Link
-                    href={`/project/${projectId}/traces/${trace.id}`}
-                    className="text-xs hover:underline"
-                  >
-                    Trace: {trace.name} ({trace.id})&nbsp;↗
-                  </Link>
-                  <div className="text-xs text-muted-foreground">
-                    {trace.timestamp.toLocaleString()}
-                  </div>
-                  <div className="mb-1 mt-2 text-xs text-muted-foreground">
-                    Scores
-                  </div>
-                  <div className="flex flex-wrap content-start items-start gap-1">
-                    <GroupedScoreBadges scores={trace.scores} />
-                  </div>
-                  <ManualScoreButton
-                    projectId={projectId}
-                    traceId={trace.id}
-                    scores={trace.scores}
-                    variant="badge"
-                  />
-                </div>
-              </ScrollArea>
-            </Card>
-          ))}
+      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
+        {session.data?.traces.slice(0, visibleTraces).map((trace) => (
+          <Card
+            className="group grid gap-3 border-border p-2 shadow-none hover:border-ring md:grid-cols-3"
+            key={trace.id}
+          >
+            <SessionIO traceId={trace.id} />
+            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
+              <Link
+                href={`/project/${projectId}/traces/${trace.id}`}
+                className="text-xs hover:underline"
+              >
+                Trace: {trace.name} ({trace.id})&nbsp;↗
+              </Link>
+              <div className="text-xs text-muted-foreground">
+                {trace.timestamp.toLocaleString()}
+              </div>
+              <div className="mb-1 mt-2 text-xs text-muted-foreground">
+                Scores
+              </div>
+              <div className="mb-1 flex flex-wrap content-start items-start gap-1">
+                <GroupedScoreBadges scores={trace.scores} />
+              </div>
+              <AnnotateButton
+                projectId={projectId}
+                traceId={trace.id}
+                scores={trace.scores}
+                variant="badge"
+                type="session"
+                source="SessionDetail"
+              />
+            </div>
+          </Card>
+        ))}
+        {session.data?.traces && session.data.traces.length > visibleTraces && (
+          <Button
+            onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
+            variant="ghost"
+            className="self-center"
+          >
+            {`Load ${Math.min(session.data.traces.length - visibleTraces, PAGE_SIZE)} More`}
+          </Button>
+        )}
       </div>
     </div>
   );
@@ -230,9 +163,8 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
-
   return (
-    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
+    <div className="col-span-2 flex flex-col gap-2 p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"

web/src/components/table/data-table.tsx

@@ -50,6 +50,8 @@ interface DataTableProps<TData, TValue> {
   help?: { description: string; href: string };
   rowHeight?: RowHeight;
   className?: string;
+  paginationClassName?: string;
+  isBorderless?: boolean;
 }
 
 export interface AsyncTableData<T> {
@@ -71,6 +73,9 @@ export function DataTable<TData extends object, TValue>({
   orderBy,
   setOrderBy,
   rowHeight,
+  className,
+  paginationClassName,
+  isBorderless = false,
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight);
@@ -105,8 +110,18 @@ export function DataTable<TData extends object, TValue>({
 
   return (
     <>
-      <div className="flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto">
-        <div className="w-full overflow-auto rounded-md border">
+      <div
+        className={cn(
+          "flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto",
+          className,
+        )}
+      >
+        <div
+          className={cn(
+            "w-full overflow-auto",
+            isBorderless ? "" : "rounded-md border",
+          )}
+        >
           <Table>
             <TableHeader>
               {table.getHeaderGroups().map((headerGroup) => (
@@ -242,7 +257,12 @@ export function DataTable<TData extends object, TValue>({
         <div className="grow"></div>
       </div>
       {pagination !== undefined ? (
-        <div className="bg:background sticky bottom-0 z-10 flex w-full justify-end font-medium">
+        <div
+          className={cn(
+            "sticky bottom-0 z-10 flex w-full justify-end bg-background font-medium",
+            paginationClassName,
+          )}
+        >
           <DataTablePagination
             table={table}
             paginationOptions={pagination.options}

web/src/components/table/use-cases/score-configs.tsx

@@ -0,0 +1,174 @@
+import React from "react";
+import { Card } from "@/src/components/ui/card";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import { DataTable } from "@/src/components/table/data-table";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { NumberParam, useQueryParams, withDefault } from "use-query-params";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
+
+type ScoreConfigTableRow = {
+  id: string;
+  name: string;
+  dataType: ScoreDataType;
+  createdAt: string;
+  updatedAt: string;
+  range: {
+    maxValue?: number | null;
+    minValue?: number | null;
+    categories?: Prisma.JsonValue | null;
+  };
+  description?: string | null;
+};
+
+function getConfigRange(
+  originalRow: ScoreConfigTableRow,
+): Prisma.JsonValue | undefined {
+  const { range, dataType } = originalRow;
+  if (isNumericDataType(dataType)) {
+    return [
+      { minValue: range.minValue ?? "-∞", maxValue: range.maxValue ?? "∞" },
+    ];
+  }
+  return range.categories;
+}
+
+export function ScoreConfigsTable({ projectId }: { projectId: string }) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "scoreConfigs",
+    "s",
+  );
+
+  const configs = api.scoreConfigs.all.useQuery({
+    projectId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  const totalCount = configs.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<ScoreConfigTableRow>[] = [
+    {
+      accessorKey: "name",
+      id: "name",
+      header: "Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "dataType",
+      id: "dataType",
+      header: "Data Type",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "range",
+      id: "range",
+      header: "Range",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const range = getConfigRange(row.original);
+
+        return !!range ? (
+          <IOTableCell data={range} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "description",
+      id: "description",
+      header: "Description",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const value = row.original.description;
+
+        return !!value ? (
+          <IOTableCell data={value} singleLine={rowHeight === "s"} />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "id",
+      id: "id",
+      header: "Config ID",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "createdAt",
+      id: "createdAt",
+      header: "Created At",
+      enableHiding: true,
+      defaultHidden: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<ScoreConfigTableRow>(
+      "scoreConfigsColumnVisibility",
+      columns,
+    );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+      />
+      <Card className="mb-4 flex max-h-[calc(100dvh-40rem)] flex-col overflow-hidden">
+        <DataTable
+          columns={columns}
+          data={
+            configs.isLoading
+              ? { isLoading: true, isError: false }
+              : configs.isError
+                ? {
+                    isLoading: false,
+                    isError: true,
+                    error: configs.error.message,
+                  }
+                : {
+                    isLoading: false,
+                    isError: false,
+                    data: configs.data?.configs.map((config) => ({
+                      id: config.id,
+                      name: config.name,
+                      dataType: config.dataType,
+                      description: config.description,
+                      createdAt: config.createdAt.toLocaleString(),
+                      updatedAt: config.updatedAt.toLocaleString(),
+                      range: {
+                        maxValue: config.maxValue,
+                        minValue: config.minValue,
+                        categories: config.categories,
+                      },
+                    })),
+                  }
+          }
+          pagination={{
+            pageCount: Math.ceil(totalCount / paginationState.pageSize),
+            onChange: setPaginationState,
+            state: paginationState,
+          }}
+          columnVisibility={columnVisibility}
+          onColumnVisibilityChange={setColumnVisibility}
+          rowHeight={rowHeight}
+          className="gap-2"
+          paginationClassName="-mx-2 mb-2"
+          isBorderless
+        />
+      </Card>
+    </>
+  );
+}

web/src/components/table/use-cases/scores.tsx

@@ -4,8 +4,10 @@ import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { Avatar, AvatarImage } from "@/src/components/ui/avatar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
   type ScoreOptions,
@@ -14,7 +16,7 @@ import {
 import { api } from "@/src/utils/api";
 import { utcDateOffsetByDays } from "@/src/utils/dates";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
-import type { FilterState } from "@langfuse/shared";
+import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useSession } from "next-auth/react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
@@ -24,7 +26,12 @@ export type ScoresTableRow = {
   timestamp: string;
   source: string;
   name: string;
-  value: number;
+  dataType: ScoreDataType;
+  value: string;
+  author: {
+    image?: string;
+    name?: string;
+  };
   comment?: string;
   observationId?: string;
   traceName?: string;
@@ -172,6 +179,29 @@ export default function ScoresTable({
         ) : undefined;
       },
     },
+    {
+      accessorKey: "userId",
+      header: "Trace User ID",
+      id: "userId",
+      headerTooltip: {
+        description: "The user ID associated with the trace.",
+        href: "https://langfuse.com/docs/tracing-features/users",
+      },
+      enableHiding: true,
+      enableSorting: true,
+      cell: ({ row }) => {
+        const value = row.getValue("userId");
+        return typeof value === "string" ? (
+          <>
+            <TableLink
+              path={`/project/${projectId}/users/${value}`}
+              value={value}
+              truncateAt={40}
+            />
+          </>
+        ) : undefined;
+      },
+    },
     {
       accessorKey: "timestamp",
       header: "Timestamp",
@@ -193,38 +223,39 @@ export default function ScoresTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "dataType",
+      header: "Data Type",
+      id: "dataType",
+      enableHiding: true,
+      enableSorting: true,
+    },
     {
       accessorKey: "value",
       header: "Value",
       id: "value",
       enableHiding: true,
       enableSorting: true,
-      cell: ({ row }) => {
-        const value: number = row.getValue("value");
-        return value % 1 === 0 ? value : value.toFixed(4);
-      },
     },
     {
-      accessorKey: "userId",
-      header: "User ID",
-      id: "userId",
-      headerTooltip: {
-        description: "The user ID associated with the trace.",
-        href: "https://langfuse.com/docs/tracing-features/users",
-      },
-      enableHiding: true,
-      enableSorting: true,
+      accessorKey: "author",
+      id: "author",
+      header: "Author",
       cell: ({ row }) => {
-        const value = row.getValue("userId");
-        return typeof value === "string" ? (
-          <>
-            <TableLink
-              path={`/project/${projectId}/users/${value}`}
-              value={value}
-              truncateAt={40}
-            />
-          </>
-        ) : undefined;
+        const { name, image } = row.getValue(
+          "author",
+        ) as ScoresTableRow["author"];
+        return (
+          <div className="flex items-center space-x-2">
+            <Avatar className="h-7 w-7">
+              <AvatarImage
+                src={image ?? undefined}
+                alt={name ?? "User Avatar"}
+              />
+            </Avatar>
+            <span>{name}</span>
+          </div>
+        );
       },
     },
     {
@@ -279,12 +310,21 @@ export default function ScoresTable({
       timestamp: score.timestamp.toLocaleString(),
       source: score.source,
       name: score.name,
-      value: score.value,
+      dataType: score.dataType,
+      value: isNumericDataType(score.dataType)
+        ? score.value % 1 === 0
+          ? String(score.value)
+          : score.value.toFixed(4)
+        : score.stringValue ?? "",
+      author: {
+        image: score.authorUserImage ?? undefined,
+        name: score.authorUserName ?? undefined,
+      },
       comment: score.comment ?? undefined,
       observationId: score.observationId ?? undefined,
       traceId: score.traceId,
       traceName: score.traceName ?? undefined,
-      userId: score.userId ?? undefined,
+      userId: score.traceUserId ?? undefined,
       jobConfigurationId: score.jobConfigurationId ?? undefined,
     };
   };

web/src/components/trace/ObservationPreview.tsx

@@ -8,7 +8,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { Badge } from "@/src/components/ui/badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
@@ -22,6 +21,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
@@ -154,11 +154,12 @@ export const ObservationPreview = (props: {
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={props.projectId}
               traceId={preloadedObservation.traceId}
               observationId={preloadedObservation.id}
               scores={props.scores}
+              type="observation"
             />
             {observationWithInputAndOutput.data?.type === "GENERATION" && (
               <JumpToPlaygroundButton
@@ -237,7 +238,7 @@ const PromptBadge = (props: { promptId: string; projectId: string }) => {
   if (prompt.isLoading || !prompt.data) return null;
   return (
     <Link
-      href={`/project/${props.projectId}/prompts/${prompt.data.name}?version=${prompt.data.version}`}
+      href={`/project/${props.projectId}/prompts/${encodeURIComponent(prompt.data.name)}?version=${prompt.data.version}`}
     >
       <Badge>
         Prompt: {prompt.data.name}

web/src/components/trace/TracePreview.tsx

@@ -9,7 +9,6 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
-import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -19,6 +18,7 @@ import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
+import { AnnotateButton } from "@/src/features/manual-scoring/components/AnnotateButton";
 
 export const TracePreview = ({
   trace,
@@ -54,13 +54,13 @@ export const TracePreview = ({
           <TabsList className="bg-background py-0">
             <TabsTrigger
               value="preview"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Preview
             </TabsTrigger>
             <TabsTrigger
               value="scores"
-              className="data-[state=active]:border-primary-accent h-full rounded-none border-b-4 border-transparent data-[state=active]:shadow-none"
+              className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
             >
               Scores
             </TabsTrigger>
@@ -95,7 +95,7 @@ export const TracePreview = ({
             </div>
           </div>
           <div className="flex flex-wrap gap-2">
-            <ManualScoreButton
+            <AnnotateButton
               projectId={trace.projectId}
               traceId={trace.id}
               scores={scores}

web/src/components/ui/drawer.tsx

@@ -0,0 +1,129 @@
+"use client";
+
+import * as React from "react";
+import { Drawer as DrawerPrimitive } from "vaul";
+
+import { cn } from "@/src/utils/tailwind";
+import { useMediaQuery } from "react-responsive";
+
+// https://tailwindcss.com/docs/responsive-design
+const TAILWIND_MD_MEDIA_QUERY = 768;
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => {
+  const isMediumScreen = useMediaQuery({
+    query: `(min-width: ${TAILWIND_MD_MEDIA_QUERY}px)`,
+  });
+  const direction = isMediumScreen ? "right" : "bottom";
+
+  return (
+    <DrawerPrimitive.Root
+      shouldScaleBackground={shouldScaleBackground}
+      direction={direction}
+      {...props}
+    />
+  );
+};
+Drawer.displayName = "Drawer";
+
+const DrawerTrigger = DrawerPrimitive.Trigger;
+
+const DrawerPortal = DrawerPrimitive.Portal;
+
+const DrawerClose = DrawerPrimitive.Close;
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-primary/20", className)}
+    {...props}
+  />
+));
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName;
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background md:inset-x-auto md:inset-y-0 md:right-0 md:mt-0 md:h-full md:w-2/5 md:rounded-l-[10px] md:rounded-t-[0px] lg:w-1/3 xl:w-1/4 2xl:w-1/4",
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+));
+DrawerContent.displayName = "DrawerContent";
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+);
+DrawerHeader.displayName = "DrawerHeader";
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+);
+DrawerFooter.displayName = "DrawerFooter";
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName;
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+));
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName;
+
+export {
+  Drawer,
+  DrawerPortal,
+  DrawerOverlay,
+  DrawerTrigger,
+  DrawerClose,
+  DrawerContent,
+  DrawerHeader,
+  DrawerFooter,
+  DrawerTitle,
+  DrawerDescription,
+};

web/src/components/ui/popover.tsx

@@ -28,4 +28,6 @@ const PopoverContent = React.forwardRef<
 ));
 PopoverContent.displayName = PopoverPrimitive.Content.displayName;
 
-export { Popover, PopoverTrigger, PopoverContent };
+const PopoverClose = PopoverPrimitive.Close;
+
+export { Popover, PopoverTrigger, PopoverContent, PopoverClose };

web/src/components/ui/toggle-group.tsx

@@ -0,0 +1,60 @@
+"use client";
+
+import * as React from "react";
+import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group";
+import { type VariantProps } from "class-variance-authority";
+import { cn } from "@/src/utils/tailwind";
+import { toggleVariants } from "@/src/components/ui/toggle";
+
+const ToggleGroupContext = React.createContext<
+  VariantProps<typeof toggleVariants>
+>({
+  size: "default",
+  variant: "default",
+});
+
+const ToggleGroup = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Root> &
+    VariantProps<typeof toggleVariants>
+>(({ className, variant, size, children, ...props }, ref) => (
+  <ToggleGroupPrimitive.Root
+    ref={ref}
+    className={cn("flex items-center justify-center gap-1", className)}
+    {...props}
+  >
+    <ToggleGroupContext.Provider value={{ variant, size }}>
+      {children}
+    </ToggleGroupContext.Provider>
+  </ToggleGroupPrimitive.Root>
+));
+
+ToggleGroup.displayName = ToggleGroupPrimitive.Root.displayName;
+
+const ToggleGroupItem = React.forwardRef<
+  React.ElementRef<typeof ToggleGroupPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Item> &
+    VariantProps<typeof toggleVariants>
+>(({ className, children, variant, size, ...props }, ref) => {
+  const context = React.useContext(ToggleGroupContext);
+
+  return (
+    <ToggleGroupPrimitive.Item
+      ref={ref}
+      className={cn(
+        toggleVariants({
+          variant: context.variant || variant,
+          size: context.size || size,
+        }),
+        className,
+      )}
+      {...props}
+    >
+      {children}
+    </ToggleGroupPrimitive.Item>
+  );
+});
+
+ToggleGroupItem.displayName = ToggleGroupPrimitive.Item.displayName;
+
+export { ToggleGroup, ToggleGroupItem };

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.47.2";
+export const VERSION = "v2.49.0";

web/src/ee/features/playground/page/components/JumpToPlaygroundButton.tsx

@@ -1,24 +1,24 @@
 import { Terminal } from "lucide-react";
 import Link from "next/link";
+import { useEffect, useState } from "react";
+import { z } from "zod";
 
+import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import { Button } from "@/src/components/ui/button";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { type PlaygroundCache } from "@/src/ee/features/playground/page/types";
+import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import {
   ChatMessageRole,
-  supportedModels as playgroundSupportedModels,
   type Observation,
   type Prompt,
+  supportedModels as playgroundSupportedModels,
   type UIModelParams,
   ZodModelConfig,
 } from "@langfuse/shared";
-import { useEffect, useState } from "react";
-import { PromptType } from "@/src/features/prompts/server/utils/validation";
-import { ChatMessageListSchema } from "@/src/features/prompts/components/NewPromptForm/validation";
-import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
 
 type JumpToPlaygroundButtonProps = (
   | {
@@ -76,9 +76,19 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
   );
 };
 
+const ParsedChatMessageListSchema = z.array(
+  z.object({
+    role: z.nativeEnum(ChatMessageRole),
+    content: z.union([
+      z.string(),
+      z.any().transform((v) => JSON.stringify(v, null, 2)),
+    ]),
+  }),
+);
+
 const parsePrompt = (prompt: Prompt): PlaygroundCache => {
   if (prompt.type === PromptType.Chat) {
-    const parsedMessages = ChatMessageListSchema.safeParse(prompt.prompt);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(prompt.prompt);
 
     return parsedMessages.success ? { messages: parsedMessages.data } : null;
   } else {
@@ -109,14 +119,16 @@ const parseGeneration = (generation: Observation): PlaygroundCache => {
   }
 
   if (typeof input === "object") {
-    const parsedMessages = ChatMessageListSchema.safeParse(input);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(input);
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };
   }
 
   if (typeof input === "object" && "messages" in input) {
-    const parsedMessages = ChatMessageListSchema.safeParse(input["messages"]);
+    const parsedMessages = ParsedChatMessageListSchema.safeParse(
+      input["messages"],
+    );
 
     if (parsedMessages.success)
       return { messages: parsedMessages.data, modelParams };

web/src/ee/features/playground/page/components/SaveToPromptButton.tsx

@@ -49,8 +49,7 @@ export const SaveToPromptButton: React.FC = () => {
       )
       .data?.prompts.filter((prompt) => prompt.type === PromptType.Chat)
       .map((prompt) => ({
-        label:
-          prompt.name.slice(0, 20) + (prompt.name.length > 25 ? "..." : ""),
+        label: prompt.name,
         value: prompt.id,
       })) ?? [];
 
@@ -126,7 +125,9 @@ export const SaveToPromptButton: React.FC = () => {
                         : "opacity-0",
                     )}
                   />
-                  {promptName.label}
+                  <span className="overflow-hidden text-ellipsis whitespace-nowrap">
+                    {promptName.label}
+                  </span>
                 </CommandItem>
               ))}
             </CommandList>

web/src/features/datasets/components/DatasetRunsTable.tsx

@@ -12,7 +12,7 @@ import { useEffect } from "react";
 import { usdFormatter } from "../../../utils/numbers";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
-import { type Prisma } from "@langfuse/shared";
+import { ScoreDataType, type Prisma } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 
@@ -120,6 +120,7 @@ export function DatasetRunsTable(props: {
             scores={Object.entries(scores).map(([k, v]) => ({
               name: k,
               value: v,
+              dataType: ScoreDataType.NUMERIC, // numeric and boolean values treated as numeric
             }))}
             variant="headings"
           />

web/src/features/datasets/server/dataset-router.ts

@@ -167,6 +167,7 @@ export const datasetRouter = createTRPCRouter({
                 JOIN traces t ON t.id = s.trace_id
               WHERE 
                 t.project_id = ${input.projectId}
+                AND s.data_type != 'CATEGORICAL'
                 AND ri.dataset_run_id = runs.id
               GROUP BY s.name
             ) s
@@ -529,18 +530,6 @@ export const datasetRouter = createTRPCRouter({
             },
           },
         },
-        include: {
-          observation: {
-            select: {
-              id: true,
-            },
-          },
-          trace: {
-            select: {
-              id: true,
-            },
-          },
-        },
         orderBy: {
           createdAt: "desc",
         },

web/src/features/filters/components/multi-select.tsx

@@ -104,7 +104,6 @@ export function MultiSelect({
                         selectedValues.add(option.value);
                       }
                       const filterValues = Array.from(selectedValues);
-
                       onValueChange(filterValues.length ? filterValues : []);
                     }}
                   >

web/src/features/filters/hooks/useFilterState.ts

@@ -15,6 +15,7 @@ import {
   withDefault,
 } from "use-query-params";
 import { promptsTableCols } from "@/src/server/api/definitions/promptsTable";
+import { usersTableCols } from "@/src/server/api/definitions/usersTable";
 
 const DEBUG_QUERY_STATE = false;
 
@@ -107,6 +108,7 @@ const tableCols = {
   sessions: sessionsViewCols,
   scores: scoresTableCols,
   prompts: promptsTableCols,
+  users: usersTableCols,
   dashboard: [
     { id: "traceName", name: "Trace Name" },
     { id: "tags", name: "Tags" },

web/src/features/manual-scoring/components/AnnotateButton.tsx

@@ -0,0 +1,807 @@
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import React from "react";
+import { Button } from "@/src/components/ui/button";
+import {
+  LockIcon,
+  MessageCircleMore,
+  MessageCircle,
+  X,
+  SquarePen,
+} from "lucide-react";
+import {
+  type ControllerRenderProps,
+  useFieldArray,
+  useForm,
+} from "react-hook-form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Drawer,
+  DrawerContent,
+  DrawerHeader,
+  DrawerTrigger,
+} from "@/src/components/ui/drawer";
+import { ScoreDataType, type Score, type ScoreConfig } from "@langfuse/shared";
+import { z } from "zod";
+import { Input } from "@/src/components/ui/input";
+import {
+  Popover,
+  PopoverClose,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { api } from "@/src/utils/api";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { Textarea } from "@/src/components/ui/textarea";
+import { HoverCardContent } from "@radix-ui/react-hover-card";
+import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
+import { ScoreConfigDetails } from "@/src/features/manual-scoring/components/ScoreConfigDetails";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
+import {
+  isNumericDataType,
+  isPresent,
+  isScoreUnsaved,
+} from "@/src/features/manual-scoring/lib/helpers";
+import { getDefaultScoreData } from "@/src/features/manual-scoring/lib/getDefaultScoreData";
+import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";
+import Header from "@/src/components/layouts/header";
+import { MultiSelectKeyValues } from "@/src/features/manual-scoring/components/multi-select-key-values";
+import { CommandItem } from "@/src/components/ui/command";
+import { useRouter } from "next/router";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+
+const AnnotationScoreDataSchema = z.object({
+  name: z.string(),
+  scoreId: z.string().optional(),
+  value: z.number().nullable().optional(),
+  stringValue: z.string().optional(),
+  dataType: z.nativeEnum(ScoreDataType),
+  configId: z.string().optional(),
+  comment: z.string().optional(),
+});
+
+const AnnotateFormSchema = z.object({
+  scoreData: z.array(AnnotationScoreDataSchema),
+});
+
+type AnnotateFormSchemaType = z.infer<typeof AnnotateFormSchema>;
+type AnnotationScoreSchemaType = z.infer<typeof AnnotationScoreDataSchema>;
+type ConfigCategory = {
+  label: string;
+  value: string;
+};
+
+export function AnnotateButton({
+  traceId,
+  scores,
+  observationId,
+  projectId,
+  variant = "button",
+  type = "trace",
+  source = "TraceDetail",
+}: {
+  traceId: string;
+  scores: Score[];
+  observationId?: string;
+  projectId: string;
+  variant?: "button" | "badge";
+  type?: "trace" | "observation" | "session";
+  source?: "TraceDetail" | "SessionDetail";
+}) {
+  const capture = usePostHogClientCapture();
+  const hasAccess = useHasAccess({
+    projectId,
+    scope: "scores:CUD",
+  });
+
+  const configsData = api.scoreConfigs.all.useQuery({
+    projectId,
+  });
+
+  const configs = configsData.data?.configs ?? [];
+
+  const [emptySelectedConfigIds, setEmptySelectedConfigIds] = useLocalStorage<
+    string[]
+  >("emptySelectedConfigIds", []);
+
+  const form = useForm<z.infer<typeof AnnotateFormSchema>>({
+    resolver: zodResolver(AnnotateFormSchema),
+    defaultValues: {
+      scoreData: getDefaultScoreData({
+        scores,
+        traceId,
+        observationId,
+        emptySelectedConfigIds,
+        configs,
+      }),
+    },
+  });
+
+  const router = useRouter();
+
+  const { fields, remove, update, replace } = useFieldArray({
+    control: form.control,
+    name: "scoreData",
+  });
+
+  const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
+    onError: (error) => {
+      trpcErrorToast(error);
+    },
+    onSettled: async (data, error) => {
+      if (!data || error) return;
+
+      const { id, name, dataType, configId } = data;
+      const updatedScoreIndex = fields.findIndex(
+        (field) => field.scoreId === id,
+      );
+      update(updatedScoreIndex, {
+        name,
+        dataType,
+        configId: configId ?? undefined,
+        value: null,
+        scoreId: undefined,
+        stringValue: undefined,
+        comment: undefined,
+      });
+
+      await Promise.all([
+        utils.scores.invalidate(),
+        utils.traces.invalidate(),
+        utils.sessions.invalidate(),
+      ]);
+    },
+  });
+
+  const utils = api.useUtils();
+
+  const onSettledUpsert = async (data?: Score, error?: unknown) => {
+    if (!data || error) return;
+
+    const { id, value, stringValue, name, dataType, configId, comment } = data;
+    const updatedScoreIndex = fields.findIndex(
+      (field) => field.configId === configId,
+    );
+
+    update(updatedScoreIndex, {
+      value,
+      name,
+      dataType,
+      scoreId: id,
+      stringValue: stringValue ?? undefined,
+      configId: configId ?? undefined,
+      comment: comment ?? undefined,
+    });
+
+    await Promise.all([
+      utils.scores.invalidate(),
+      utils.traces.invalidate(),
+      utils.sessions.invalidate(),
+    ]);
+  };
+
+  const mutCreateScores = api.scores.createAnnotationScore.useMutation({
+    onError: (error) => {
+      trpcErrorToast(error);
+    },
+    onSettled: onSettledUpsert,
+  });
+
+  const mutUpdateScores = api.scores.updateAnnotationScore.useMutation({
+    onError: (error) => {
+      trpcErrorToast(error);
+    },
+    onSettled: onSettledUpsert,
+  });
+
+  if (!hasAccess && variant === "badge") return null;
+
+  function handleOnCheckedChange(
+    values: Record<string, string>[],
+    changedValueId?: string,
+  ) {
+    if (values.length === 0) {
+      const populatedScoreFields = fields.filter(({ scoreId }) => !!scoreId);
+      replace(populatedScoreFields);
+      setEmptySelectedConfigIds(
+        populatedScoreFields
+          .filter(({ configId }) => !!configId)
+          .map(({ configId }) => configId as string),
+      );
+      return;
+    }
+    if (!changedValueId) return;
+
+    const configToChange = configs.find(({ id }) => id === changedValueId);
+    if (!configToChange) return;
+    const { id, name, dataType } = configToChange;
+
+    const index = fields.findIndex(({ configId }) => configId === id);
+
+    if (index === -1) {
+      replace([
+        ...fields,
+        {
+          name,
+          dataType,
+          configId: id,
+        },
+      ]);
+      setEmptySelectedConfigIds([...emptySelectedConfigIds, changedValueId]);
+    } else {
+      remove(index);
+      setEmptySelectedConfigIds(
+        emptySelectedConfigIds.filter((id) => id !== changedValueId),
+      );
+    }
+  }
+
+  function handleOnValueChange(
+    score: AnnotationScoreSchemaType,
+    index: number,
+    configCategories: ConfigCategory[],
+  ): ((value: string) => void) | undefined {
+    return async (stringValue) => {
+      const selectedCategory = configCategories.find(
+        ({ label }) => label === stringValue,
+      );
+      if (selectedCategory) {
+        const newValue = Number(selectedCategory.value);
+
+        update(index, {
+          ...score,
+          value: newValue,
+        });
+
+        if (!!stringValue) {
+          if (!!score.scoreId) {
+            await mutUpdateScores.mutateAsync({
+              projectId,
+              ...score,
+              id: score.scoreId,
+              value: newValue,
+              stringValue,
+            });
+            capture("score:update", {
+              type: type,
+              source: source,
+              dataType: score.dataType,
+            });
+          } else {
+            await mutCreateScores.mutateAsync({
+              projectId,
+              traceId,
+              ...score,
+              observationId,
+              value: newValue,
+              stringValue,
+            });
+            capture("score:create", {
+              type: type,
+              source: source,
+              dataType: score.dataType,
+            });
+          }
+        }
+      }
+    };
+  }
+
+  function handleCommentUpdate({
+    field,
+    score,
+    comment,
+  }: {
+    field: ControllerRenderProps<
+      AnnotateFormSchemaType,
+      `scoreData.${number}.comment`
+    >;
+    score: AnnotationScoreSchemaType;
+    comment?: string | null;
+  }): React.MouseEventHandler<HTMLButtonElement> | undefined {
+    return async () => {
+      const { value, scoreId } = score;
+      if (!!field.value && !!scoreId && isPresent(value)) {
+        await mutUpdateScores.mutateAsync({
+          projectId,
+          ...score,
+          value,
+          id: scoreId,
+          comment,
+        });
+        capture(comment ? "score:update_comment" : "score:delete_comment", {
+          type: type,
+          source: source,
+        });
+      }
+    };
+  }
+
+  function handleOnBlur({
+    config,
+    field,
+    index,
+    score,
+  }: {
+    config: ScoreConfig;
+    field: ControllerRenderProps<
+      AnnotateFormSchemaType,
+      `scoreData.${number}.value`
+    >;
+    index: number;
+    score: AnnotationScoreSchemaType;
+  }): React.FocusEventHandler<HTMLInputElement> | undefined {
+    return async () => {
+      const { maxValue, minValue, dataType } = config;
+
+      if (isNumericDataType(dataType)) {
+        if (
+          (isPresent(maxValue) && Number(field.value) > maxValue) ||
+          (isPresent(minValue) && Number(field.value) < minValue)
+        ) {
+          form.setError(`scoreData.${index}.value`, {
+            type: "custom",
+            message: `Not in range: [${minValue ?? "-∞"},${maxValue ?? "∞"}]`,
+          });
+          return;
+        }
+      }
+
+      form.clearErrors(`scoreData.${index}.value`);
+
+      if (isPresent(field.value)) {
+        if (!!score.scoreId) {
+          await mutUpdateScores.mutateAsync({
+            projectId,
+            ...score,
+            value: Number(field.value),
+            id: score.scoreId,
+          });
+          capture("score:update", {
+            type: type,
+            source: source,
+            dataType: score.dataType,
+          });
+        } else {
+          await mutCreateScores.mutateAsync({
+            projectId,
+            traceId,
+            ...score,
+            observationId,
+            value: Number(field.value),
+          });
+          capture("score:create", {
+            type: type,
+            source: source,
+            dataType: score.dataType,
+          });
+        }
+      }
+    };
+  }
+
+  return (
+    <Drawer>
+      <DrawerTrigger asChild>
+        {variant === "button" ? (
+          <Button
+            variant="secondary"
+            disabled={!hasAccess}
+            onClick={() =>
+              capture(
+                Boolean(scores.length)
+                  ? "score:update_form_open"
+                  : "score:create_form_open",
+                {
+                  type: type,
+                  source: source,
+                },
+              )
+            }
+          >
+            {!hasAccess ? (
+              <LockIcon className="mr-2 h-3 w-3" />
+            ) : (
+              <SquarePen className="mr-2 h-5 w-5" />
+            )}
+            <span>Annotate</span>
+          </Button>
+        ) : (
+          <Button
+            className="h-6 rounded-full px-3 text-xs"
+            onClick={() =>
+              capture(
+                Boolean(scores.length)
+                  ? "score:update_form_open"
+                  : "score:create_form_open",
+                {
+                  type: type,
+                  source: source,
+                },
+              )
+            }
+          >
+            Annotate
+          </Button>
+        )}
+      </DrawerTrigger>
+      <DrawerContent className="h-1/3">
+        <div className="mx-auto w-full overflow-y-auto md:max-h-full">
+          <DrawerHeader className="sticky top-0 z-10 bg-background">
+            <Header
+              title="Annotate"
+              level="h3"
+              help={{
+                description: `Annotate ${observationId ? "observation" : "trace"} with scores to capture human evaluation across different dimensions.`,
+                href: "https://langfuse.com/docs/scores/manually",
+              }}
+            ></Header>
+            <div className="grid grid-flow-col items-center">
+              <MultiSelectKeyValues
+                title="Value"
+                align="end"
+                items="empty scores"
+                className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
+                onValueChange={handleOnCheckedChange}
+                options={configs.map((config) => ({
+                  key: config.id,
+                  value: config.name,
+                  disabled: fields.some(
+                    (field) => !!field.scoreId && field.configId === config.id,
+                  ),
+                }))}
+                values={fields
+                  .filter((field) => !!field.configId)
+                  .map((field) => ({
+                    value: field.name,
+                    key: field.configId as string,
+                  }))}
+                controlButtons={
+                  <CommandItem
+                    onSelect={() => {
+                      capture("score_configs:manage_configs_item_click", {
+                        type: type,
+                        source: source,
+                      });
+                      router.push(`/project/${projectId}/settings`);
+                    }}
+                  >
+                    Manage score configs
+                  </CommandItem>
+                }
+              />
+            </div>
+          </DrawerHeader>
+          <Form {...form}>
+            <form className="flex flex-col gap-4">
+              <div className="grid grid-flow-row gap-2 px-4">
+                <FormField
+                  control={form.control}
+                  name="scoreData"
+                  render={() => (
+                    <>
+                      {fields.map((score, index) => {
+                        const config = configs.find(
+                          (config) => config.id === score.configId,
+                        );
+                        if (!config) return null;
+
+                        return (
+                          <div
+                            key={score.id}
+                            className="grid w-full grid-cols-[1fr,2fr] items-center gap-8 text-left"
+                          >
+                            <div className="grid h-full grid-cols-[1fr,auto] items-center">
+                              {config.description ||
+                              isPresent(config.maxValue) ||
+                              isPresent(config.minValue) ? (
+                                <HoverCard>
+                                  <HoverCardTrigger asChild>
+                                    <span className="line-clamp-2 break-words text-xs font-medium underline decoration-muted-gray decoration-dashed underline-offset-2">
+                                      {score.name}
+                                    </span>
+                                  </HoverCardTrigger>
+                                  <HoverCardContent>
+                                    <ScoreConfigDetails config={config} />
+                                  </HoverCardContent>
+                                </HoverCard>
+                              ) : (
+                                <span
+                                  className="line-clamp-2 break-words text-xs font-medium"
+                                  title={score.name}
+                                >
+                                  {score.name}
+                                </span>
+                              )}
+                              <Popover>
+                                <PopoverTrigger asChild>
+                                  <Button
+                                    variant="link"
+                                    type="button"
+                                    size="xs"
+                                    title="Add or view score comment"
+                                    className="h-full items-start px-0 pl-1 disabled:text-primary/50 disabled:opacity-100"
+                                    disabled={isScoreUnsaved(score.scoreId)}
+                                  >
+                                    {score.comment ? (
+                                      <MessageCircleMore className="h-4 w-4" />
+                                    ) : (
+                                      <MessageCircle className="h-4 w-4" />
+                                    )}
+                                  </Button>
+                                </PopoverTrigger>
+                                <PopoverContent>
+                                  <FormField
+                                    control={form.control}
+                                    name={`scoreData.${index}.comment`}
+                                    render={({ field }) => (
+                                      <FormItem className="space-y-4">
+                                        <FormLabel className="text-sm">
+                                          Comment (optional)
+                                        </FormLabel>
+                                        {!!field.value &&
+                                          field.value !== score.comment && (
+                                            <HoverCard>
+                                              <HoverCardTrigger asChild>
+                                                <span className="ml-2 mr-2 rounded-sm bg-input p-1 text-xs">
+                                                  Draft
+                                                </span>
+                                              </HoverCardTrigger>
+                                              <HoverCardContent side="top">
+                                                {!!score.comment && (
+                                                  <div className="mb-4 max-w-48 rounded border bg-background p-2 shadow-sm">
+                                                    <p className="text-xs">
+                                                      Saved comment:{" "}
+                                                      {score.comment}
+                                                    </p>
+                                                  </div>
+                                                )}
+                                              </HoverCardContent>
+                                            </HoverCard>
+                                          )}
+                                        <FormControl>
+                                          <>
+                                            <Textarea
+                                              {...field}
+                                              className="text-xs"
+                                              value={field.value || ""}
+                                            />
+                                            {field.value !== score.comment && (
+                                              <div className="grid w-full grid-cols-[1fr,1fr] gap-2">
+                                                <Button
+                                                  variant="secondary"
+                                                  type="button"
+                                                  size="sm"
+                                                  className="text-xs"
+                                                  disabled={!field.value}
+                                                  loading={
+                                                    mutUpdateScores.isLoading
+                                                  }
+                                                  onClick={handleCommentUpdate({
+                                                    field,
+                                                    score,
+                                                    comment: field.value,
+                                                  })}
+                                                >
+                                                  Save
+                                                </Button>
+                                                <PopoverClose asChild>
+                                                  <Button
+                                                    variant="secondary"
+                                                    type="button"
+                                                    size="sm"
+                                                    className="text-xs"
+                                                    disabled={!field.value}
+                                                    onClick={() => {
+                                                      form.setValue(
+                                                        `scoreData.${index}.comment`,
+                                                        score.comment ?? "",
+                                                      );
+                                                    }}
+                                                  >
+                                                    Discard
+                                                  </Button>
+                                                </PopoverClose>
+                                              </div>
+                                            )}
+                                            {field.value === score.comment && (
+                                              <div className="flex justify-end">
+                                                <Button
+                                                  variant="destructive"
+                                                  type="button"
+                                                  size="sm"
+                                                  className="text-xs"
+                                                  disabled={
+                                                    !field.value ||
+                                                    !score.comment
+                                                  }
+                                                  loading={
+                                                    mutUpdateScores.isLoading
+                                                  }
+                                                  onClick={handleCommentUpdate({
+                                                    field,
+                                                    score,
+                                                    comment: null,
+                                                  })}
+                                                >
+                                                  Delete
+                                                </Button>
+                                              </div>
+                                            )}
+                                          </>
+                                        </FormControl>
+                                        <FormMessage className="text-xs" />
+                                      </FormItem>
+                                    )}
+                                  />
+                                </PopoverContent>
+                              </Popover>
+                            </div>
+                            <div className="grid grid-cols-[11fr,1fr]">
+                              <FormField
+                                control={form.control}
+                                name={`scoreData.${index}.value`}
+                                render={({ field }) => (
+                                  <FormItem>
+                                    <FormControl>
+                                      {isNumericDataType(score.dataType) ? (
+                                        <Input
+                                          {...field}
+                                          value={field.value ?? undefined}
+                                          type="number"
+                                          className="text-xs"
+                                          onBlur={handleOnBlur({
+                                            config,
+                                            field,
+                                            index,
+                                            score,
+                                          })}
+                                          onKeyDown={handleOnKeyDown}
+                                        />
+                                      ) : config.categories &&
+                                        (
+                                          (config.categories as ConfigCategory[]) ??
+                                          []
+                                        ).length > 3 ? (
+                                        <Select
+                                          defaultValue={score.stringValue}
+                                          onValueChange={handleOnValueChange(
+                                            score,
+                                            index,
+                                            (config.categories as ConfigCategory[]) ??
+                                              [],
+                                          )}
+                                        >
+                                          <SelectTrigger>
+                                            <div className="text-xs">
+                                              <SelectValue placeholder="Select category" />
+                                            </div>
+                                          </SelectTrigger>
+                                          <SelectContent>
+                                            {(
+                                              (config.categories as ConfigCategory[]) ??
+                                              []
+                                            ).map(
+                                              (category: ConfigCategory) => (
+                                                <SelectItem
+                                                  key={category.value}
+                                                  value={category.label}
+                                                  className="text-xs"
+                                                >
+                                                  {category.label}
+                                                </SelectItem>
+                                              ),
+                                            )}
+                                          </SelectContent>
+                                        </Select>
+                                      ) : (
+                                        <ToggleGroup
+                                          type="single"
+                                          defaultValue={score.stringValue}
+                                          className={`grid grid-cols-${((config.categories as ConfigCategory[]) ?? [])?.length}`}
+                                          onValueChange={handleOnValueChange(
+                                            score,
+                                            index,
+                                            (config.categories as ConfigCategory[]) ??
+                                              [],
+                                          )}
+                                        >
+                                          {(
+                                            (config.categories as ConfigCategory[]) ??
+                                            []
+                                          ).map((category: ConfigCategory) => (
+                                            <ToggleGroupItem
+                                              key={category.value}
+                                              value={category.label}
+                                              variant="outline"
+                                              className="grid grid-flow-col gap-1 text-nowrap px-1 text-xs font-normal"
+                                            >
+                                              <span
+                                                className="truncate"
+                                                title={category.label}
+                                              >
+                                                {category.label}
+                                              </span>
+                                              <span className="text-primary/60">{`(${category.value})`}</span>
+                                            </ToggleGroupItem>
+                                          ))}
+                                        </ToggleGroup>
+                                      )}
+                                    </FormControl>
+                                    <FormMessage className="text-xs" />
+                                  </FormItem>
+                                )}
+                              />
+                              <Button
+                                variant="link"
+                                type="button"
+                                className="px-0 pl-1"
+                                title="Delete score from trace/observation"
+                                disabled={isScoreUnsaved(score.scoreId)}
+                                loading={mutDeleteScore.isLoading}
+                                onClick={async () => {
+                                  if (score.scoreId) {
+                                    await mutDeleteScore.mutateAsync({
+                                      id: score.scoreId,
+                                      projectId,
+                                    });
+                                    capture("score:delete", {
+                                      type: type,
+                                      source: source,
+                                    });
+                                    form.clearErrors(
+                                      `scoreData.${index}.value`,
+                                    );
+                                  }
+                                }}
+                              >
+                                <X className="h-4 w-4" />
+                              </Button>
+                            </div>
+                          </div>
+                        );
+                      })}
+                    </>
+                  )}
+                />
+              </div>
+            </form>
+          </Form>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
+
+function handleOnKeyDown(e: React.KeyboardEvent<HTMLInputElement>) {
+  if (e.key === "Enter") {
+    e.preventDefault();
+    const form = e.currentTarget.form;
+    if (!form) return;
+
+    const currentTabIndex = e.currentTarget.tabIndex;
+    const nextElement = form.querySelector(
+      `[tabindex="${currentTabIndex + 1}"]`,
+    );
+
+    if (nextElement instanceof HTMLElement) {
+      nextElement.focus();
+    } else {
+      e.currentTarget.blur();
+    }
+  }
+}

web/src/features/manual-scoring/components/CreateScoreConfigButton.tsx

@@ -0,0 +1,423 @@
+import React, { useState } from "react";
+import { Button } from "@/src/components/ui/button";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { z } from "zod";
+import {
+  Dialog,
+  DialogContent,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
+import { PlusIcon, Trash2 } from "lucide-react";
+import { useFieldArray, useForm } from "react-hook-form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { Input } from "@/src/components/ui/input";
+import { ScoreDataType } from "@langfuse/shared";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { api } from "@/src/utils/api";
+import { Textarea } from "@/src/components/ui/textarea";
+import {
+  isBooleanDataType,
+  isCategoricalDataType,
+  isNumericDataType,
+} from "@/src/features/manual-scoring/lib/helpers";
+import DocPopup from "@/src/components/layouts/doc-popup";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+
+const availableDataTypes = [
+  ScoreDataType.NUMERIC,
+  ScoreDataType.CATEGORICAL,
+  ScoreDataType.BOOLEAN,
+] as const;
+
+const category = z.object({
+  label: z.string().min(1),
+  value: z.coerce.number(),
+});
+
+const formSchema = z.object({
+  name: z.string().min(1).max(35),
+  dataType: z.enum(availableDataTypes),
+  minValue: z.coerce.number().optional(),
+  maxValue: z.coerce.number().optional(),
+  categories: z.array(category).optional(),
+  description: z.string().optional(),
+});
+
+export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
+  const [open, setOpen] = useState(false);
+  const [formError, setFormError] = useState<string | null>(null);
+  const capture = usePostHogClientCapture();
+
+  const hasAccess = useHasAccess({
+    projectId: projectId,
+    scope: "scoreConfigs:CUD",
+  });
+
+  const utils = api.useUtils();
+  const createScoreConfig = api.scoreConfigs.create.useMutation({
+    onSuccess: () => utils.scoreConfigs.invalidate(),
+    onError: (error) =>
+      setFormError(error.message ?? "An error occurred while creating config."),
+  });
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    defaultValues: {
+      dataType: ScoreDataType.NUMERIC,
+      minValue: undefined,
+      maxValue: undefined,
+      name: "",
+    },
+  });
+
+  const { fields, append, remove, replace } = useFieldArray({
+    control: form.control,
+    name: "categories",
+  });
+
+  if (!hasAccess) return null;
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    const error = validateForm(values);
+    setFormError(error);
+    if (error) return;
+
+    return createScoreConfig
+      .mutateAsync({
+        projectId,
+        ...values,
+      })
+      .then(() => {
+        capture("score_configs:create_form_submit", {
+          dataType: values.dataType,
+        });
+        form.reset();
+        setOpen(false);
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  }
+
+  return (
+    <>
+      <Dialog
+        open={open}
+        onOpenChange={(v) => {
+          setOpen(v);
+          form.reset();
+        }}
+      >
+        <DialogTrigger asChild>
+          <Button variant="secondary" loading={createScoreConfig.isLoading}>
+            <PlusIcon className="-ml-0.5 mr-1.5 h-5 w-5" aria-hidden="true" />
+            Add new score config
+          </Button>
+        </DialogTrigger>
+        <DialogContent>
+          <DialogHeader>
+            <DialogTitle>Add new score config</DialogTitle>
+          </DialogHeader>
+          <Form {...form}>
+            <form
+              className="space-y-6"
+              // eslint-disable-next-line @typescript-eslint/no-misused-promises
+              onSubmit={form.handleSubmit(onSubmit)}
+            >
+              <FormField
+                control={form.control}
+                name="name"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Name</FormLabel>
+                    <FormControl>
+                      <Input
+                        {...field}
+                        type="text"
+                        onBlur={(e) => field.onChange(e.target.value.trimEnd())}
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                control={form.control}
+                name="dataType"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Data type</FormLabel>
+                    <Select
+                      defaultValue={field.value}
+                      onValueChange={(value) => {
+                        field.onChange(
+                          value as (typeof availableDataTypes)[number],
+                        );
+                        form.clearErrors();
+                        if (isNumericDataType(value as ScoreDataType)) {
+                          remove();
+                        } else {
+                          form.setValue("minValue", undefined);
+                          form.setValue("maxValue", undefined);
+                          if (isBooleanDataType(value as ScoreDataType)) {
+                            replace([
+                              { label: "True", value: 1 },
+                              { label: "False", value: 0 },
+                            ]);
+                          } else {
+                            replace([{ label: "", value: 0 }]);
+                          }
+                        }
+                      }}
+                    >
+                      <FormControl>
+                        <SelectTrigger>
+                          <SelectValue placeholder="Select a data type" />
+                        </SelectTrigger>
+                      </FormControl>
+                      <SelectContent>
+                        {availableDataTypes.map((role) => (
+                          <SelectItem value={role} key={role}>
+                            {role}
+                          </SelectItem>
+                        ))}
+                      </SelectContent>
+                    </Select>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              {isNumericDataType(form.getValues("dataType")) ? (
+                <>
+                  <FormField
+                    control={form.control}
+                    name="minValue"
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormLabel>Minimum (optional)</FormLabel>
+                        <FormControl>
+                          <Input {...field} type="number" />
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+                  <FormField
+                    control={form.control}
+                    name="maxValue"
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormLabel>Maximum (optional)</FormLabel>
+                        <FormControl>
+                          <Input {...field} type="number" />
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+                </>
+              ) : (
+                <div className="grid grid-flow-row gap-2">
+                  <FormField
+                    control={form.control}
+                    name="categories"
+                    render={() => (
+                      <>
+                        {fields.length > 0 && (
+                          <div className="mb-2 grid grid-cols-[1fr,3fr] items-center gap-2 text-left sm:grid-cols-[1fr,7fr]">
+                            <FormLabel className="grid grid-flow-col">
+                              Value
+                              <DocPopup
+                                description={`This is how the ${
+                                  isCategoricalDataType(
+                                    form.getValues("dataType"),
+                                  )
+                                    ? "category"
+                                    : "boolean"
+                                } label is mapped to an integer value internally.`}
+                                size="xs"
+                              ></DocPopup>
+                            </FormLabel>
+                            <FormLabel>Label</FormLabel>
+                          </div>
+                        )}
+                        {fields.map((category, index) => (
+                          <div
+                            key={`${category.id}-langfuseObject`}
+                            className="items-top mb-2 grid grid-cols-[1fr,3fr] gap-2 text-left sm:grid-cols-[1fr,7fr]"
+                          >
+                            <FormField
+                              control={form.control}
+                              name={`categories.${index}.value`}
+                              render={({ field }) => (
+                                <FormItem>
+                                  <FormControl>
+                                    <Input
+                                      {...field}
+                                      readOnly
+                                      disabled
+                                      inputMode="numeric"
+                                      className="text-center"
+                                    />
+                                  </FormControl>
+                                  <FormMessage />
+                                </FormItem>
+                              )}
+                            />
+                            <div className="grid grid-cols-[1fr,auto] gap-2">
+                              <FormField
+                                control={form.control}
+                                name={`categories.${index}.label`}
+                                render={({ field }) => (
+                                  <FormItem>
+                                    <FormControl>
+                                      <Input
+                                        {...field}
+                                        type="text"
+                                        onBlur={(e) =>
+                                          field.onChange(
+                                            e.target.value.trimEnd(),
+                                          )
+                                        }
+                                        readOnly={isBooleanDataType(
+                                          form.getValues("dataType"),
+                                        )}
+                                      />
+                                    </FormControl>
+                                    <FormMessage />
+                                  </FormItem>
+                                )}
+                              />
+                              {isCategoricalDataType(
+                                form.getValues("dataType"),
+                              ) && (
+                                <Button
+                                  onClick={() => remove(index)}
+                                  variant="outline"
+                                  size="icon"
+                                  disabled={
+                                    index === 0 || index !== fields.length - 1
+                                  }
+                                >
+                                  <Trash2 className="h-4 w-4" />
+                                </Button>
+                              )}
+                            </div>
+                          </div>
+                        ))}
+                        {isCategoricalDataType(form.getValues("dataType")) && (
+                          <div className="grid-cols-auto grid">
+                            <Button
+                              type="button"
+                              variant="secondary"
+                              disabled={
+                                isBooleanDataType(form.getValues("dataType")) &&
+                                fields.length === 2
+                              }
+                              onClick={() =>
+                                append({ label: "", value: fields.length })
+                              }
+                            >
+                              Add category
+                            </Button>
+                          </div>
+                        )}
+                      </>
+                    )}
+                  />
+                </div>
+              )}
+              <FormField
+                control={form.control}
+                name="description"
+                render={({ field }) => (
+                  <>
+                    <FormItem>
+                      <FormLabel>Description (optional)</FormLabel>
+                      <FormControl>
+                        <Textarea
+                          {...field}
+                          placeholder="Provide an optional description of the score config..."
+                        />
+                      </FormControl>
+                      <FormMessage />
+                    </FormItem>
+                  </>
+                )}
+              />
+              <Button
+                type="submit"
+                className="w-full"
+                loading={form.formState.isSubmitting}
+              >
+                Submit
+              </Button>
+            </form>
+            {formError ? (
+              <p className="text-red text-center">
+                <span className="font-bold">Error:</span> {formError}
+              </p>
+            ) : null}
+          </Form>
+        </DialogContent>
+      </Dialog>
+    </>
+  );
+}
+
+function validateForm(values: z.infer<typeof formSchema>): string | null {
+  if (isNumericDataType(values.dataType)) {
+    if (
+      !!values.maxValue &&
+      !!values.minValue &&
+      values.maxValue <= values.minValue
+    ) {
+      return "Maximum value must be greater than Minimum value.";
+    }
+  } else if (isCategoricalDataType(values.dataType)) {
+    if (!values.categories || values.categories.length === 0) {
+      return "At least one category is required for categorical data types.";
+    }
+  } else if (isBooleanDataType(values.dataType)) {
+    if (values.categories?.length !== 2)
+      return "Boolean data type must have exactly 2 categories.";
+    const isBooleanCategoryInvalid = values.categories?.some(
+      (category) => category.value !== 0 && category.value !== 1,
+    );
+    if (isBooleanCategoryInvalid)
+      return "Boolean data type must have categories with values 0 and 1.";
+  }
+
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of values.categories || []) {
+    if (uniqueNames.has(category.label)) {
+      return "Category names must be unique.";
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      return "Category values must be unique.";
+    }
+    uniqueValues.add(category.value);
+  }
+
+  return null;
+}

web/src/features/manual-scoring/components/ManualScoreButton.tsx

@@ -1,248 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import {
-  Dialog,
-  DialogContent,
-  DialogHeader,
-  DialogTitle,
-  DialogTrigger,
-} from "@/src/components/ui/dialog";
-import {
-  Form,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormDescription,
-  FormControl,
-  FormMessage,
-} from "@/src/components/ui/form";
-import { Textarea } from "@/src/components/ui/textarea";
-import { api } from "@/src/utils/api";
-import { zodResolver } from "@hookform/resolvers/zod";
-import { type Score } from "@langfuse/shared/src/db";
-import { useState } from "react";
-import * as z from "zod";
-
-import { useForm } from "react-hook-form";
-import { Slider } from "@/src/components/ui/slider";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { LockIcon } from "lucide-react";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-
-const SCORE_NAME = "manual-score";
-
-const formSchema = z.object({
-  score: z.number(),
-  comment: z.string().optional(),
-});
-
-export function ManualScoreButton({
-  traceId,
-  scores,
-  observationId,
-  projectId,
-  variant = "button",
-}: {
-  traceId: string;
-  scores: Score[];
-  observationId?: string;
-  projectId: string;
-  variant?: "button" | "badge";
-}) {
-  const capture = usePostHogClientCapture();
-  const hasAccess = useHasAccess({
-    projectId,
-    scope: "scores:CUD",
-  });
-  const score = scores.find(
-    (s) =>
-      s.name === SCORE_NAME &&
-      s.traceId === traceId &&
-      (observationId !== undefined
-        ? s.observationId === observationId
-        : s.observationId === null),
-  );
-
-  // variant === "badge" type is session
-  // if observationId is defined type is observation
-  // else type is trace
-  const type =
-    variant === "badge" ? "session" : observationId ? "observation" : "trace";
-  const source = type === "session" ? "SessionDetail" : "TraceDetail";
-
-  const utils = api.useUtils();
-  const onSuccess = async () => {
-    await Promise.all([
-      utils.scores.invalidate(),
-      utils.traces.invalidate(),
-      utils.sessions.invalidate(),
-    ]);
-  };
-  const mutCreateScore = api.scores.createAnnotationScore.useMutation({
-    onSuccess,
-  });
-  const mutUpdateScore = api.scores.updateAnnotationScore.useMutation({
-    onSuccess,
-  });
-  const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
-    onSuccess,
-  });
-
-  const handleDelete = async () => {
-    if (score) {
-      await mutDeleteScore.mutateAsync({ id: score.id, projectId });
-      capture("score:delete", {
-        type: type,
-        source: source,
-      });
-      onOpenChange(false);
-    }
-  };
-
-  const [open, setOpen] = useState(false);
-
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
-    defaultValues: {
-      score: 0,
-      comment: "",
-    },
-  });
-
-  const onOpenChange = (value: boolean) => {
-    if (!hasAccess) return;
-    if (!value) {
-      form.reset();
-      setOpen(false);
-    } else {
-      capture(score ? "score:update_form_open" : "score:create_form_open", {
-        type: type,
-        source: source,
-      });
-      form.setValue("score", score?.value ?? 0);
-      form.setValue("comment", score?.comment ?? "");
-      setOpen(true);
-    }
-  };
-
-  const onSubmit = async (values: z.infer<typeof formSchema>) => {
-    if (score) {
-      await mutUpdateScore.mutateAsync({
-        id: score.id,
-        value: values.score,
-        comment: values.comment,
-        projectId,
-      });
-      capture("score:update_form_submit", {
-        type: type,
-        source: source,
-      });
-    } else {
-      await mutCreateScore.mutateAsync({
-        name: SCORE_NAME,
-        value: values.score,
-        comment: values.comment,
-        traceId,
-        observationId,
-        projectId,
-      });
-      capture("score:create_form_submit", {
-        type: type,
-        source: source,
-      });
-    }
-    onOpenChange(false);
-  };
-
-  if (!hasAccess && variant === "badge") return null;
-
-  return (
-    <Dialog open={open} onOpenChange={onOpenChange}>
-      <DialogTrigger asChild>
-        {variant === "button" ? (
-          <Button variant="secondary" disabled={!hasAccess}>
-            <span>{score ? `Update score: ${score.value}` : "Add score"}</span>
-            {!hasAccess ? <LockIcon className="ml-2 h-3 w-3" /> : null}
-          </Button>
-        ) : (
-          <Button className="h-6 rounded-full px-3 text-xs">
-            {score ? "Update score" : "Add score"}
-          </Button>
-        )}
-      </DialogTrigger>
-      <DialogContent>
-        <DialogHeader>
-          <DialogTitle className="mb-5">
-            {score ? "Update Score" : "Create Score"}
-          </DialogTitle>
-        </DialogHeader>
-        <Form {...form}>
-          <form
-            // eslint-disable-next-line @typescript-eslint/no-misused-promises
-            onSubmit={form.handleSubmit(onSubmit)}
-            className="space-y-8"
-          >
-            <FormField
-              control={form.control}
-              name="score"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Score</FormLabel>
-                  <FormControl>
-                    <Slider
-                      {...field}
-                      min={-1}
-                      max={1}
-                      step={0.01}
-                      onValueChange={(value) => {
-                        if (value[0] !== undefined) field.onChange(value[0]);
-                      }}
-                      value={[field.value]}
-                      onChange={undefined}
-                    />
-                  </FormControl>
-                  <FormDescription className="flex justify-between">
-                    <span>-1 (bad)</span>
-                    <span>1 (good)</span>
-                  </FormDescription>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-            <FormField
-              control={form.control}
-              name="comment"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Comment (optional)</FormLabel>
-                  <FormControl>
-                    <Textarea {...field} />
-                  </FormControl>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-            <div className="flex justify-end space-x-4">
-              <Button type="submit" loading={form.formState.isSubmitting}>
-                {form.formState.isSubmitting
-                  ? "Loading ..."
-                  : score
-                    ? "Update"
-                    : "Create"}
-              </Button>
-              {score && (
-                <Button
-                  type="button"
-                  variant="destructive"
-                  onClick={() => void handleDelete()}
-                  loading={mutDeleteScore.isLoading}
-                >
-                  Delete
-                </Button>
-              )}
-            </div>
-          </form>
-        </Form>
-      </DialogContent>
-    </Dialog>
-  );
-}

web/src/features/manual-scoring/components/ScoreConfigDetails.tsx

@@ -0,0 +1,23 @@
+import {
+  isNumericDataType,
+  isPresent,
+} from "@/src/features/manual-scoring/lib/helpers";
+import { type ScoreConfig } from "@langfuse/shared";
+import React from "react";
+
+export function ScoreConfigDetails({ config }: { config: ScoreConfig }) {
+  const { name, description, minValue, maxValue, dataType } = config;
+  if (!description && !minValue && !maxValue) return null;
+  const isNameTruncated = name.length > 20;
+
+  return (
+    <div className="max-w-48 overflow-hidden text-wrap rounded border bg-background p-2 text-xs font-light">
+      {!!description && <p>{`Description: ${description}`}</p>}
+      {isNumericDataType(dataType) &&
+      (isPresent(minValue) || isPresent(maxValue)) ? (
+        <p>{`Range: [${minValue ?? "-∞"}, ${maxValue ?? "∞"}]`}</p>
+      ) : null}
+      {isNameTruncated && <p>{`Full name: ${name}`}</p>}
+    </div>
+  );
+}

web/src/features/manual-scoring/components/ScoreConfigSettings.tsx

@@ -0,0 +1,34 @@
+import React from "react";
+import Header from "@/src/components/layouts/header";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { CreateScoreConfigButton } from "@/src/features/manual-scoring/components/CreateScoreConfigButton";
+import { ScoreConfigsTable } from "@/src/components/table/use-cases/score-configs";
+
+export function ScoreConfigSettings({ projectId }: { projectId: string }) {
+  const hasReadAccess = useHasAccess({
+    projectId: projectId,
+    scope: "scoreConfigs:read",
+  });
+
+  if (!hasReadAccess) return null;
+
+  return (
+    <div>
+      <Header title="Score Configs" level="h3" />
+      <p className="mb-4 text-sm">
+        Score configs define which scores are available for{" "}
+        <a
+          href="https://langfuse.com/docs/scores/manually"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          annotation
+        </a>{" "}
+        in your project. Please note that all score configs are immutable.
+      </p>
+      <ScoreConfigsTable projectId={projectId} />
+      <CreateScoreConfigButton projectId={projectId} />
+    </div>
+  );
+}

web/src/features/manual-scoring/components/multi-select-key-values.tsx

@@ -0,0 +1,190 @@
+import * as React from "react";
+import { Check, ChevronDown } from "lucide-react";
+
+import { cn } from "@/src/utils/tailwind";
+import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
+import {
+  Command,
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandItem,
+  CommandList,
+  CommandSeparator,
+} from "@/src/components/ui/command";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { Separator } from "@/src/components/ui/separator";
+
+type MultiSelectOptions = {
+  value: string;
+  key?: string;
+  count?: number;
+  disabled?: boolean;
+};
+
+export function MultiSelectKeyValues<
+  T extends { key: string; value: string } | string,
+>({
+  title,
+  values,
+  onValueChange,
+  options,
+  className,
+  disabled,
+  items = "items",
+  align = "center",
+  controlButtons,
+}: {
+  title?: string;
+  values: T[];
+  onValueChange: (values: T[], changedValue?: string) => void;
+  options: MultiSelectOptions[] | readonly MultiSelectOptions[];
+  className?: string;
+  disabled?: boolean;
+  items?: string;
+  align?: "center" | "end" | "start";
+  controlButtons?: React.ReactNode;
+}) {
+  const selectedValueKeys = new Set(
+    values.map((value) => (typeof value === "string" ? value : value.key)),
+  );
+  const showClearItems = selectedValueKeys.size > 0;
+
+  function formatFilterValues(): T[] {
+    if (values.length > 0 && typeof values[0] === "string") {
+      return Array.from(selectedValueKeys) as T[];
+    }
+
+    return options
+      .filter((option) => !!option.key && selectedValueKeys.has(option.key))
+      .map((option) => ({
+        key: option.key as string,
+        value: option.value,
+      })) as T[];
+  }
+
+  return (
+    <Popover modal>
+      <PopoverTrigger asChild>
+        <Button
+          variant="outline"
+          className={cn(
+            "flex h-10 w-full items-center justify-between rounded-md border border-input bg-transparent px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+            className,
+          )}
+          disabled={disabled}
+        >
+          Select
+          <ChevronDown className="h-4 w-4 opacity-50" />
+          {selectedValueKeys.size > 0 && (
+            <>
+              <Separator orientation="vertical" className="mx-2 h-4" />
+              <Badge
+                variant="secondary"
+                className="rounded-sm px-1 font-normal lg:hidden"
+              >
+                {selectedValueKeys.size}
+              </Badge>
+              <div className="hidden space-x-1 overflow-x-auto lg:flex">
+                {selectedValueKeys.size > 2 ? (
+                  <Badge
+                    variant="secondary"
+                    className="rounded-sm px-1 font-normal"
+                  >
+                    {selectedValueKeys.size} selected
+                  </Badge>
+                ) : (
+                  options
+                    .filter((option) =>
+                      selectedValueKeys.has(option.key ?? option.value),
+                    )
+                    .map((option) => (
+                      <Badge
+                        variant="secondary"
+                        key={option.key}
+                        className="rounded-sm px-1 font-normal"
+                      >
+                        {option.value}
+                      </Badge>
+                    ))
+                )}
+              </div>
+            </>
+          )}
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[200px] p-0" align={align}>
+        <Command>
+          <CommandInput placeholder={title} />
+          <CommandList>
+            <CommandEmpty>No results found.</CommandEmpty>
+            <CommandGroup>
+              {options.map((option) => {
+                const isSelected = selectedValueKeys.has(
+                  option.key ?? option.value,
+                );
+                return (
+                  <CommandItem
+                    key={option.key ?? option.value}
+                    value={option.key ?? option.value}
+                    keywords={[option.value]}
+                    onSelect={(value) => {
+                      if (isSelected) {
+                        selectedValueKeys.delete(value);
+                      } else {
+                        selectedValueKeys.add(value);
+                      }
+                      const filterValues = formatFilterValues();
+
+                      onValueChange(
+                        filterValues.length ? filterValues : [],
+                        value,
+                      );
+                    }}
+                    disabled={option.disabled}
+                  >
+                    <div
+                      className={cn(
+                        "mr-2 flex h-4 w-4 items-center justify-center rounded-sm border border-primary",
+                        isSelected
+                          ? "bg-primary text-primary-foreground"
+                          : "opacity-50 [&_svg]:invisible",
+                        option.disabled ? "opacity-50" : null,
+                      )}
+                    >
+                      <Check className={cn("h-4 w-4")} />
+                    </div>
+                    <span className="overflow-x-scroll">{option.value}</span>
+                    {option.count !== undefined ? (
+                      <span className="ml-auto flex h-4 w-4 items-center justify-center pl-1 font-mono text-xs">
+                        {option.count}
+                      </span>
+                    ) : null}
+                  </CommandItem>
+                );
+              })}
+            </CommandGroup>
+            {controlButtons || showClearItems ? (
+              <>
+                <CommandSeparator />
+                <CommandGroup heading="Controls">
+                  {showClearItems && (
+                    <CommandItem onSelect={() => onValueChange([])}>
+                      Clear {items}
+                    </CommandItem>
+                  )}
+                  {controlButtons}
+                </CommandGroup>
+              </>
+            ) : null}
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/features/manual-scoring/lib/getDefaultScoreData.ts

@@ -0,0 +1,56 @@
+import { type Score, ScoreSource, type ScoreConfig } from "@langfuse/shared";
+
+export const getDefaultScoreData = ({
+  scores,
+  emptySelectedConfigIds,
+  configs,
+  traceId,
+  observationId,
+}: {
+  scores: Score[];
+  emptySelectedConfigIds: string[];
+  configs: ScoreConfig[];
+  traceId: string;
+  observationId?: string;
+}) => {
+  const populatedScores = scores
+    .filter(
+      (s) =>
+        s.source === ScoreSource.ANNOTATION &&
+        s.traceId === traceId &&
+        (observationId !== undefined
+          ? s.observationId === observationId
+          : s.observationId === null),
+    )
+    .map(({ id, name, value, dataType, stringValue, configId, comment }) => ({
+      scoreId: id,
+      name,
+      value,
+      dataType,
+      stringValue: stringValue ?? undefined,
+      configId: configId ?? undefined,
+      comment: comment ?? undefined,
+    }));
+
+  const populatedScoresConfigIds = new Set(
+    populatedScores.map((s) => s.configId),
+  );
+
+  const emptyScores = configs
+    .filter(
+      (c) =>
+        !populatedScoresConfigIds.has(c.id) &&
+        emptySelectedConfigIds.includes(c.id),
+    )
+    .map(({ name, dataType, id }) => ({
+      scoreId: undefined,
+      name,
+      value: undefined,
+      dataType,
+      stringValue: undefined,
+      configId: id,
+      comment: undefined,
+    }));
+
+  return [...populatedScores, ...emptyScores];
+};

web/src/features/manual-scoring/lib/helpers.ts

@@ -0,0 +1,15 @@
+import { ScoreDataType } from "@langfuse/shared";
+
+export const isNumericDataType = (dataType: ScoreDataType) =>
+  dataType === ScoreDataType.NUMERIC;
+
+export const isCategoricalDataType = (dataType: ScoreDataType) =>
+  dataType === ScoreDataType.CATEGORICAL;
+
+export const isBooleanDataType = (dataType: ScoreDataType) =>
+  dataType === ScoreDataType.BOOLEAN;
+
+export const isPresent = <T>(value: T): value is NonNullable<T> =>
+  value !== null && value !== undefined && value !== "";
+
+export const isScoreUnsaved = (scoreId?: string): boolean => !scoreId;

web/src/features/models/components/NewModelForm.tsx

@@ -28,6 +28,7 @@ import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { JsonEditor } from "@/src/components/json-editor";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import Link from "next/link";
 
 const formSchema = z.object({
   modelName: z.string().min(1),
@@ -255,11 +256,33 @@ export const NewModelForm = (props: {
             name="inputPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Input price (USD)</FormLabel>
+                <FormLabel>
+                  Input price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
-                <FormDescription>Cost per input unit.</FormDescription>
+                {field.value !== null && field.value !== "" ? (
+                  <FormDescription>
+                    <ul className="font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  </FormDescription>
+                ) : null}
                 <FormMessage />
               </FormItem>
             )}
@@ -269,11 +292,33 @@ export const NewModelForm = (props: {
             name="outputPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Output price (USD)</FormLabel>
+                <FormLabel>
+                  Output price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
-                <FormDescription>Cost per output unit.</FormDescription>
+                {field.value !== null && field.value !== "" ? (
+                  <FormDescription>
+                    <ul className="font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  </FormDescription>
+                ) : null}
                 <FormMessage />
               </FormItem>
             )}
@@ -283,12 +328,34 @@ export const NewModelForm = (props: {
             name="totalPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Total price (USD)</FormLabel>
+                <FormLabel>
+                  Total price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
                 <FormDescription>
-                  Cost per unit, if no separate input/output prices.
+                  {field.value !== null && field.value !== "" ? (
+                    <ul className="mt-2 font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  ) : (
+                    "Enter total price only if no separate input and output prices are provided."
+                  )}
                 </FormDescription>
                 <FormMessage />
               </FormItem>
@@ -328,7 +395,15 @@ export const NewModelForm = (props: {
                 Optionally, Langfuse can tokenize the input and output of a
                 generation if no unit counts are ingested. This is useful for
                 e.g. streamed OpenAI completions. For details on the supported
-                tokenizers, see the docs.
+                tokenizers, see the{" "}
+                <Link
+                  href="https://langfuse.com/docs/model-usage-and-cost"
+                  className="underline"
+                  target="_blank"
+                >
+                  docs
+                </Link>
+                .
               </FormDescription>
               <FormMessage />
             </FormItem>
@@ -346,8 +421,15 @@ export const NewModelForm = (props: {
                   onChange={field.onChange}
                 />
                 <FormDescription>
-                  The config for the tokenizer. Required for openai. See the
-                  docs for details.
+                  The config for the tokenizer. Required for openai. See the{" "}
+                  <Link
+                    href="https://langfuse.com/docs/model-usage-and-cost"
+                    className="underline"
+                    target="_blank"
+                  >
+                    docs
+                  </Link>{" "}
+                  for details.
                 </FormDescription>
                 <FormMessage />
               </FormItem>

web/src/features/posthog-analytics/usePostHogClientCapture.ts

@@ -28,12 +28,15 @@ const events = {
   ],
   generations: ["export"],
   score: [
+    "create",
+    "update",
     "delete",
     "update_form_open",
     "create_form_open",
-    "update_form_submit",
-    "create_form_submit",
+    "update_comment",
+    "delete_comment",
   ],
+  score_configs: ["create_form_submit", "manage_configs_item_click"],
   models: ["delete_button_click", "new_form_submit", "new_form_open"],
   prompts: [
     "new_form_submit",

web/src/features/prompts/components/prompt-detail.tsx

@@ -247,37 +247,3 @@ export const PromptDetail = () => {
     </div>
   );
 };
-
-export function UpdatePrompt({
-  projectId,
-  prompt,
-  isLoading,
-}: {
-  projectId: string;
-  prompt: Prompt | undefined;
-  isLoading: boolean;
-}) {
-  const hasAccess = useHasAccess({ projectId, scope: "prompts:CUD" });
-
-  const handlePromptEdit = () => {
-    void router.push(
-      `/project/${projectId}/prompts/${prompt?.id}/edit`,
-      undefined,
-      {
-        shallow: true,
-      },
-    );
-  };
-
-  return (
-    <Button
-      variant="outline"
-      size="icon"
-      onClick={() => handlePromptEdit()}
-      disabled={!hasAccess}
-      loading={isLoading}
-    >
-      <Pencil className="h-5 w-5" />
-    </Button>
-  );
-}

web/src/features/prompts/server/actions/createPrompt.ts

@@ -6,6 +6,8 @@ import { ValidationError } from "@langfuse/shared";
 import { jsonSchema } from "@/src/utils/zod";
 import { type PrismaClient } from "@langfuse/shared/src/db";
 import { LATEST_PROMPT_LABEL } from "@/src/features/prompts/constants";
+import { removeLabelsFromPreviousPromptVersions } from "@/src/features/prompts/server/utils/updatePromptLabels";
+import { updatePromptTagsOnAllVersions } from "@/src/features/prompts/server/utils/updatePromptTags";
 
 export type CreatePromptParams = CreatePromptTRPCType & {
   createdBy: string;
@@ -21,6 +23,7 @@ export const createPrompt = async ({
   config,
   createdBy,
   prisma,
+  tags,
 }: CreatePromptParams) => {
   const latestPrompt = await prisma.prompt.findFirst({
     where: { projectId, name },
@@ -35,14 +38,8 @@ export const createPrompt = async ({
 
   const finalLabels = [...labels, LATEST_PROMPT_LABEL]; // Newly created prompts are always labeled as 'latest'
 
-  const previousLabeledPrompts = await prisma.prompt.findMany({
-    where: {
-      projectId,
-      name,
-      labels: { hasSome: finalLabels },
-    },
-    orderBy: [{ version: "desc" }],
-  });
+  // If tags are undefined, use the tags from the latest prompt version
+  const finalTags = [...new Set(tags ?? latestPrompt?.tags ?? [])];
 
   const create = [
     prisma.prompt.create({
@@ -52,7 +49,7 @@ export const createPrompt = async ({
         createdBy,
         labels: [...new Set(finalLabels)], // Ensure labels are unique
         type,
-        tags: latestPrompt?.tags,
+        tags: finalTags,
         version: latestPrompt?.version ? latestPrompt.version + 1 : 1,
         project: { connect: { id: projectId } },
         config: jsonSchema.parse(config),
@@ -62,18 +59,28 @@ export const createPrompt = async ({
 
   if (finalLabels.length > 0)
     // If we're creating a new labeled prompt, we must remove those labels on previous prompts since labels are unique
-    previousLabeledPrompts.forEach((prevPrompt) => {
-      create.push(
-        prisma.prompt.update({
-          where: { id: prevPrompt.id },
-          data: {
-            labels: prevPrompt.labels.filter(
-              (prevLabel) => !finalLabels.includes(prevLabel),
-            ),
-          },
-        }),
-      );
-    });
+    create.push(
+      ...(await removeLabelsFromPreviousPromptVersions({
+        prisma,
+        projectId,
+        promptName: name,
+        labelsToRemove: finalLabels,
+      })),
+    );
+
+  const haveTagsChanged =
+    JSON.stringify([...new Set(finalTags)].sort()) !==
+    JSON.stringify([...new Set(latestPrompt?.tags)].sort());
+  if (haveTagsChanged)
+    // If we're creating a new prompt with tags, we must update those tags on previous prompts since tags are consistent across versions
+    create.push(
+      ...(await updatePromptTagsOnAllVersions({
+        prisma,
+        projectId,
+        promptName: name,
+        tags: finalTags,
+      })),
+    );
 
   const [createdPrompt] = await prisma.$transaction(create);
 

web/src/features/prompts/server/routers/promptRouter.ts

@@ -626,6 +626,7 @@ export const promptRouter = createTRPCRouter({
           LEFT JOIN scores s ON o.trace_id = s.trace_id AND s.observation_id = o.id AND s.project_id = ${input.projectId}
           WHERE
               o.type = 'GENERATION'
+              AND s.data_type != 'CATEGORICAL'
               AND o.prompt_id IS NOT NULL
               AND o.project_id = ${input.projectId}
               AND p.id IN (${Prisma.join(input.promptIds)})
@@ -681,6 +682,8 @@ export const promptRouter = createTRPCRouter({
             traces_by_prompt_id tp
           JOIN prompts AS p ON tp.prompt_id = p.id AND p.project_id = ${input.projectId}
           LEFT JOIN scores s ON tp.trace_id = s.trace_id AND s.observation_id IS NULL AND s.project_id = ${input.projectId}
+          WHERE 
+              s.data_type != 'CATEGORICAL'
         ), average_scores_by_prompt AS (
           SELECT 
               prompt_id,

web/src/features/prompts/server/utils/updatePromptLabels.ts

@@ -0,0 +1,33 @@
+import { type PrismaClient } from "@langfuse/shared/src/db";
+
+export const removeLabelsFromPreviousPromptVersions = async ({
+  prisma,
+  projectId,
+  promptName,
+  labelsToRemove,
+}: {
+  prisma: PrismaClient;
+  projectId: string;
+  promptName: string;
+  labelsToRemove: string[];
+}) => {
+  const previouslyLabeledPrompts = await prisma.prompt.findMany({
+    where: {
+      projectId,
+      name: promptName,
+      labels: { hasSome: labelsToRemove },
+    },
+    orderBy: [{ version: "desc" }],
+  });
+
+  return previouslyLabeledPrompts.map((prevPrompt) =>
+    prisma.prompt.update({
+      where: { id: prevPrompt.id },
+      data: {
+        labels: prevPrompt.labels.filter(
+          (prevLabel) => !labelsToRemove.includes(prevLabel),
+        ),
+      },
+    }),
+  );
+};

web/src/features/prompts/server/utils/updatePromptTags.ts

@@ -0,0 +1,28 @@
+import { type PrismaClient } from "@langfuse/shared/src/db";
+
+export const updatePromptTagsOnAllVersions = async ({
+  prisma,
+  projectId,
+  promptName,
+  tags,
+}: {
+  prisma: PrismaClient;
+  projectId: string;
+  promptName: string;
+  tags: string[];
+}) => {
+  const previousVersions = await prisma.prompt.findMany({
+    where: { projectId, name: promptName },
+  });
+
+  if (previousVersions.length === 0) return [];
+
+  return previousVersions.map((prevVersion) =>
+    prisma.prompt.update({
+      where: { id: prevVersion.id },
+      data: {
+        tags: [...new Set(tags)], // Ensure tags are unique
+      },
+    }),
+  );
+};

web/src/features/prompts/server/utils/validation.ts

@@ -27,6 +27,7 @@ export const CreateTextPromptSchema = z.object({
   type: z.literal(PromptType.Text).optional(),
   prompt: z.string(),
   config: jsonSchema.nullable().default({}),
+  tags: z.array(z.string()).nullish(),
 });
 
 export const CreateChatPromptSchema = z.object({
@@ -35,6 +36,7 @@ export const CreateChatPromptSchema = z.object({
   type: z.literal(PromptType.Chat),
   prompt: z.array(ChatMessageSchema),
   config: jsonSchema.nullable().default({}),
+  tags: z.array(z.string()).nullish(),
 });
 
 export const CreatePromptSchema = z.union([

web/src/features/rbac/constants/roleAccessRights.ts

@@ -17,6 +17,9 @@ const scopes = [
 
   "scores:CUD",
 
+  "scoreConfigs:CUD",
+  "scoreConfigs:read",
+
   "project:delete",
   "project:update",
   "project:transfer",
@@ -57,6 +60,8 @@ export const roleAccessRights: Record<ProjectRole, Scope[]> = {
     "objects:tag",
     "traces:delete",
     "scores:CUD",
+    "scoreConfigs:CUD",
+    "scoreConfigs:read",
     "project:delete",
     "project:update",
     "project:transfer",
@@ -87,6 +92,8 @@ export const roleAccessRights: Record<ProjectRole, Scope[]> = {
     "objects:tag",
     "traces:delete",
     "scores:CUD",
+    "scoreConfigs:CUD",
+    "scoreConfigs:read",
     "datasets:CUD",
     "prompts:CUD",
     "prompts:read",
@@ -107,6 +114,8 @@ export const roleAccessRights: Record<ProjectRole, Scope[]> = {
     "objects:bookmark",
     "objects:tag",
     "scores:CUD",
+    "scoreConfigs:CUD",
+    "scoreConfigs:read",
     "datasets:CUD",
     "prompts:CUD",
     "prompts:read",
@@ -120,6 +129,7 @@ export const roleAccessRights: Record<ProjectRole, Scope[]> = {
   VIEWER: [
     "prompts:read",
     "evalTemplate:read",
+    "scoreConfigs:read",
     "evalJob:read",
     "evalJobExecution:read",
   ],

web/src/pages/_app.tsx

@@ -53,10 +53,12 @@ if (
   setProjectInPosthog();
   posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY, {
     api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://eu.posthog.com",
+    ui_host: "https://eu.posthog.com",
     // Enable debug mode in development
     loaded: (posthog) => {
       if (process.env.NODE_ENV === "development") posthog.debug();
     },
+    autocapture: false,
   });
 }
 

web/src/pages/api/public/dataset-run-items.ts

@@ -16,7 +16,7 @@ const DatasetRunItemPostSchema = z
     traceId: z.string().nullish(),
   })
   .refine((data) => data.observationId || data.traceId, {
-    message: "ObservationId or traceId must be provided",
+    message: "observationId or traceId must be provided",
     path: ["observationId", "traceId"], // Specify the path of the error
   });
 
@@ -77,39 +77,32 @@ export default async function handler(
         });
       }
 
-      const observation = observationId
-        ? await prisma.observation.findUnique({
-            where: {
-              id: observationId,
-              projectId: authCheck.scope.projectId,
-            },
-          })
-        : undefined;
-      if (observationId && !observation) {
-        console.error("Observation not found");
-        return res.status(404).json({
-          message: "Observation not found",
-        });
-      }
+      let finalTraceId = traceId;
 
-      const trace = traceId
-        ? await prisma.trace.findUnique({
-            where: { id: traceId, projectId: authCheck.scope.projectId },
-          })
-        : undefined;
-      if (traceId && !trace) {
-        console.error("Trace not found");
-        return res.status(404).json({
-          message: "Trace not found",
-        });
+      // Backwards compatibility: historically, dataset run items were linked to observations, not traces
+      if (!traceId && observationId) {
+        const observation = observationId
+          ? await prisma.observation.findUnique({
+              where: {
+                id: observationId,
+                projectId: authCheck.scope.projectId,
+              },
+            })
+          : undefined;
+        if (observationId && !observation) {
+          console.error("Observation not found");
+          return res.status(404).json({
+            message: "Observation not found",
+          });
+        }
+        finalTraceId = observation?.traceId;
       }
 
       // double check, should not be necessary due to zod schema + validations above
-      const saveTraceId = trace?.id ?? observation?.traceId;
-      if (!!!saveTraceId) {
-        console.error("Observation or Trace not found");
+      if (!finalTraceId) {
+        console.error("No traceId set");
         return res.status(404).json({
-          message: "Observation or Trace not found",
+          message: "No traceId set",
         });
       }
 
@@ -135,8 +128,8 @@ export default async function handler(
       const runItem = await prisma.datasetRunItems.create({
         data: {
           datasetItemId: datasetItemId,
-          traceId: saveTraceId,
-          observationId: observation?.id ?? undefined,
+          traceId: finalTraceId,
+          observationId,
           datasetRunId: run.id,
         },
       });

web/src/pages/api/public/metrics/daily.ts

@@ -68,12 +68,15 @@ export default async function handler(
             DATE_TRUNC('DAY',
               o.start_time) "date",
             o.model,
-            SUM(o.prompt_tokens) inputUsage,
-            SUM(o.completion_tokens) outputUsage,
-            SUM(o.total_tokens) totalUsage
+            count(distinct o.id)::integer as "countObservations",
+            count(distinct t.id)::integer as "countTraces",
+            SUM(o.prompt_tokens) "inputUsage",
+            SUM(o.completion_tokens) "outputUsage",
+            SUM(o.total_tokens) "totalUsage",
+            COALESCE(SUM(o.calculated_total_cost), 0)::DOUBLE PRECISION as "totalCost"
           FROM
             traces t
-          LEFT JOIN observations o ON o.trace_id = t.id AND o.project_id = t.project_id
+          LEFT JOIN observations_view o ON o.trace_id = t.id AND o.project_id = t.project_id
           WHERE o.start_time IS NOT NULL
             AND t.project_id = ${authCheck.scope.projectId}
             ${traceNameCondition}
@@ -94,11 +97,17 @@ export default async function handler(
             json_agg(json_build_object('model',
                 model,
                 'inputUsage',
-                inputUsage,
+                "inputUsage",
                 'outputUsage',
-                outputUsage,
+                "outputUsage",
                 'totalUsage',
-                totalUsage)) daily_usage_json
+                "totalUsage",
+                'totalCost',
+                "totalCost",
+                'countObservations',
+                "countObservations",
+                'countTraces',
+                "countTraces")) daily_usage_json
           FROM
             model_usage
           GROUP BY

web/src/pages/api/public/traces.ts

@@ -25,6 +25,7 @@ const GetTracesSchema = z.object({
   userId: z.string().nullish(),
   name: z.string().nullish(),
   tags: z.union([z.array(z.string()), z.string()]).nullish(),
+  sessionId: z.string().nullish(),
   fromTimestamp: stringDate,
   orderBy: z
     .string() // orderBy=timestamp.asc
@@ -105,6 +106,9 @@ export default async function handler(
             ", ",
           )}] <@ t."tags"`
         : Prisma.empty;
+      const sessionCondition = obj.sessionId
+        ? Prisma.sql`AND t."session_id" = ${obj.sessionId}`
+        : Prisma.empty;
       const fromTimestampCondition = obj.fromTimestamp
         ? Prisma.sql`AND t."timestamp" >= ${obj.fromTimestamp}::timestamp with time zone at time zone 'UTC'`
         : Prisma.empty;
@@ -145,6 +149,7 @@ export default async function handler(
             ${userCondition}
             ${nameCondition}
             ${tagsCondition}
+            ${sessionCondition}
             ${orderByCondition}
             LIMIT ${obj.limit} OFFSET ${skipValue}
           ) AS t
@@ -168,6 +173,7 @@ export default async function handler(
           projectId: authCheck.scope.projectId,
           name: obj.name ? obj.name : undefined,
           userId: obj.userId ? obj.userId : undefined,
+          sessionId: obj.sessionId ? obj.sessionId : undefined,
           timestamp: obj.fromTimestamp
             ? { gte: new Date(obj.fromTimestamp) }
             : undefined,

web/src/pages/project/[projectId]/prompts/[promptName]/metrics.tsx

@@ -16,6 +16,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { Skeleton } from "@/src/components/ui/skeleton";
+import { ScoreDataType } from "@langfuse/shared";
 
 type PromptVersionTableRow = {
   version: number;
@@ -247,6 +248,7 @@ export default function PromptVersionTable() {
               scores={Object.entries(scores).map(([k, v]) => ({
                 name: k,
                 value: v,
+                dataType: ScoreDataType.NUMERIC, // numeric and boolean values treated as numeric
               }))}
               variant="headings"
             />
@@ -273,6 +275,7 @@ export default function PromptVersionTable() {
               scores={Object.entries(scores).map(([k, v]) => ({
                 name: k,
                 value: v,
+                dataType: ScoreDataType.NUMERIC, // numeric and boolean values treated as numeric
               }))}
               variant="headings"
             />

web/src/pages/project/[projectId]/settings.tsx

@@ -16,6 +16,7 @@ import { Card } from "@tremor/react";
 import { Button } from "@/src/components/ui/button";
 import Link from "next/link";
 import { LlmApiKeyList } from "@/src/features/public-api/components/LLMApiKeyList";
+import { ScoreConfigSettings } from "@/src/features/manual-scoring/components/ScoreConfigSettings";
 
 export default function SettingsPage() {
   const router = useRouter();
@@ -29,6 +30,7 @@ export default function SettingsPage() {
         <LlmApiKeyList projectId={projectId} />
         <ProjectMembersTable projectId={projectId} />
         <ProjectUsageChart projectId={projectId} />
+        <ScoreConfigSettings projectId={projectId} />
         <Integrations projectId={projectId} />
         <Instructions />
         <RenameProject projectId={projectId} />
@@ -107,7 +109,7 @@ function Instructions() {
           <li key={itemIdx}>
             <div className="group relative flex items-start space-x-3 py-4">
               <div className="flex-shrink-0">
-                <span className="group-hover:border-primary-accent group-hover:text-primary-accent inline-flex h-10 w-10 items-center justify-center rounded-lg border border-border text-muted-foreground">
+                <span className="inline-flex h-10 w-10 items-center justify-center rounded-lg border border-border text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent">
                   <item.icon className="h-6 w-6" aria-hidden="true" />
                 </span>
               </div>

web/src/pages/project/[projectId]/settings/posthog-integration.tsx

@@ -15,6 +15,7 @@ import { Switch } from "@/src/components/ui/switch";
 import { env } from "@/src/env.mjs";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { posthogIntegrationFormSchema } from "@/src/features/posthog-integration/types";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { type RouterOutput } from "@/src/utils/types";
 import { zodResolver } from "@hookform/resolvers/zod";
@@ -28,7 +29,13 @@ import { type z } from "zod";
 export default function PosthogIntegrationSettings() {
   const router = useRouter();
   const projectId = router.query.projectId as string;
-  const state = api.posthogIntegration.get.useQuery({ projectId });
+  const hasAccess = useHasAccess({ projectId, scope: "integrations:CRUD" });
+  const state = api.posthogIntegration.get.useQuery(
+    { projectId },
+    {
+      enabled: hasAccess,
+    },
+  );
   if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === undefined) return null;
 
   return (
@@ -46,7 +53,7 @@ export default function PosthogIntegrationSettings() {
           </Button>
         }
         status={
-          state.isInitialLoading
+          state.isInitialLoading || !hasAccess
             ? undefined
             : state.data?.enabled
               ? "active"
@@ -63,15 +70,20 @@ export default function PosthogIntegrationSettings() {
         on a daily schedule to PostHog. When first activated, it will sync all
         historical data from the beginning of your project.
       </p>
-      <div className="flex flex-col gap-10"></div>
-
-      {!state.isInitialLoading && (
+      {!hasAccess && (
+        <p className="text-sm">
+          You current role does not grant you access to these settings, please
+          reach out to your project admin or owner.
+        </p>
+      )}
+      {hasAccess && (
         <>
           <Header level="h3" title="Configuration" />
           <Card className="p-4">
             <PostHogIntegrationSettings
               state={state.data}
               projectId={projectId}
+              isLoading={state.isLoading}
             />
           </Card>
         </>
@@ -97,9 +109,11 @@ export default function PosthogIntegrationSettings() {
 const PostHogIntegrationSettings = ({
   state,
   projectId,
+  isLoading,
 }: {
   state?: RouterOutput["posthogIntegration"]["get"];
   projectId: string;
+  isLoading: boolean;
 }) => {
   const capture = usePostHogClientCapture();
   const posthogForm = useForm<z.infer<typeof posthogIntegrationFormSchema>>({
@@ -109,6 +123,7 @@ const PostHogIntegrationSettings = ({
       posthogProjectApiKey: state?.posthogApiKey ?? "",
       enabled: state?.enabled ?? false,
     },
+    disabled: isLoading,
   });
 
   useEffect(() => {
@@ -205,12 +220,14 @@ const PostHogIntegrationSettings = ({
         <Button
           loading={mut.isLoading}
           onClick={posthogForm.handleSubmit(onSubmit)}
+          disabled={isLoading}
         >
           Save
         </Button>
         <Button
           variant="ghost"
           loading={mutDelete.isLoading}
+          disabled={isLoading || !!!state}
           onClick={() => {
             if (
               confirm(

web/src/pages/project/[projectId]/users.tsx

@@ -1,19 +1,24 @@
-import Header from "@/src/components/layouts/header";
-
-import { api } from "@/src/utils/api";
-import { type RouterOutput, type RouterInput } from "@/src/utils/types";
-import { useEffect, useState } from "react";
-import TableLink from "@/src/components/table/table-link";
-import { DataTable } from "@/src/components/table/data-table";
 import { useRouter } from "next/router";
-import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { useEffect } from "react";
+import { NumberParam, useQueryParams, withDefault } from "use-query-params";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { type Score } from "@langfuse/shared";
-import { useQueryParams, withDefault, NumberParam } from "use-query-params";
-import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import Header from "@/src/components/layouts/header";
+import { DataTable } from "@/src/components/table/data-table";
+import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { Skeleton } from "@/src/components/ui/skeleton";
-import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
+import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { api } from "@/src/utils/api";
+import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { type RouterInput, type RouterOutput } from "@/src/utils/types";
+import { type Score } from "@langfuse/shared";
+import { utcDateOffsetByDays } from "@/src/utils/dates";
+import { useSession } from "next-auth/react";
+import { usersTableCols } from "@/src/server/api/definitions/usersTable";
 
 export type ScoreFilterInput = Omit<RouterInput["users"]["all"], "projectId">;
 
@@ -26,8 +31,22 @@ type RowData = {
 
 export default function UsersPage() {
   const router = useRouter();
+  const session = useSession();
   const projectId = router.query.projectId as string;
-  const [queryOptions] = useState<ScoreFilterInput>({});
+
+  const [userFilterState, setUserFilterState] = useQueryFilterState(
+    [
+      {
+        column: "timestamp",
+        type: "datetime",
+        operator: ">",
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
+        ),
+      },
+    ],
+    "users",
+  );
 
   const { setDetailPageList } = useDetailPageLists();
 
@@ -37,7 +56,7 @@ export default function UsersPage() {
   });
 
   const users = api.users.all.useQuery({
-    ...queryOptions,
+    filter: userFilterState,
     page: paginationState.pageIndex,
     limit: paginationState.pageSize,
     projectId,
@@ -212,7 +231,12 @@ export default function UsersPage() {
           href: "https://langfuse.com/docs/user-explorer",
         }}
       />
-
+      <DataTableToolbar
+        filterColumnDefinition={usersTableCols}
+        filterState={userFilterState}
+        setFilterState={setUserFilterState}
+        columns={columns}
+      />
       <DataTable
         columns={columns}
         data={

web/src/server/api/definitions/scoresTable.ts

@@ -2,6 +2,7 @@ import {
   type OptionsDefinition,
   type ColumnDefinition,
   ScoreSource,
+  ScoreDataType,
 } from "@langfuse/shared";
 
 export const scoresTableCols: ColumnDefinition[] = [
@@ -36,6 +37,13 @@ export const scoresTableCols: ColumnDefinition[] = [
     internal: 's."source"::text',
     options: Object.values(ScoreSource).map((value) => ({ value })),
   },
+  {
+    name: "Data Type",
+    id: "dataType",
+    type: "stringOptions",
+    internal: 's."data_type"::text',
+    options: Object.values(ScoreDataType).map((value) => ({ value })),
+  },
   {
     name: "Name",
     id: "name",

web/src/server/api/definitions/usersTable.ts

@@ -0,0 +1,10 @@
+import { type ColumnDefinition } from "@langfuse/shared";
+
+export const usersTableCols: ColumnDefinition[] = [
+  {
+    name: "Timestamp",
+    id: "timestamp",
+    type: "datetime",
+    internal: 't."timestamp"',
+  },
+];

web/src/server/api/root.ts

@@ -16,6 +16,8 @@ import { modelRouter } from "@/src/server/api/routers/models";
 import { evalRouter } from "@/src/ee/features/evals/server/router";
 import { posthogIntegrationRouter } from "@/src/features/posthog-integration/posthog-integration-router";
 import { llmApiKeyRouter } from "@/src/features/llm-api-key/server/router";
+import { scoreConfigsRouter } from "@/src/server/api/routers/scoreConfigs";
+import { publicRouter } from "@/src/server/api/routers/public";
 
 /**
  * This is the primary router for your server.
@@ -27,6 +29,7 @@ export const appRouter = createTRPCRouter({
   sessions: sessionRouter,
   generations: generationsRouter,
   scores: scoresRouter,
+  scoreConfigs: scoreConfigsRouter,
   dashboard: dashboardRouter,
   projects: projectsRouter,
   users: userRouter,
@@ -40,6 +43,7 @@ export const appRouter = createTRPCRouter({
   evals: evalRouter,
   posthogIntegration: posthogIntegrationRouter,
   llmApiKey: llmApiKeyRouter,
+  public: publicRouter,
 });
 
 // export type definition of API

web/src/server/api/routers/generations/getAllQuery.ts

@@ -2,7 +2,11 @@ import { type z } from "zod";
 
 import { protectedProjectProcedure } from "@/src/server/api/trpc";
 import { paginationZod } from "@/src/utils/zod";
-import { type ObservationView, Prisma } from "@langfuse/shared/src/db";
+import {
+  type ObservationView,
+  Prisma,
+  type ScoreDataType,
+} from "@langfuse/shared/src/db";
 
 import { GenerationTableOptions } from "./utils/GenerationTableOptions";
 import { getAllGenerations } from "@/src/server/api/routers/generations/db/getAllGenerationsSqlQuery";
@@ -14,6 +18,8 @@ const getAllGenerationsInput = GenerationTableOptions.extend({
 export type ScoreSimplified = {
   name: string;
   value: number;
+  dataType: ScoreDataType;
+  stringValue?: string | null;
   comment?: string | null;
 };
 

web/src/server/api/routers/public.ts

@@ -0,0 +1,94 @@
+import { VERSION } from "@/src/constants/VERSION";
+import { env } from "@/src/env.mjs";
+import { createTRPCRouter, publicProcedure } from "@/src/server/api/trpc";
+import { TRPCError } from "@trpc/server";
+import { z } from "zod";
+
+const versionSchema = z.string().regex(/^v\d+\.\d+\.\d+$/); // e.g. v1.2.3
+
+const compareVersions = (
+  current: string,
+  latest: string,
+): "major" | "minor" | "patch" | null => {
+  const currentValidated = versionSchema.parse(current);
+  const latestValidated = versionSchema.parse(latest);
+
+  const parseVersion = (version: string) => {
+    if (version.startsWith("v")) {
+      version = version.slice(1);
+    }
+    return version.split(".").map(Number);
+  };
+
+  const [currentMajor, currentMinor, currentPatch] =
+    parseVersion(currentValidated);
+  const [latestMajor, latestMinor, latestPatch] = parseVersion(latestValidated);
+
+  if (latestMajor > currentMajor) return "major";
+  if (latestMajor === currentMajor && latestMinor > currentMinor)
+    return "minor";
+  if (
+    latestMajor === currentMajor &&
+    latestMinor === currentMinor &&
+    latestPatch > currentPatch
+  )
+    return "patch";
+
+  return null;
+};
+
+const ReleaseApiRes = z.array(
+  z.object({
+    repo: z.string(),
+    latestRelease: z.string(),
+    publishedAt: z.string().datetime(),
+    url: z.string().url(),
+  }),
+);
+
+export const publicRouter = createTRPCRouter({
+  checkUpdate: publicProcedure.query(async () => {
+    // Skip update check on Langfuse Cloud
+    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) return null;
+
+    let body;
+    try {
+      const response = await fetch(
+        `https://langfuse.com/api/latest-releases?repo=langfuse/langfuse&version=${VERSION}`,
+      );
+      body = await response.json();
+    } catch (error) {
+      throw new TRPCError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "Failed to fetch or json parse the latest releases",
+        cause: error,
+      });
+    }
+
+    const releases = ReleaseApiRes.safeParse(body);
+    if (!releases.success) {
+      throw new TRPCError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "Release API response is invalid",
+      });
+    }
+    const langfuseRelease = releases.data.find(
+      (release) => release.repo === "langfuse/langfuse",
+    );
+    if (!langfuseRelease) {
+      throw new TRPCError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "Release API response is invalid",
+      });
+    }
+
+    const updateType = compareVersions(VERSION, langfuseRelease.latestRelease);
+
+    return {
+      updateType,
+      currentVersion: VERSION,
+      latestRelease: langfuseRelease.latestRelease,
+      url: langfuseRelease.url,
+    };
+  }),
+});

web/src/server/api/routers/scoreConfigs.ts

@@ -0,0 +1,106 @@
+import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import { optionalPaginationZod } from "@/src/utils/zod";
+
+import { ScoreDataType } from "@langfuse/shared/src/db";
+import { z } from "zod";
+
+const ScoreConfigAllInput = z.object({
+  projectId: z.string(), // Required for protectedProjectProcedure
+});
+
+const category = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+const ScoreConfigAllInputPaginated = ScoreConfigAllInput.extend({
+  ...optionalPaginationZod,
+});
+
+export const scoreConfigsRouter = createTRPCRouter({
+  all: protectedProjectProcedure
+    .input(ScoreConfigAllInputPaginated)
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "scoreConfigs:read",
+      });
+
+      try {
+        const configs = await ctx.prisma.scoreConfig.findMany({
+          where: {
+            projectId: input.projectId,
+          },
+          ...(input.limit !== undefined && input.page !== undefined
+            ? { take: input.limit, skip: input.page * input.limit }
+            : undefined),
+          orderBy: {
+            createdAt: "desc",
+          },
+        });
+
+        const configsCount = await ctx.prisma.scoreConfig.count({
+          where: {
+            projectId: input.projectId,
+          },
+        });
+
+        return {
+          configs,
+          totalCount: configsCount,
+        };
+      } catch (error) {
+        console.log(error);
+      }
+    }),
+
+  create: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string().min(1).max(35),
+        dataType: z.nativeEnum(ScoreDataType),
+        minValue: z.number().optional(),
+        maxValue: z.number().optional(),
+        categories: z.array(category).optional(),
+        description: z.string().optional(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "scoreConfigs:CUD",
+      });
+
+      try {
+        const existingConfig = await ctx.prisma.scoreConfig.findFirst({
+          where: {
+            projectId: input.projectId,
+            name: input.name,
+            dataType: input.dataType,
+          },
+        });
+
+        if (existingConfig)
+          throw new Error(
+            "Score config with this name and data type already exists",
+          );
+
+        const config = await ctx.prisma.scoreConfig.create({
+          data: {
+            ...input,
+          },
+        });
+
+        return config;
+      } catch (error) {
+        console.log(error);
+      }
+    }),
+});

web/src/server/api/routers/scores.ts

@@ -7,7 +7,7 @@ import {
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type ProjectRole, Prisma, type Score } from "@langfuse/shared/src/db";
 import { paginationZod } from "@/src/utils/zod";
-import { singleFilter } from "@langfuse/shared";
+import { ScoreDataType, singleFilter } from "@langfuse/shared";
 import { tableColumnsToSqlFilterAndPrefix } from "@langfuse/shared";
 import {
   type ScoreOptions,
@@ -46,8 +46,10 @@ export const scoresRouter = createTRPCRouter({
         Array<
           Score & {
             traceName: string | null;
-            userId: string | null;
+            traceUserId: string | null;
             jobConfigurationId: string | null;
+            authorUserImage: string | null;
+            authorUserName: string | null;
           }
         >
       >(
@@ -56,14 +58,19 @@ export const scoresRouter = createTRPCRouter({
           s.id,
           s.name,
           s.value,
+          s.string_value AS "stringValue",
           s.timestamp,
           s.source,
+          s.data_type AS "dataType",
           s.comment,
-          s.trace_id as "traceId",
-          s.observation_id as "observationId",
-          t.user_id as "userId",
-          t.name as "traceName",
-          je.job_configuration_id as "jobConfigurationId"
+          s.trace_id AS "traceId",
+          s.observation_id AS "observationId",
+          s.author_user_id AS "authorUserId",
+          t.user_id AS "traceUserId",
+          t.name AS "traceName",
+          je.job_configuration_id AS "jobConfigurationId",
+          u.image AS "authorUserImage", 
+          u.name AS "authorUserName"
           `,
           input.projectId,
           filterCondition,
@@ -126,10 +133,13 @@ export const scoresRouter = createTRPCRouter({
       z.object({
         projectId: z.string(),
         traceId: z.string(),
-        value: z.number(),
-        name: z.string(),
-        comment: z.string().optional(),
         observationId: z.string().optional(),
+        name: z.string(),
+        value: z.number(),
+        stringValue: z.string().optional(),
+        comment: z.string().optional().nullable(),
+        configId: z.string().optional(),
+        dataType: z.nativeEnum(ScoreDataType),
       }),
     )
     .mutation(async ({ input, ctx }) => {
@@ -149,30 +159,63 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No trace with this id in this project.");
       }
 
-      const score = await ctx.prisma.score.create({
-        data: {
+      try {
+        const existingScore = await ctx.prisma.score.findFirst({
+          where: {
+            projectId: input.projectId,
+            traceId: input.traceId,
+            observationId: input.observationId,
+            source: "ANNOTATION",
+            configId: input.configId,
+          },
+        });
+
+        if (existingScore) {
+          return ctx.prisma.score.update({
+            where: {
+              id: existingScore.id,
+              projectId: input.projectId,
+            },
+            data: {
+              value: input.value,
+              stringValue: input.stringValue,
+              comment: input.comment,
+              authorUserId: ctx.session.user.id,
+            },
+          });
+        }
+
+        const score = await ctx.prisma.score.create({
+          data: {
+            projectId: input.projectId,
+            traceId: input.traceId,
+            observationId: input.observationId,
+            value: input.value,
+            stringValue: input.stringValue,
+            dataType: input.dataType,
+            configId: input.configId,
+            name: input.name,
+            comment: input.comment,
+            authorUserId: ctx.session.user.id,
+            source: "ANNOTATION",
+          },
+        });
+        await auditLog({
           projectId: input.projectId,
-          traceId: input.traceId,
-          observationId: input.observationId,
-          value: input.value,
-          name: input.name,
-          comment: input.comment,
-          authorUserId: ctx.session.user.id,
-          source: "ANNOTATION",
-        },
-      });
-      await auditLog({
-        projectId: input.projectId,
-        userId: ctx.session.user.id,
-        userProjectRole: ctx.session.user.projects.find(
-          (p) => p.id === input.projectId,
-        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-        resourceType: "score",
-        resourceId: score.id,
-        action: "create",
-        after: score,
-      });
-      return score;
+          userId: ctx.session.user.id,
+          userProjectRole: ctx.session.user.projects.find(
+            (p) => p.id === input.projectId,
+          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+          resourceType: "score",
+          resourceId: score.id,
+          action: "create",
+          after: score,
+        });
+        return score;
+      } catch (error) {
+        console.log(error);
+        throw error;
+      }
     }),
   updateAnnotationScore: protectedProjectProcedure
     .input(
@@ -180,7 +223,10 @@ export const scoresRouter = createTRPCRouter({
         projectId: z.string(),
         id: z.string(),
         value: z.number(),
-        comment: z.string().optional(),
+        stringValue: z.string().optional(),
+        comment: z.string().optional().nullable(),
+        configId: z.string().optional(),
+        dataType: z.nativeEnum(ScoreDataType),
       }),
     )
     .mutation(async ({ input, ctx }) => {
@@ -200,29 +246,35 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No annotation score with this id in this project.");
       }
 
-      await auditLog({
-        projectId: input.projectId,
-        userId: ctx.session.user.id,
-        userProjectRole: ctx.session.user.projects.find(
-          (p) => p.id === input.projectId,
-        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-        resourceType: "score",
-        resourceId: score.id,
-        action: "update",
-        after: score,
-      });
-
-      return ctx.prisma.score.update({
-        where: {
-          id: score.id,
+      try {
+        await auditLog({
           projectId: input.projectId,
-        },
-        data: {
-          value: input.value,
-          comment: input.comment,
-          authorUserId: ctx.session.user.id,
-        },
-      });
+          userId: ctx.session.user.id,
+          userProjectRole: ctx.session.user.projects.find(
+            (p) => p.id === input.projectId,
+          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+          resourceType: "score",
+          resourceId: score.id,
+          action: "update",
+          after: score,
+        });
+
+        return ctx.prisma.score.update({
+          where: {
+            id: score.id,
+            projectId: input.projectId,
+          },
+          data: {
+            value: input.value,
+            stringValue: input.stringValue,
+            comment: input.comment,
+            authorUserId: ctx.session.user.id,
+          },
+        });
+      } catch (error) {
+        console.log(error);
+        throw error;
+      }
     }),
   deleteAnnotationScore: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), id: z.string() }))
@@ -279,6 +331,7 @@ const generateScoresQuery = (
   FROM scores s
   JOIN traces t ON t.id = s.trace_id AND t.project_id = ${projectId}
   LEFT JOIN job_executions je ON je.job_output_score_id = s.id AND je.project_id = ${projectId}
+  LEFT JOIN users u ON u.id = s.author_user_id
   WHERE s.project_id = ${projectId}
   ${filterCondition}
   ${orderCondition}

web/src/server/api/routers/sessions.ts

@@ -62,56 +62,54 @@ export const sessionRouter = createTRPCRouter({
             totalTokens: number;
           }>
         >(Prisma.sql`
-      WITH observation_metrics AS (
         SELECT
-          t.session_id,
-          EXTRACT(EPOCH FROM COALESCE(MAX(o."end_time"), MAX(o."start_time"), MAX(t.timestamp))) - EXTRACT(EPOCH FROM COALESCE(MIN(o."start_time"), MIN(t.timestamp)))::double precision AS "sessionDuration",
-          SUM(COALESCE(o."calculated_input_cost", 0)) AS "inputCost",
-          SUM(COALESCE(o."calculated_output_cost", 0)) AS "outputCost",
-          SUM(COALESCE(o."calculated_total_cost", 0)) AS "totalCost",
-          SUM(o.prompt_tokens) AS "promptTokens",
-          SUM(o.completion_tokens) AS "completionTokens",
-          SUM(o.total_tokens) AS "totalTokens"
-        FROM traces t
-        LEFT JOIN observations_view o ON o.trace_id = t.id
-        WHERE
-          t."project_id" = ${input.projectId}
-          AND o."project_id" = ${input.projectId}
-          AND t.session_id IS NOT NULL
-        GROUP BY 1
-      ),
-      trace_metrics AS (
-        SELECT
-          session_id,
-          array_agg(distinct t.user_id) "userIds",
-          count(t.id)::int "countTraces"
-        FROM traces t
-        WHERE
-          t."project_id" = ${input.projectId}
-          AND t.session_id IS NOT NULL
-        GROUP BY 1
-      )
-
-      SELECT
         s.id,
-        s."created_at" "createdAt",
+        s. "created_at" AS "createdAt",
         s.bookmarked,
         s.public,
-        t."userIds",
-        t."countTraces",
-        o."sessionDuration",
-        COALESCE(o."totalCost", 0) AS "totalCost",
-        COALESCE(o."inputCost", 0) AS "inputCost",
-        COALESCE(o."outputCost", 0) AS "outputCost",
-        COALESCE(o."promptTokens", 0) AS "promptTokens",
-        COALESCE(o."completionTokens", 0) AS "completionTokens",
-        COALESCE(o."totalTokens", 0) AS "totalTokens",
+        t. "userIds",
+        t. "countTraces",
+        o. "sessionDuration",
+        o. "totalCost" AS "totalCost",
+        o. "inputCost" AS "inputCost",
+        o. "outputCost" AS "outputCost",
+        o. "promptTokens" AS "promptTokens",
+        o. "completionTokens" AS "completionTokens",
+        o. "totalTokens" AS "totalTokens",
         (count(*) OVER ())::int AS "totalCount"
-      FROM trace_sessions s
-      LEFT JOIN trace_metrics t ON t.session_id = s.id
-      LEFT JOIN observation_metrics o ON o.session_id = s.id
+      FROM
+        trace_sessions AS s
+        LEFT JOIN LATERAL (
+          SELECT
+            t.session_id,
+            MAX(t. "timestamp") AS "max_timestamp",
+            MIN(t. "timestamp") AS "min_timestamp",
+            array_agg(t.id) AS "traceIds",
+            array_agg(DISTINCT t.user_id) AS "userIds",
+            count(t.id)::int AS "countTraces"
+          FROM
+            traces t
+          WHERE
+            t.project_id = ${input.projectId}
+            AND t.session_id = s.id
+          GROUP BY
+            t.session_id) AS t ON TRUE
+        LEFT JOIN LATERAL (
+          SELECT
+            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
+            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
+            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
+            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
+            SUM(o.prompt_tokens) AS "promptTokens",
+            SUM(o.completion_tokens) AS "completionTokens",
+            SUM(o.total_tokens) AS "totalTokens"
+          FROM
+            observations_view o
+          WHERE
+            o.project_id = ${input.projectId}
+            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
       WHERE
-        s."project_id" = ${input.projectId}
+        s. "project_id" = ${input.projectId}
         ${filterCondition}
       ${orderByCondition}
       LIMIT ${input.limit}

web/src/server/api/routers/users.ts

@@ -4,11 +4,17 @@ import {
   createTRPCRouter,
   protectedProjectProcedure,
 } from "@/src/server/api/trpc";
-import { Prisma, type Score } from "@langfuse/shared/src/db";
 import { paginationZod } from "@/src/utils/zod";
+import {
+  singleFilter,
+  tableColumnsToSqlFilterAndPrefix,
+} from "@langfuse/shared";
+import { Prisma, type Score } from "@langfuse/shared/src/db";
+import { usersTableCols } from "@/src/server/api/definitions/usersTable";
 
 const UserFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
+  filter: z.array(singleFilter).nullable(),
 });
 
 const UserAllOptions = UserFilterOptions.extend({
@@ -19,6 +25,12 @@ export const userRouter = createTRPCRouter({
   all: protectedProjectProcedure
     .input(UserAllOptions)
     .query(async ({ input, ctx }) => {
+      const filterCondition = tableColumnsToSqlFilterAndPrefix(
+        input.filter ?? [],
+        usersTableCols,
+        "users",
+      );
+
       const totalUsers = (
         await ctx.prisma.$queryRaw<
           Array<{
@@ -28,6 +40,7 @@ export const userRouter = createTRPCRouter({
         SELECT COUNT(DISTINCT t.user_id)::int AS "totalCount"
         FROM traces t
         WHERE t.project_id = ${input.projectId}
+        ${filterCondition}
       `
       )[0].totalCount;
 
@@ -46,6 +59,7 @@ export const userRouter = createTRPCRouter({
           t.user_id IS NOT NULL
           AND t.user_id != ''
           AND t.project_id = ${input.projectId}
+          ${filterCondition}
         GROUP BY
           t.user_id
         ORDER BY

web/src/server/api/services/tableDefinitions.ts

@@ -200,7 +200,7 @@ export const tableDefinitions: TableDefinitions = {
     ],
   },
   traces_scores: {
-    table: ` traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id`,
+    table: ` traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id`,
     columns: [
       tracesProjectId,
       { name: "value", id: "value", type: "number", internal: 's."value"' },

web/vercel.json

@@ -2,7 +2,7 @@
   "crons": [
     {
       "path": "/api/cron/ingestion-metrics",
-      "schedule": "*/15 * * * *"
+      "schedule": "0 */6 * * *"
     }
   ]
 }

worker/Dockerfile

@@ -5,7 +5,7 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
@@ -56,4 +56,4 @@ COPY --from=builder --chown=expressjs:expressjs /app .
 EXPOSE 3030
 ENV PORT=3030
 
-CMD ["node", "worker/dist/src/index.js"]
+CMD ["node", "worker/dist/index.js"]

worker/generated/types.ts

@@ -1,491 +0,0 @@
-import type { ColumnType } from "kysely";
-export type Generated<T> =
-  T extends ColumnType<infer S, infer I, infer U>
-    ? ColumnType<S, I | undefined, U>
-    : ColumnType<T, T | undefined, T>;
-export type Timestamp = ColumnType<Date, Date | string, Date | string>;
-
-export const ProjectRole = {
-  OWNER: "OWNER",
-  ADMIN: "ADMIN",
-  MEMBER: "MEMBER",
-  VIEWER: "VIEWER",
-} as const;
-export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
-export const ObservationType = {
-  SPAN: "SPAN",
-  EVENT: "EVENT",
-  GENERATION: "GENERATION",
-} as const;
-export type ObservationType =
-  (typeof ObservationType)[keyof typeof ObservationType];
-export const ObservationLevel = {
-  DEBUG: "DEBUG",
-  DEFAULT: "DEFAULT",
-  WARNING: "WARNING",
-  ERROR: "ERROR",
-} as const;
-export type ObservationLevel =
-  (typeof ObservationLevel)[keyof typeof ObservationLevel];
-export const ScoreSource = {
-  ANNOTATION: "ANNOTATION",
-  API: "API",
-  EVAL: "EVAL",
-} as const;
-export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
-export const ScoreDataType = {
-  CATEGORICAL: "CATEGORICAL",
-  NUMERIC: "NUMERIC",
-} as const;
-export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
-export const PricingUnit = {
-  PER_1000_TOKENS: "PER_1000_TOKENS",
-  PER_1000_CHARS: "PER_1000_CHARS",
-} as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-  PROMPT: "PROMPT",
-  COMPLETION: "COMPLETION",
-  TOTAL: "TOTAL",
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
-export const DatasetStatus = {
-  ACTIVE: "ACTIVE",
-  ARCHIVED: "ARCHIVED",
-} as const;
-export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
-export const JobType = {
-  EVAL: "EVAL",
-} as const;
-export type JobType = (typeof JobType)[keyof typeof JobType];
-export const JobConfigState = {
-  ACTIVE: "ACTIVE",
-  INACTIVE: "INACTIVE",
-} as const;
-export type JobConfigState =
-  (typeof JobConfigState)[keyof typeof JobConfigState];
-export const JobExecutionStatus = {
-  COMPLETED: "COMPLETED",
-  ERROR: "ERROR",
-  PENDING: "PENDING",
-  CANCELLED: "CANCELLED",
-} as const;
-export type JobExecutionStatus =
-  (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
-export type Account = {
-  id: string;
-  user_id: string;
-  type: string;
-  provider: string;
-  providerAccountId: string;
-  refresh_token: string | null;
-  access_token: string | null;
-  expires_at: number | null;
-  expires_in: number | null;
-  ext_expires_in: number | null;
-  token_type: string | null;
-  scope: string | null;
-  id_token: string | null;
-  session_state: string | null;
-};
-export type ApiKey = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  note: string | null;
-  public_key: string;
-  hashed_secret_key: string;
-  fast_hashed_secret_key: string | null;
-  display_secret_key: string;
-  last_used_at: Timestamp | null;
-  expires_at: Timestamp | null;
-  project_id: string;
-};
-export type AuditLog = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  user_id: string;
-  project_id: string;
-  user_project_role: ProjectRole;
-  resource_type: string;
-  resource_id: string;
-  action: string;
-  before: string | null;
-  after: string | null;
-};
-export type CronJobs = {
-  name: string;
-  last_run: Timestamp | null;
-  job_started_at: Timestamp | null;
-  state: string | null;
-};
-export type Dataset = {
-  id: string;
-  name: string;
-  description: string | null;
-  metadata: unknown | null;
-  project_id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type DatasetItem = {
-  id: string;
-  status: Generated<DatasetStatus>;
-  input: unknown | null;
-  expected_output: unknown | null;
-  metadata: unknown | null;
-  source_trace_id: string | null;
-  source_observation_id: string | null;
-  dataset_id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type DatasetRunItems = {
-  id: string;
-  dataset_run_id: string;
-  dataset_item_id: string;
-  trace_id: string;
-  observation_id: string | null;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type DatasetRuns = {
-  id: string;
-  name: string;
-  description: string | null;
-  metadata: unknown | null;
-  dataset_id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type EvalTemplate = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  name: string;
-  version: number;
-  prompt: string;
-  model: string;
-  model_params: unknown;
-  vars: Generated<string[]>;
-  output_schema: unknown;
-};
-export type Events = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  data: unknown;
-  headers: Generated<unknown>;
-  url: string | null;
-  method: string | null;
-};
-export type JobConfiguration = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  job_type: JobType;
-  status: Generated<JobConfigState>;
-  eval_template_id: string | null;
-  score_name: string;
-  filter: unknown;
-  target_object: string;
-  variable_mapping: unknown;
-  sampling: string;
-  delay: number;
-};
-export type JobExecution = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  job_configuration_id: string;
-  status: JobExecutionStatus;
-  start_time: Timestamp | null;
-  end_time: Timestamp | null;
-  error: string | null;
-  job_input_trace_id: string | null;
-  job_output_score_id: string | null;
-};
-export type LlmApiKeys = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  provider: string;
-  display_secret_key: string;
-  secret_key: string;
-  project_id: string;
-};
-export type MembershipInvitation = {
-  id: string;
-  email: string;
-  role: ProjectRole;
-  project_id: string;
-  sender_id: string | null;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type Model = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string | null;
-  model_name: string;
-  match_pattern: string;
-  start_date: Timestamp | null;
-  input_price: string | null;
-  output_price: string | null;
-  total_price: string | null;
-  unit: string;
-  tokenizer_id: string | null;
-  tokenizer_config: unknown | null;
-};
-export type Observation = {
-  id: string;
-  trace_id: string | null;
-  project_id: string;
-  type: ObservationType;
-  start_time: Generated<Timestamp>;
-  end_time: Timestamp | null;
-  name: string | null;
-  metadata: unknown | null;
-  parent_observation_id: string | null;
-  level: Generated<ObservationLevel>;
-  status_message: string | null;
-  version: string | null;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  model: string | null;
-  internal_model: string | null;
-  modelParameters: unknown | null;
-  input: unknown | null;
-  output: unknown | null;
-  prompt_tokens: Generated<number>;
-  completion_tokens: Generated<number>;
-  total_tokens: Generated<number>;
-  unit: string | null;
-  input_cost: string | null;
-  output_cost: string | null;
-  total_cost: string | null;
-  completion_start_time: Timestamp | null;
-  prompt_id: string | null;
-};
-export type ObservationView = {
-  id: string;
-  trace_id: string | null;
-  project_id: string;
-  type: ObservationType;
-  start_time: Generated<Timestamp>;
-  end_time: Timestamp | null;
-  name: string | null;
-  metadata: unknown | null;
-  parent_observation_id: string | null;
-  level: Generated<ObservationLevel>;
-  status_message: string | null;
-  version: string | null;
-  created_at: Generated<Timestamp>;
-  model: string | null;
-  modelParameters: unknown | null;
-  input: unknown | null;
-  output: unknown | null;
-  prompt_tokens: Generated<number>;
-  completion_tokens: Generated<number>;
-  total_tokens: Generated<number>;
-  unit: string | null;
-  completion_start_time: Timestamp | null;
-  prompt_id: string | null;
-  model_id: string | null;
-  input_price: string | null;
-  output_price: string | null;
-  total_price: string | null;
-  calculated_input_cost: string | null;
-  calculated_output_cost: string | null;
-  calculated_total_cost: string | null;
-  latency: number | null;
-  time_to_first_token: number | null;
-};
-export type PosthogIntegration = {
-  project_id: string;
-  encrypted_posthog_api_key: string;
-  posthog_host_name: string;
-  last_sync_at: Timestamp | null;
-  enabled: boolean;
-  created_at: Generated<Timestamp>;
-};
-export type Pricing = {
-  id: string;
-  model_name: string;
-  pricing_unit: Generated<PricingUnit>;
-  price: string;
-  currency: Generated<string>;
-  token_type: TokenType;
-};
-export type Project = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  name: string;
-  cloud_config: unknown | null;
-};
-export type ProjectMembership = {
-  project_id: string;
-  user_id: string;
-  role: ProjectRole;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type Prompt = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  created_by: string;
-  prompt: unknown;
-  name: string;
-  version: number;
-  type: Generated<string>;
-  is_active: boolean | null;
-  config: Generated<unknown>;
-  tags: Generated<string[]>;
-  labels: Generated<string[]>;
-};
-export type Score = {
-  id: string;
-  timestamp: Generated<Timestamp>;
-  project_id: string;
-  name: string;
-  value: number;
-  source: ScoreSource;
-  author_user_id: string | null;
-  comment: string | null;
-  trace_id: string;
-  observation_id: string | null;
-  config_id: string | null;
-  string_value: string | null;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  data_type: Generated<ScoreDataType>;
-};
-export type ScoreConfig = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  name: string;
-  data_type: ScoreDataType;
-  is_archived: Generated<boolean>;
-  min_value: number | null;
-  max_value: number | null;
-  categories: unknown | null;
-  description: string | null;
-};
-export type Session = {
-  id: string;
-  session_token: string;
-  user_id: string;
-  expires: Timestamp;
-};
-export type SsoConfig = {
-  domain: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  auth_provider: string;
-  auth_config: unknown | null;
-};
-export type Trace = {
-  id: string;
-  external_id: string | null;
-  timestamp: Generated<Timestamp>;
-  name: string | null;
-  user_id: string | null;
-  metadata: unknown | null;
-  release: string | null;
-  version: string | null;
-  project_id: string;
-  public: Generated<boolean>;
-  bookmarked: Generated<boolean>;
-  tags: Generated<string[]>;
-  input: unknown | null;
-  output: unknown | null;
-  session_id: string | null;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-};
-export type TraceSession = {
-  id: string;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  project_id: string;
-  bookmarked: Generated<boolean>;
-  public: Generated<boolean>;
-};
-export type TraceView = {
-  id: string;
-  external_id: string | null;
-  timestamp: Generated<Timestamp>;
-  name: string | null;
-  user_id: string | null;
-  metadata: unknown | null;
-  release: string | null;
-  version: string | null;
-  project_id: string;
-  public: Generated<boolean>;
-  bookmarked: Generated<boolean>;
-  tags: Generated<string[]>;
-  input: unknown | null;
-  output: unknown | null;
-  session_id: string | null;
-  duration: number | null;
-};
-export type User = {
-  id: string;
-  name: string | null;
-  email: string | null;
-  email_verified: Timestamp | null;
-  password: string | null;
-  image: string | null;
-  admin: Generated<boolean>;
-  created_at: Generated<Timestamp>;
-  updated_at: Generated<Timestamp>;
-  feature_flags: Generated<string[]>;
-};
-export type VerificationToken = {
-  identifier: string;
-  token: string;
-  expires: Timestamp;
-};
-export type DB = {
-  Account: Account;
-  api_keys: ApiKey;
-  audit_logs: AuditLog;
-  cron_jobs: CronJobs;
-  dataset_items: DatasetItem;
-  dataset_run_items: DatasetRunItems;
-  dataset_runs: DatasetRuns;
-  datasets: Dataset;
-  eval_templates: EvalTemplate;
-  events: Events;
-  job_configurations: JobConfiguration;
-  job_executions: JobExecution;
-  llm_api_keys: LlmApiKeys;
-  membership_invitations: MembershipInvitation;
-  models: Model;
-  observations: Observation;
-  observations_view: ObservationView;
-  posthog_integrations: PosthogIntegration;
-  pricings: Pricing;
-  project_memberships: ProjectMembership;
-  projects: Project;
-  prompts: Prompt;
-  score_configs: ScoreConfig;
-  scores: Score;
-  Session: Session;
-  sso_configs: SsoConfig;
-  trace_sessions: TraceSession;
-  traces: Trace;
-  traces_view: TraceView;
-  users: User;
-  verification_tokens: VerificationToken;
-};

worker/package.json

@@ -1,9 +1,8 @@
 {
   "name": "worker",
-  "version": "2.47.2",
+  "version": "2.49.0",
   "description": "",
   "license": "MIT",
-  "main": "index.js",
   "private": true,
   "engines": {
     "node": ">=20"
@@ -11,7 +10,7 @@
   "scripts": {
     "test": "dotenv -e ../.env -- vitest run --reporter=basic --disable-console-intercept",
     "coverage": "vitest run --coverage",
-    "start": "dotenv -e ../.env -- node dist/src/index.js",
+    "start": "dotenv -e ../.env -- node dist/index.js",
     "build": "tsc",
     "dev": "dotenv -e ../.env -- nodemon src/index.ts",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
@@ -50,7 +49,7 @@
     "@types/lodash": "^4.17.4",
     "@types/node": "^20.11.19",
     "@types/pg": "^8.11.6",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

worker/src/api/index.ts

@@ -67,7 +67,7 @@ router
   )
   .post<{}, EventsResponse>("/events", async (req, res) => {
     const { body } = req;
-    logger.debug(`Received events, ${JSON.stringify(body)}`);
+    logger.info(`Received events, ${JSON.stringify(body)}`);
 
     const events = eventBody.parse(body);
 

worker/src/app.ts

@@ -23,17 +23,16 @@ if (isSentryEnabled) {
   Sentry.init({
     dsn: String(env.SENTRY_DSN),
     integrations: [
-      // enable HTTP calls tracing
       new Sentry.Integrations.Http({ tracing: true }),
-      // enable Express.js middleware tracing
+
       new Sentry.Integrations.Express({ app }),
       nodeProfilingIntegration(),
       Sentry.metrics.metricsAggregatorIntegration(),
     ],
-    // Performance Monitoring
-    tracesSampleRate: 0.1, //  Capture 100% of the transactions
-    // Set sampling rate for profiling - this is relative to tracesSampleRate
-    profilesSampleRate: 0.1,
+
+    tracesSampleRate: 0.01, //  Capture 100% of the transactions
+
+    profilesSampleRate: 0.01,
     sampleRate: 0.1,
   });
 

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.47.2";
+export const VERSION = "v2.49.0";

worker/src/redis/consumer.ts

@@ -1,5 +1,10 @@
 import { Job, Queue, Worker } from "bullmq";
-import { BaseError, QueueName, TQueueJobTypes } from "@langfuse/shared";
+import {
+  ApiError,
+  BaseError,
+  QueueName,
+  TQueueJobTypes,
+} from "@langfuse/shared";
 import { evaluate, createEvalJobs } from "../eval-service";
 import { kyselyPrisma } from "@langfuse/shared/src/db";
 import logger from "../logger";
@@ -59,11 +64,6 @@ export const evalJobExecutor = redis
             await evaluate({ event: job.data.payload });
             return true;
           } catch (e) {
-            logger.error(
-              e,
-              `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId} ${e}`
-            );
-
             const displayError =
               e instanceof BaseError ? e.message : "An internal error occurred";
 
@@ -76,7 +76,20 @@ export const evalJobExecutor = redis
               .where("project_id", "=", job.data.payload.projectId)
               .execute();
 
-            Sentry.captureException(e);
+            // do not log expected errors (api failures + missing api keys not provided by the user)
+            if (
+              !(e instanceof ApiError) &&
+              !(
+                e instanceof BaseError &&
+                e.message.includes("API key for provider")
+              )
+            ) {
+              logger.error(
+                e,
+                `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId} ${e}`
+              );
+              Sentry.captureException(e);
+            }
 
             throw e;
           } finally {