chore: release v2.60.0

* chore(scores): drop not null 'value' column

* fix: resulting TS

* fix: adjust seeder to new schema

* push

* docs

* types: add union types for scores

* push

* push

* push

* push

* feat(api): POST score-configs POST scores (#2436)

* feat(api): add POST score-configs

* feat(api): add POST scores with configs

* fix: types

* push

* docs(scores): add

* push

* push

* feat(api): allow filtering by `configId`

* push

* push

* push

* push

* push

* push

* push

* push

* push

* push

* push

* refactor: pass `value` as `string | number` in score ingestion

* fix test

* push

* test

* push

* refactor: use `withMiddlewares` handler

* push

* push

* refactor scores api

* refactor: config api to new schema

* rename

* refactor

* push

* rm: explicit `trpcErrorToast(error)`

* rename

* fixup

* fixup

* fixup

* fixup

* fixup

* feat: add filter by `dataType` to GET all scores endpoint

* refactor: reference score data types manually instead of via the enum to protect against changes of the enum

* refactor

* push

* refactor

* push

* docs: adjust and regenerate docs

* push

* push

* push

* push

* push

* add ingestion docs

* push

* push

* push

* nit

* don't use database calls in score tests

* rm typecast from public api score-config

* cast prisma query raw as unknown

* refactor: extract helpers to public-api types

* unify types

* comments

* fix types

* push

* push

* push

* push

* push

* push

* push

* tyoes

* clean up types

* fern docs

* fix typo

* push

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

README.md

@@ -159,7 +159,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

fern/apis/client/definition/score.yml

@@ -17,15 +17,131 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -120,25 +120,93 @@ types:
       updatedAt: datetime
       projectId: string
       dataType: ScoreDataType
-      isArchived: boolean
-      minValue: optional<double>
-      maxValue: optional<double>
-      categories: optional<list<ConfigCategory>>
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
       description: optional<string>
   ConfigCategory:
     properties:
       value: double
       label: string
-  Score:
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
   Dataset:
     properties:
       id: string

fern/apis/server/definition/ingestion.yml

@@ -185,9 +185,54 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/models.yml

@@ -35,7 +35,7 @@ service:
       response: commons.Model
     delete:
       method: DELETE
-      docs: Create a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
       path: /models/{id}
       path-parameters:
         id: string

fern/apis/server/definition/score-configs.yml

@@ -6,6 +6,12 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
     get:
       docs: Get all score configs
       method: GET
@@ -18,7 +24,7 @@ service:
             docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
       response: ScoreConfigs
     get-by-id:
       docs: Get a score config
@@ -34,3 +40,19 @@ types:
     properties:
       data: list<commons.ScoreConfig>
       meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -25,8 +25,12 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
@@ -42,6 +46,12 @@ service:
           scoreIds:
             type: optional<string>
             docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -66,9 +76,55 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+
   Scores:
     properties:
       data: list<commons.Score>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.59.1",
+  "version": "2.60.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/prisma/generated/types.ts

@@ -357,7 +357,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
     author_user_id: string | null;
     comment: string | null;

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240704103901_scores_make_value_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "value" DROP NOT NULL;

packages/shared/prisma/schema.prisma

@@ -410,14 +410,14 @@ model Score {
     projectId     String         @map("project_id")
     project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
-    value         Float
+    value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
     source        ScoreSource
     authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
-    stringValue   String?        @map("string_value")
+    stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")

packages/shared/prisma/seed.ts

@@ -17,6 +17,11 @@ import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
 const options = {
   environment: { type: "string" },
 } as const;
@@ -459,7 +464,15 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
-  configIdsAndNames: Map<string, { name: string; id: string }[]>
+  configParams: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -518,13 +531,34 @@ function createObjects(
 
     traces.push(trace);
 
-    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const configArray = configParams.get(projectId) ?? [];
     const randomIndex = Math.floor(Math.random() * 3);
     const config =
       configArray.length >= randomIndex - 1 && configArray[randomIndex];
-    const { name: annotationScoreName, id: configId } = config || {
+    const {
+      name: annotationScoreName,
+      id: configId,
+      dataType,
+      categories,
+    } = config || {
       name: "manual-score",
       id: undefined,
+      dataType: ScoreDataType.NUMERIC,
+      categories: null,
+    };
+
+    const value = Math.floor(Math.random() * 2);
+    const scoreNumericAndStringValue = {
+      ...(dataType === ScoreDataType.NUMERIC && { value }),
+      ...(dataType === ScoreDataType.CATEGORICAL && {
+        value,
+        stringValue: categories?.find((category) => category.value === value)
+          ?.label,
+      }),
+      ...(dataType === ScoreDataType.BOOLEAN && {
+        value,
+        stringValue: value === 1 ? "True" : "False",
+      }),
     };
 
     const traceScores = [
@@ -533,12 +567,12 @@ function createObjects(
             {
               traceId: trace.id,
               name: annotationScoreName,
-              value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
-              dataType: ScoreDataType.NUMERIC,
+              dataType,
+              ...scoreNumericAndStringValue,
               ...(configId ? { configId } : {}),
             },
           ]
@@ -556,6 +590,20 @@ function createObjects(
             },
           ]
         : []),
+      ...(Math.random() < 0.8
+        ? [
+            {
+              traceId: trace.id,
+              name: "Completeness",
+              timestamp: traceTs,
+              source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.CATEGORICAL,
+              stringValue:
+                Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+            },
+          ]
+        : []),
     ];
 
     scores.push(...traceScores);
@@ -976,8 +1024,15 @@ async function generatePrompts(project: Project) {
 }
 
 async function generateConfigsForProject(projects: Project[]) {
-  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
-    new Map();
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
 
   await Promise.all(
     projects.map(async (project) => {
@@ -989,7 +1044,12 @@ async function generateConfigsForProject(projects: Project[]) {
 }
 
 async function generateConfigs(project: Project) {
-  const configNameAndId: { name: string; id: string }[] = [];
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
 
   const configs = [
     {
@@ -1046,7 +1106,12 @@ async function generateConfigs(project: Project) {
         id: config.id,
       },
     });
-    configNameAndId.push({ name: config.name, id: config.id });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
   }
 
   return configNameAndId;

packages/shared/src/errors/InternalServerError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InternalServerError extends BaseError {
+  constructor(description = "Internal Server Error") {
+    super("InternalServerError", 500, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -5,3 +5,4 @@ export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
+export { InternalServerError } from "./InternalServerError";

packages/shared/src/features/annotation/types.ts

@@ -1,15 +1,59 @@
 import z from "zod";
-import { ScoreConfig } from "../../db";
+import { type ScoreDataType } from "../../db";
 
-const configCategory = z.object({
-  label: z.string().min(1),
+const NUMERIC: ScoreDataType = "NUMERIC";
+const CATEGORICAL: ScoreDataType = "CATEGORICAL";
+const BOOLEAN: ScoreDataType = "BOOLEAN";
+
+export const availableDataTypes = [NUMERIC, CATEGORICAL, BOOLEAN] as const;
+
+const NumericData = z.object({
   value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
 });
 
-export const categoriesList = z.array(configCategory);
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
 
-export type ConfigCategory = z.infer<typeof configCategory>;
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
 
-export type CastedConfig = Omit<ScoreConfig, "categories"> & {
-  categories: ConfigCategory[] | null;
-};
+const CreateAnnotationScoreBase = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  traceId: z.string(),
+  configId: z.string().optional(),
+  observationId: z.string().optional(),
+  comment: z.string().optional().nullish(),
+});
+
+const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
+  id: z.string(),
+});
+
+/**
+ * CreateAnnotationScoreData is only used for annotation scores created via the UI.
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const CreateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  CreateAnnotationScoreBase.merge(NumericData),
+  CreateAnnotationScoreBase.merge(CategoricalData),
+  CreateAnnotationScoreBase.merge(BooleanData),
+]);
+
+/**
+ * UpdateAnnotationScoreData is only used for annotation scores updated via the UI
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  UpdateAnnotationScoreBase.merge(NumericData),
+  UpdateAnnotationScoreBase.merge(CategoricalData),
+  UpdateAnnotationScoreBase.merge(BooleanData),
+]);

packages/shared/src/features/ingestion/types.ts

@@ -145,15 +145,75 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
   return false;
 });
 
-export const ScoreBody = z.object({
+const BaseScoreBody = z.object({
   id: z.string().nullish(),
   name: NonEmptyString,
-  value: z.number(),
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
 });
 
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      })
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
   id: z.string().nullish(),

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.59.1",
+  "version": "2.60.0",
   "private": true,
   "license": "MIT",
   "engines": {

web/public/generated/api-client/openapi.yml

@@ -59,21 +59,39 @@ components:
           type: string
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
         comment:
           type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
         - value
-    Score:
-      title: Score
+    NumericScore:
+      title: NumericScore
       type: object
       properties:
         id:
@@ -85,6 +103,9 @@ components:
         value:
           type: number
           format: double
+          description: The numeric value of the score
+        source:
+          $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
         timestamp:
@@ -92,12 +113,163 @@ components:
           format: date-time
         comment:
           type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
         - name
         - value
+        - source
         - timestamp
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal
+      required:
+        - id
+        - traceId
+        - name
+        - value
+        - stringValue
+        - source
+        - timestamp
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and stringValue must map to a config category
+      required:
+        - id
+        - traceId
+        - name
+        - stringValue
+        - source
+        - timestamp
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    ScoreSource:
+      title: ScoreSource
+      type: string
+      enum:
+        - ANNOTATION
+        - API
+        - EVAL
+    ScoreDataType:
+      title: ScoreDataType
+      type: string
+      enum:
+        - NUMERIC
+        - CATEGORICAL
+        - BOOLEAN
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: string
+        - type: number
+          format: double
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
   securitySchemes:
     BearerAuth:
       type: http

web/public/generated/api/openapi.yml

@@ -845,7 +845,7 @@ paths:
       security: *ref_0
     delete:
       description: >-
-        Create a model. Cannot delete models managed by Langfuse. You can create
+        Delete a model. Cannot delete models managed by Langfuse. You can create
         your own definition with the same modelName to override the definition
         though.
       operationId: models_delete
@@ -1250,6 +1250,53 @@ paths:
             schema:
               $ref: '#/components/schemas/CreatePromptRequest'
   /api/public/score-configs:
+    post:
+      description: >-
+        Create a score configuration (config). Score configs are used to define
+        the structure of scores
+      operationId: scoreConfigs_create
+      tags:
+        - ScoreConfigs
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreConfig'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateScoreConfigRequest'
     get:
       description: Get all score configs
       operationId: scoreConfigs_get
@@ -1267,7 +1314,7 @@ paths:
           in: query
           description: >-
             Limit of items per page. If you encounter api issues due to too
-            large page sizes, try to reduce the limit.
+            large page sizes, try to reduce the limit
           required: false
           schema:
             type: integer
@@ -1421,12 +1468,14 @@ paths:
             nullable: true
         - name: userId
           in: query
+          description: Retrieve only scores with this userId associated to the trace.
           required: false
           schema:
             type: string
             nullable: true
         - name: name
           in: query
+          description: Retrieve only scores with this name.
           required: false
           schema:
             type: string
@@ -1468,6 +1517,20 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: configId
+          in: query
+          description: Retrieve only scores with a specific configId.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: dataType
+          in: query
+          description: Retrieve only scores with a specific dataType.
+          required: false
+          schema:
+            $ref: '#/components/schemas/ScoreDataType'
+            nullable: true
       responses:
         '200':
           description: ''
@@ -2069,19 +2132,27 @@ components:
           $ref: '#/components/schemas/ScoreDataType'
         isArchived:
           type: boolean
+          description: Whether the score config is archived. Defaults to false
         minValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets minimum value for numerical scores. If not set, the minimum
+            value defaults to -∞
         maxValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets maximum value for numerical scores. If not set, the maximum
+            value defaults to +∞
         categories:
           type: array
           items:
             $ref: '#/components/schemas/ConfigCategory'
           nullable: true
+          description: Configures custom categories for categorical scores
         description:
           type: string
           nullable: true
@@ -2105,8 +2176,8 @@ components:
       required:
         - value
         - label
-    Score:
-      title: Score
+    NumericScore:
+      title: NumericScore
       type: object
       properties:
         id:
@@ -2118,6 +2189,7 @@ components:
         value:
           type: number
           format: double
+          description: The numeric value of the score
         source:
           $ref: '#/components/schemas/ScoreSource'
         observationId:
@@ -2129,6 +2201,13 @@ components:
         comment:
           type: string
           nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
@@ -2136,6 +2215,143 @@ components:
         - value
         - source
         - timestamp
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+          nullable: true
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+          nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal
+      required:
+        - id
+        - traceId
+        - name
+        - value
+        - stringValue
+        - source
+        - timestamp
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+          nullable: true
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+          nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and stringValue must map to a config category
+      required:
+        - id
+        - traceId
+        - name
+        - stringValue
+        - source
+        - timestamp
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: string
+        - type: number
+          format: double
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
     Dataset:
       title: Dataset
       type: object
@@ -2891,17 +3107,37 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
@@ -3467,6 +3703,47 @@ components:
       required:
         - data
         - meta
+    CreateScoreConfigRequest:
+      title: CreateScoreConfigRequest
+      type: object
+      properties:
+        name:
+          type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+        categories:
+          type: array
+          items:
+            $ref: '#/components/schemas/ConfigCategory'
+          nullable: true
+          description: >-
+            Configure custom categories for categorical scores. Pass a list of
+            objects with `label` and `value` properties. Categories are
+            autogenerated for boolean configs and cannot be passed
+        minValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a minimum value for numerical scores. If not set, the
+            minimum value defaults to -∞
+        maxValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a maximum value for numerical scores. If not set, the
+            maximum value defaults to +∞
+        description:
+          type: string
+          nullable: true
+          description: >-
+            Description is shown across the Langfuse UI and can be used to e.g.
+            explain the config categories in detail, why a numeric range was
+            set, or provide additional context on config name or usage
+      required:
+        - name
+        - dataType
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object
@@ -3476,17 +3753,36 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            The data type of the score. When passing a configId this field is
+            inferred. Otherwise, this field must be passed or will default to
+            numeric.
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. The unique langfuse identifier
+            of a score config. When passing this field, the dataType and
+            stringValue fields are automatically populated.
       required:
         - traceId
         - name

web/public/generated/postman/collection.json

@@ -645,7 +645,7 @@
           "_type": "endpoint",
           "name": "Delete",
           "request": {
-            "description": "Create a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.",
+            "description": "Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.",
             "url": {
               "raw": "{{baseUrl}}/api/public/models/:id",
               "host": [
@@ -954,6 +954,39 @@
       "description": null,
       "name": "Score Configs",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a score configuration (config). Score configs are used to define the structure of scores",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"dataType\": \"NUMERIC\",\n    \"categories\": [\n        {\n            \"value\": 0,\n            \"label\": \"example\"\n        }\n    ],\n    \"minValue\": 0,\n    \"maxValue\": 0,\n    \"description\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -978,7 +1011,7 @@
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit"
                 }
               ],
               "variable": []
@@ -1052,7 +1085,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"value\": 0,\n    \"observationId\": \"example\",\n    \"comment\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"novelty\",\n    \"value\": 0.9,\n    \"traceId\": \"cdef-1234-5678-90ab\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -1068,7 +1101,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=&configId=&dataType=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -1091,12 +1124,12 @@
                 {
                   "key": "userId",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this userId associated to the trace."
                 },
                 {
                   "key": "name",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this name."
                 },
                 {
                   "key": "fromTimestamp",
@@ -1122,6 +1155,16 @@
                   "key": "scoreIds",
                   "value": "",
                   "description": "Comma-separated list of score IDs to limit the results to."
+                },
+                {
+                  "key": "configId",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific configId."
+                },
+                {
+                  "key": "dataType",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific dataType."
                 }
               ],
               "variable": []

web/scripts/observations-backfill-calculated-cost.ts

@@ -136,10 +136,10 @@ const backfillCalculatedGenerationCost = async () => {
 
     log("✅ Finished batch update loop.");
 
-    // // Drop the temporary column
-    // log("Dropping temporary column...");
-    // await prisma.$executeRaw`ALTER TABLE observations DROP COLUMN IF EXISTS tmp_has_calculated_cost;`;
-    // log("✅ Dropped temporary column");
+    // Drop the temporary column
+    log("Dropping temporary column...");
+    await prisma.$executeRaw`ALTER TABLE observations DROP COLUMN IF EXISTS tmp_has_calculated_cost;`;
+    log("✅ Dropped temporary column");
 
     log("✅ Finished backfillCalculatedGenerationCost");
   } catch (err) {

web/src/__tests__/score-configs.servertest.ts

@@ -1,24 +1,30 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { v4 as uuidv4 } from "uuid";
-import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
-import { type CastedConfig } from "@langfuse/shared";
-
-const CONFIG_ID_ONE = uuidv4();
-const CONFIG_ID_TWO = uuidv4();
-const CONFIG_ID_THREE = uuidv4();
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  type ScoreConfig,
+  prisma,
+  type ScoreDataType,
+} from "@langfuse/shared/src/db";
+import {
+  GetScoreConfigResponse,
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+} from "@/src/features/public-api/types/score-configs";
 
 const configOne = [
   {
-    id: CONFIG_ID_ONE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Boolean Config",
     description: "Test Description",
-    dataType: ScoreDataType.BOOLEAN,
+    dataType: "BOOLEAN" as ScoreDataType,
     categories: [
-      { label: "False", value: 0 },
       { label: "True", value: 1 },
+      { label: "False", value: 0 },
     ],
     createdAt: new Date("2024-05-10T00:00:00.000Z"),
     updatedAt: new Date("2024-05-10T00:00:00.000Z"),
@@ -26,11 +32,10 @@ const configOne = [
 ];
 const configTwo = [
   {
-    id: CONFIG_ID_TWO,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Numeric Config",
     description: "Test Description",
-    dataType: ScoreDataType.NUMERIC,
+    dataType: "NUMERIC" as ScoreDataType,
     minValue: 0,
     createdAt: new Date("2024-05-11T00:00:00.000Z"),
     updatedAt: new Date("2024-05-11T00:00:00.000Z"),
@@ -39,11 +44,10 @@ const configTwo = [
 
 const configThree = [
   {
-    id: CONFIG_ID_THREE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Categorical Config",
     description: "Test Description",
-    dataType: ScoreDataType.CATEGORICAL,
+    dataType: "CATEGORICAL" as ScoreDataType,
     categories: [
       { label: "A", value: 0 },
       { label: "B", value: 1 },
@@ -69,11 +73,18 @@ describe("/api/public/score-configs API Endpoint", () => {
   );
 
   it("should GET a score config", async () => {
-    const configId = CONFIG_ID_ONE;
+    const { id: configId } = (await prisma.scoreConfig.findFirst({
+      where: {
+        projectId: configOne[0].projectId,
+        name: configOne[0].name,
+      },
+    })) as ScoreConfig;
 
-    const getScoreConfig = await makeAPICall<{
-      id: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(200);
     expect(getScoreConfig.body).toMatchObject({
@@ -84,24 +95,12 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
-  it("test invalid config id input", async () => {
-    const configId = "invalid-config-id";
-
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
-
-    expect(getScoreConfig.status).toBe(404);
-    expect(getScoreConfig.body).toMatchObject({
-      message: "Score config not found within authorized project",
-    });
-  });
-
   it("should GET all score configs", async () => {
-    const fetchedConfigs = await makeAPICall<{
-      data: CastedConfig[];
-      meta: object;
-    }>("GET", `/api/public/score-configs?limit=50&page=1`);
+    const fetchedConfigs = await makeZodVerifiedAPICall(
+      GetScoreConfigsResponse,
+      "GET",
+      `/api/public/score-configs?limit=50&page=1`,
+    );
 
     expect(fetchedConfigs.status).toBe(200);
     expect(fetchedConfigs.body.meta).toMatchObject({
@@ -120,6 +119,20 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
+  it("test invalid config id input", async () => {
+    const configId = "invalid-config-id";
+
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
+
+    expect(getScoreConfig.status).toBe(404);
+    expect(getScoreConfig.body).toMatchObject({
+      message: "Score config not found within authorized project",
+    });
+  });
+
   it("should return 500 when hitting corrupted score config", async () => {
     const configId = "corrupted-config-id";
 
@@ -131,13 +144,234 @@ describe("/api/public/score-configs API Endpoint", () => {
       },
     });
 
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(500);
     expect(getScoreConfig.body).toMatchObject({
-      message: "Internal Server Error",
+      message: "Requested score config is corrupted",
+    });
+  });
+
+  it("should POST a numeric score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "numeric-config-name",
+        dataType: "NUMERIC",
+        maxValue: 0,
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("numeric-config-name");
+    expect(scoreConfig.body.dataType).toBe("NUMERIC");
+    expect(scoreConfig.body.maxValue).toBe(0);
+  });
+
+  it("should POST a boolean score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "boolean-config-name",
+        dataType: "BOOLEAN",
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("boolean-config-name");
+    expect(scoreConfig.body.dataType).toBe("BOOLEAN");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "True", value: 1 },
+      { label: "False", value: 0 },
+    ]);
+  });
+
+  it("should POST a categorical score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("categorical-config-name");
+    expect(scoreConfig.body.dataType).toBe("CATEGORICAL");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "Good", value: 1 },
+      { label: "Bad", value: 0 },
+    ]);
+  });
+
+  it("should fail POST of numeric score config with invalid range", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        PostScoreConfigResponse,
+        "POST",
+        "/api/public/score-configs",
+        {
+          name: "invalid-numeric-config-name",
+          dataType: "NUMERIC",
+          maxValue: 0,
+          minValue: 1,
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"custom\",\"message\":\"Maximum value must be greater than Minimum value\",\"path\":[]}]}`,
+      );
+    }
+  });
+
+  it("should fail POST of boolean score config with custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-boolean-config-name",
+        dataType: "BOOLEAN",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with NO custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with invalid custom categories format", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { key: "first", value: 1 },
+          { key: "second", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Category must be an array of objects with label value pairs, where labels and values are unique.",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category label", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "first", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category label: first, category labels must be unique",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category value", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "second", value: 1 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category value: 1, category values must be unique",
+          path: ["categories"],
+        },
+      ],
     });
   });
 });

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -19,11 +19,11 @@ describe("Traces TRPC Router", () => {
           id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           role: "ADMIN",
           name: "test",
+          cloudConfig: { defaultLookBackDays: null },
         },
       ],
       featureFlags: {
         templateFlag: true,
-        evals: true,
       },
       admin: true,
     },
@@ -36,7 +36,6 @@ describe("Traces TRPC Router", () => {
     const trace = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };
@@ -57,11 +56,42 @@ describe("Traces TRPC Router", () => {
     expect(traces).toMatchObject({ traces: [trace] });
   });
 
+  test("traces.all RPC must not return input, output, metadata", async () => {
+    const trace = {
+      name: "trace-name",
+      userId: "user-1",
+      input: { a: 1 },
+      output: { b: 2 },
+      metadata: { c: 3 },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    await prisma.trace.create({
+      data: { ...trace, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const traces = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: null,
+    });
+    expect(traces.traces).toBeDefined();
+    expect(traces.traces).toHaveLength(1);
+
+    const returnedTrace = traces.traces[0];
+    expect(returnedTrace).not.toHaveProperty("input");
+    expect(returnedTrace).not.toHaveProperty("output");
+    expect(returnedTrace).not.toHaveProperty("metadata");
+  });
+
   test("traces.all RPC orders traces by userId", async () => {
     const traceTmpl = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
         <span key={i} className="group/score ml-1 first:ml-0">
           {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
             ? s.stringValue
-            : s.value.toFixed(2)}
+            : s.value?.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">

web/src/components/table/use-cases/score-configs.tsx

@@ -6,11 +6,7 @@ import { api } from "@/src/utils/api";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import { DataTable } from "@/src/components/table/data-table";
-import {
-  type ScoreDataType,
-  type Prisma,
-  type ConfigCategory,
-} from "@langfuse/shared";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 import {
@@ -28,6 +24,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { type ConfigCategory } from "@/src/features/public-api/types/score-configs";
 
 type ScoreConfigTableRow = {
   id: string;
@@ -38,7 +35,7 @@ type ScoreConfigTableRow = {
   range: {
     maxValue?: number | null;
     minValue?: number | null;
-    categories?: Prisma.JsonValue | null;
+    categories?: ConfigCategory[] | null;
   };
   description?: string | null;
   isArchived: boolean;
@@ -46,7 +43,7 @@ type ScoreConfigTableRow = {
 
 function getConfigRange(
   originalRow: ScoreConfigTableRow,
-): Prisma.JsonValue | undefined {
+): undefined | Prisma.JsonValue {
   const { range, dataType } = originalRow;
 
   if (isNumericDataType(dataType)) {
@@ -57,7 +54,7 @@ function getConfigRange(
   }
 
   if (isCategoricalDataType(dataType) || isBooleanDataType(dataType)) {
-    const configCategories = (range.categories as ConfigCategory[]) ?? [];
+    const configCategories = range.categories ?? [];
 
     return configCategories.reduce(
       (acc, category) => {

web/src/components/table/use-cases/scores.tsx

@@ -16,6 +16,7 @@ import {
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
 import { utcDateOffsetByDays } from "@/src/utils/dates";
+import { isPresent } from "@/src/utils/typeChecks";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
@@ -116,9 +117,18 @@ export default function ScoresTable({
   });
   const totalCount = scores.data?.totalCount ?? 0;
 
-  const filterOptions = api.scores.filterOptions.useQuery({
-    projectId,
-  });
+  const filterOptions = api.scores.filterOptions.useQuery(
+    {
+      projectId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
 
   const rawColumns: LangfuseColumnDef<ScoresTableRow>[] = [
     {
@@ -310,11 +320,12 @@ export default function ScoresTable({
       source: score.source,
       name: score.name,
       dataType: score.dataType,
-      value: isNumericDataType(score.dataType)
-        ? score.value % 1 === 0
-          ? String(score.value)
-          : score.value.toFixed(4)
-        : score.stringValue ?? "",
+      value:
+        isNumericDataType(score.dataType) && isPresent(score.value)
+          ? score.value % 1 === 0
+            ? String(score.value)
+            : score.value.toFixed(4)
+          : score.stringValue ?? "",
       author: {
         image: score.authorUserImage ?? undefined,
         name: score.authorUserName ?? undefined,

web/src/components/table/use-cases/traces.tsx

@@ -34,11 +34,11 @@ import {
   type TraceOptions,
   tracesTableColsWithOptions,
   type ObservationLevel,
-  type Score,
 } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -56,7 +56,7 @@ export type TracesTableRow = {
   input?: unknown;
   output?: unknown;
   metadata?: unknown;
-  scores: Score[];
+  scores: ValidatedScore[];
   tags: string[];
   usage: {
     promptTokens: number;
@@ -463,7 +463,7 @@ export default function TracesTable({
       header: "Scores",
       enableColumnFilter: !omittedFilter.find((f) => f === "scores"),
       cell: ({ row }) => {
-        const values: Score[] = row.getValue("scores");
+        const values: ValidatedScore[] = row.getValue("scores");
         return <GroupedScoreBadges scores={values} variant="headings" />;
       },
       enableHiding: true,

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type ScoreSource, type Score } from "@langfuse/shared";
+import { type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -22,11 +22,12 @@ import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string;
   traceId: string;
 }) => {
@@ -59,7 +60,7 @@ export const ObservationPreview = (props: {
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, ValidatedScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,7 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, type $Enums } from "@langfuse/shared";
+import { type Trace, type $Enums } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -22,7 +23,7 @@ export const ObservationTree = (props: {
   collapseAll: () => void;
   expandAll: () => void;
   trace: Trace;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics: boolean;
@@ -61,7 +62,7 @@ const ObservationTreeTraceNode = (props: {
   trace: Trace & { latency?: number };
   expandAll: () => void;
   collapseAll: () => void;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics?: boolean;
@@ -119,7 +120,7 @@ const ObservationTreeNode = (props: {
   observations: NestedObservation[];
   collapsedObservations: string[];
   toggleCollapsedObservation: (id: string) => void;
-  scores: Score[];
+  scores: ValidatedScore[];
   indentationLevel: number;
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;

web/src/components/trace/ScoresPreview.tsx

@@ -1,10 +1,10 @@
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { type Score } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const ScoresPreview = ({
   itemScoresBySource,
 }: {
-  itemScoresBySource: Map<string, Score[]>;
+  itemScoresBySource: Map<string, ValidatedScore[]>;
 }) => {
   if (!Boolean(itemScoresBySource.size)) return null;
 

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Trace, type Score, type ScoreSource } from "@langfuse/shared";
+import { type Trace, type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -19,6 +19,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const TracePreview = ({
   trace,
@@ -27,7 +28,7 @@ export const TracePreview = ({
 }: {
   trace: Trace & { latency?: number };
   observations: ObservationReturnType[];
-  scores: Score[];
+  scores: ValidatedScore[];
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -41,7 +42,7 @@ export const TracePreview = ({
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, ValidatedScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">

web/src/components/trace/TraceTimelineView.tsx

@@ -1,6 +1,7 @@
 import { Card } from "@/src/components/ui/card";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Score, type Trace } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 import React, { useEffect, useMemo, useRef, useState } from "react";
 import { SimpleTreeView } from "@mui/x-tree-view/SimpleTreeView";
@@ -187,7 +188,7 @@ function TraceTreeItem({
   traceStartTime: Date;
   totalScaleSpan: number;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   observations: Array<ObservationReturnType>;
   cardWidth: number;
 }) {
@@ -266,7 +267,7 @@ export function TraceTimelineView({
   trace: Trace & { latency?: number };
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
 }) {
   const { latency, name, id } = trace;
   const [backgroundColor, setBackgroundColor] = useState("");

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";
@@ -34,11 +34,12 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePos
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
 import { Alert, AlertDescription, AlertTitle } from "@/src/components/ui/alert";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
   trace: Trace;
-  scores: Score[];
+  scores: ValidatedScore[];
   projectId: string;
 }) {
   const capture = usePostHogClientCapture();

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.59.1";
+export const VERSION = "v2.60.0";

web/src/features/dashboard/components/BaseTimeSeriesChart.tsx

@@ -42,7 +42,13 @@ export function BaseTimeSeriesChart(props: {
   }
 
   const convertDate = (date: number, agg: DateTimeAggregationOption) => {
-    if (agg === "24 hours" || agg === "1 hour" || agg === "30 minutes") {
+    const showMinutes: DateTimeAggregationOption[] = [
+      "5 minutes",
+      "30 minutes",
+      "1 hour",
+      "3 hours",
+    ];
+    if (showMinutes.includes(agg)) {
       return new Date(date).toLocaleTimeString("en-US", {
         year: "2-digit",
         month: "numeric",

web/src/features/dashboard/components/MetricTable.tsx

@@ -8,6 +8,8 @@ import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";
 import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 export const MetricTable = ({
   className,
   projectId,
@@ -20,7 +22,9 @@ export const MetricTable = ({
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "totalTokens", agg: "SUM" },

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -20,6 +20,8 @@ import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 export const ModelUsageChart = ({
   className,
   projectId,
@@ -34,7 +36,9 @@ export const ModelUsageChart = ({
   const tokens = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "totalTokens", agg: "SUM" },
         { column: "calculatedTotalCost", agg: "SUM" },

web/src/features/dashboard/components/UserChart.tsx

@@ -15,6 +15,8 @@ import {
   totalCostDashboardFormatted,
 } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 type BarChartDataPoint = {
   name: string;
   value: number;
@@ -34,7 +36,9 @@ export const UserChart = ({
   const user = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "user" },

web/src/features/dashboard/lib/timeseries-aggregation.ts

@@ -6,8 +6,10 @@ export const dateTimeAggregationOptions = [
   "1 month",
   "7 days",
   "24 hours",
+  "3 hours",
   "1 hour",
   "30 minutes",
+  "5 minutes",
 ] as const;
 
 export type DateTimeAggregationOption =
@@ -51,6 +53,12 @@ export const dateTimeAggregationSettings: Record<
       date.toLocaleTimeString("en-US", { hour: "numeric" }),
     minutes: 24 * 60,
   },
+  "3 hours": {
+    date_trunc: "minute",
+    date_formatter: (date) =>
+      date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
+    minutes: 3 * 60,
+  },
   "1 hour": {
     date_trunc: "minute",
     date_formatter: (date) =>
@@ -63,6 +71,12 @@ export const dateTimeAggregationSettings: Record<
       date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
     minutes: 30,
   },
+  "5 minutes": {
+    date_trunc: "minute",
+    date_formatter: (date) =>
+      date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
+    minutes: 5,
+  },
 };
 
 export const findClosestInterval = (

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -7,7 +7,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
-import { type Score } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 import { usdFormatter } from "../../../utils/numbers";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
@@ -30,7 +30,7 @@ type RowData = {
   output?: unknown;
   expectedOutput?: unknown;
 
-  scores: Score[];
+  scores: ValidatedScore[];
   latency?: number;
   totalCost?: string;
 };

web/src/features/datasets/server/dataset-router.ts

@@ -13,6 +13,7 @@ import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { DB } from "@/src/server/db";
 import { paginationZod } from "@langfuse/shared";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 export const datasetRouter = createTRPCRouter({
   allDatasetMeta: protectedProjectProcedure
@@ -168,6 +169,7 @@ export const datasetRouter = createTRPCRouter({
               WHERE 
                 t.project_id = ${input.projectId}
                 AND s.data_type != 'CATEGORICAL'
+                AND s.value IS NOT NULL
                 AND ri.dataset_run_id = runs.id
               GROUP BY s.name
             ) s
@@ -620,6 +622,10 @@ export const datasetRouter = createTRPCRouter({
         `,
       );
 
+      const validatedTraceScores = filterAndValidateDbScoreList(traceScores);
+      const validatedObservationScores =
+        filterAndValidateDbScoreList(observationScores);
+
       const items = runItems.map((ri) => {
         return {
           id: ri.id,
@@ -628,8 +634,8 @@ export const datasetRouter = createTRPCRouter({
           observation: observations.find((o) => o.id === ri.observationId),
           trace: traces.find((t) => t.id === ri.traceId),
           scores: [
-            ...traceScores.filter((s) => s.traceId === ri.traceId),
-            ...observationScores.filter(
+            ...validatedTraceScores.filter((s) => s.traceId === ri.traceId),
+            ...validatedObservationScores.filter(
               (s) =>
                 s.observationId === ri.observationId &&
                 s.traceId === ri.traceId,

web/src/features/manual-scoring/components/AnnotateDrawer.tsx

@@ -31,11 +31,15 @@ import {
   DrawerTrigger,
 } from "@/src/components/ui/drawer";
 import {
-  type ConfigCategory,
   ScoreDataType,
-  type Score,
-  type ScoreConfig,
+  CreateAnnotationScoreData,
+  UpdateAnnotationScoreData,
 } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
+import {
+  type ValidatedScoreConfig,
+  type ConfigCategory,
+} from "@/src/features/public-api/types/score-configs";
 import { z } from "zod";
 import { Input } from "@/src/components/ui/input";
 import {
@@ -56,10 +60,8 @@ import { Textarea } from "@/src/components/ui/textarea";
 import { HoverCardContent } from "@radix-ui/react-hover-card";
 import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
 import { ScoreConfigDetails } from "@/src/features/manual-scoring/components/ScoreConfigDetails";
-import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 import {
   isNumericDataType,
-  isPresent,
   isScoreUnsaved,
 } from "@/src/features/manual-scoring/lib/helpers";
 import { getDefaultScoreData } from "@/src/features/manual-scoring/lib/getDefaultScoreData";
@@ -71,6 +73,7 @@ import { useRouter } from "next/router";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { cn } from "@/src/utils/tailwind";
+import { isPresent } from "@/src/utils/typeChecks";
 
 const AnnotationScoreDataSchema = z.object({
   name: z.string(),
@@ -134,7 +137,7 @@ export function AnnotateDrawer({
   source = "TraceDetail",
 }: {
   traceId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   observationId?: string;
   projectId: string;
   variant?: "button" | "badge";
@@ -178,9 +181,6 @@ export function AnnotateDrawer({
   });
 
   const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: async (data, error) => {
       if (!data || error) return;
 
@@ -214,7 +214,7 @@ export function AnnotateDrawer({
 
   const utils = api.useUtils();
 
-  const onSettledUpsert = async (data?: Score, error?: unknown) => {
+  const onSettledUpsert = async (data?: ValidatedScore, error?: unknown) => {
     if (!data || error) return;
 
     const { id, value, stringValue, name, dataType, configId, comment } = data;
@@ -240,16 +240,10 @@ export function AnnotateDrawer({
   };
 
   const mutCreateScores = api.scores.createAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: onSettledUpsert,
   });
 
   const mutUpdateScores = api.scores.updateAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: onSettledUpsert,
   });
 
@@ -314,27 +308,43 @@ export function AnnotateDrawer({
 
         if (!!stringValue) {
           if (!!score.scoreId) {
-            await mutUpdateScores.mutateAsync({
-              projectId,
-              ...score,
+            const validatedScore = UpdateAnnotationScoreData.parse({
               id: score.scoreId,
+              projectId,
+              traceId,
+              name: score.name,
+              dataType: score.dataType,
+              configId: score.configId,
+              comment: score.comment,
+              observationId,
               value: newValue,
               stringValue,
             });
+
+            await mutUpdateScores.mutateAsync({
+              ...validatedScore,
+            });
             capture("score:update", {
               type: type,
               source: source,
               dataType: score.dataType,
             });
           } else {
-            await mutCreateScores.mutateAsync({
+            const validatedScore = CreateAnnotationScoreData.parse({
               projectId,
               traceId,
-              ...score,
+              name: score.name,
+              dataType: score.dataType,
+              configId: score.configId,
+              comment: score.comment,
               observationId,
               value: newValue,
               stringValue,
             });
+
+            await mutCreateScores.mutateAsync({
+              ...validatedScore,
+            });
             capture("score:create", {
               type: type,
               source: source,
@@ -361,13 +371,23 @@ export function AnnotateDrawer({
     return async () => {
       const { value, scoreId } = score;
       if (!!field.value && !!scoreId && isPresent(value)) {
-        await mutUpdateScores.mutateAsync({
-          projectId,
-          ...score,
-          value,
+        const validatedScore = UpdateAnnotationScoreData.parse({
           id: scoreId,
+          projectId,
+          traceId,
+          name: score.name,
+          dataType: score.dataType,
+          configId: score.configId,
+          stringValue: score.stringValue,
+          observationId,
+          value,
           comment,
         });
+
+        await mutUpdateScores.mutateAsync({
+          ...validatedScore,
+        });
+
         capture(comment ? "score:update_comment" : "score:delete_comment", {
           type: type,
           source: source,
@@ -382,7 +402,7 @@ export function AnnotateDrawer({
     index,
     score,
   }: {
-    config: ScoreConfig;
+    config: ValidatedScoreConfig;
     field: ControllerRenderProps<
       AnnotateFormSchemaType,
       `scoreData.${number}.value`
@@ -409,25 +429,45 @@ export function AnnotateDrawer({
 
       if (isPresent(field.value)) {
         if (!!score.scoreId) {
-          await mutUpdateScores.mutateAsync({
-            projectId,
-            ...score,
-            value: Number(field.value),
+          const validatedScore = UpdateAnnotationScoreData.parse({
             id: score.scoreId,
+            projectId,
+            traceId,
+            name: score.name,
+            dataType: score.dataType,
+            configId: score.configId,
+            stringValue: score.stringValue,
+            comment: score.comment,
+            observationId,
+            value: Number(field.value),
           });
+
+          await mutUpdateScores.mutateAsync({
+            ...validatedScore,
+          });
+
           capture("score:update", {
             type: type,
             source: source,
             dataType: score.dataType,
           });
         } else {
-          await mutCreateScores.mutateAsync({
+          const validatedScore = CreateAnnotationScoreData.parse({
             projectId,
             traceId,
-            ...score,
+            name: score.name,
+            dataType: score.dataType,
+            configId: score.configId,
+            stringValue: score.stringValue,
+            comment: score.comment,
             observationId,
             value: Number(field.value),
           });
+
+          await mutCreateScores.mutateAsync({
+            ...validatedScore,
+          });
+
           capture("score:create", {
             type: type,
             source: source,

web/src/features/manual-scoring/components/CreateScoreConfigButton.tsx

@@ -1,7 +1,6 @@
 import React, { useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { z } from "zod";
 import {
   Dialog,
   DialogContent,
@@ -21,7 +20,7 @@ import {
   FormMessage,
 } from "@/src/components/ui/form";
 import { Input } from "@/src/components/ui/input";
-import { ScoreDataType } from "@langfuse/shared";
+import { ScoreDataType, availableDataTypes } from "@langfuse/shared";
 import {
   Select,
   SelectContent,
@@ -36,29 +35,66 @@ import {
   isCategoricalDataType,
   isNumericDataType,
 } from "@/src/features/manual-scoring/lib/helpers";
+import { isPresent } from "@/src/utils/typeChecks";
 import DocPopup from "@/src/components/layouts/doc-popup";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { z } from "zod";
 
-const availableDataTypes = [
-  ScoreDataType.NUMERIC,
-  ScoreDataType.CATEGORICAL,
-  ScoreDataType.BOOLEAN,
-] as const;
-
-const category = z.object({
+const Category = z.object({
   label: z.string().min(1),
-  value: z.coerce.number(),
+  value: z.number(),
 });
 
-const formSchema = z.object({
+const createConfigSchema = z.object({
   name: z.string().min(1).max(35),
   dataType: z.enum(availableDataTypes),
   minValue: z.coerce.number().optional(),
   maxValue: z.coerce.number().optional(),
-  categories: z.array(category).optional(),
+  categories: z.array(Category).optional(),
   description: z.string().optional(),
 });
 
+type CreateConfig = z.infer<typeof createConfigSchema>;
+
+const validateScoreConfig = (values: CreateConfig): string | null => {
+  const { dataType, maxValue, minValue, categories } = values;
+
+  if (isNumericDataType(dataType)) {
+    if (isPresent(maxValue) && isPresent(minValue) && maxValue <= minValue) {
+      return "Maximum value must be greater than Minimum value.";
+    }
+  } else if (isCategoricalDataType(dataType)) {
+    if (!categories || categories.length === 0) {
+      return "At least one category is required for categorical data types.";
+    }
+  } else if (isBooleanDataType(dataType)) {
+    if (categories?.length !== 2)
+      return "Boolean data type must have exactly 2 categories.";
+    const isBooleanCategoryInvalid = categories?.some(
+      (category) => category.value !== 0 && category.value !== 1,
+    );
+    if (isBooleanCategoryInvalid)
+      return "Boolean data type must have categories with values 0 and 1.";
+  }
+
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories || []) {
+    if (uniqueNames.has(category.label)) {
+      return "Category names must be unique.";
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      return "Category values must be unique.";
+    }
+    uniqueValues.add(category.value);
+  }
+
+  return null;
+};
+
 export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
   const [open, setOpen] = useState(false);
   const [formError, setFormError] = useState<string | null>(null);
@@ -76,8 +112,8 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
       setFormError(error.message ?? "An error occurred while creating config."),
   });
 
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
+  const form = useForm<CreateConfig>({
+    resolver: zodResolver(createConfigSchema),
     defaultValues: {
       dataType: ScoreDataType.NUMERIC,
       minValue: undefined,
@@ -93,8 +129,8 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
 
   if (!hasAccess) return null;
 
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    const error = validateForm(values);
+  function onSubmit(values: CreateConfig) {
+    const error = validateScoreConfig(values);
     setFormError(error);
     if (error) return;
 
@@ -380,44 +416,3 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
     </>
   );
 }
-
-function validateForm(values: z.infer<typeof formSchema>): string | null {
-  if (isNumericDataType(values.dataType)) {
-    if (
-      !!values.maxValue &&
-      !!values.minValue &&
-      values.maxValue <= values.minValue
-    ) {
-      return "Maximum value must be greater than Minimum value.";
-    }
-  } else if (isCategoricalDataType(values.dataType)) {
-    if (!values.categories || values.categories.length === 0) {
-      return "At least one category is required for categorical data types.";
-    }
-  } else if (isBooleanDataType(values.dataType)) {
-    if (values.categories?.length !== 2)
-      return "Boolean data type must have exactly 2 categories.";
-    const isBooleanCategoryInvalid = values.categories?.some(
-      (category) => category.value !== 0 && category.value !== 1,
-    );
-    if (isBooleanCategoryInvalid)
-      return "Boolean data type must have categories with values 0 and 1.";
-  }
-
-  const uniqueNames = new Set<string>();
-  const uniqueValues = new Set<number>();
-
-  for (const category of values.categories || []) {
-    if (uniqueNames.has(category.label)) {
-      return "Category names must be unique.";
-    }
-    uniqueNames.add(category.label);
-
-    if (uniqueValues.has(category.value)) {
-      return "Category values must be unique.";
-    }
-    uniqueValues.add(category.value);
-  }
-
-  return null;
-}

web/src/features/manual-scoring/components/ScoreConfigDetails.tsx

@@ -1,11 +1,13 @@
-import {
-  isNumericDataType,
-  isPresent,
-} from "@/src/features/manual-scoring/lib/helpers";
-import { type ScoreConfig } from "@langfuse/shared";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
+import { type ValidatedScoreConfig } from "@/src/features/public-api/types/score-configs";
+import { isPresent } from "@/src/utils/typeChecks";
 import React from "react";
 
-export function ScoreConfigDetails({ config }: { config: ScoreConfig }) {
+export function ScoreConfigDetails({
+  config,
+}: {
+  config: ValidatedScoreConfig;
+}) {
   const { name, description, minValue, maxValue, dataType } = config;
   if (!description && !isPresent(minValue) && !isPresent(maxValue)) return null;
   const isNameTruncated = name.length > 20;

web/src/features/manual-scoring/lib/getDefaultScoreData.ts

@@ -1,4 +1,6 @@
-import { type Score, ScoreSource, type ScoreConfig } from "@langfuse/shared";
+import { ScoreSource } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
+import { type ValidatedScoreConfig } from "@/src/features/public-api/types/score-configs";
 
 export const getDefaultScoreData = ({
   scores,
@@ -7,9 +9,9 @@ export const getDefaultScoreData = ({
   traceId,
   observationId,
 }: {
-  scores: Score[];
+  scores: ValidatedScore[];
   emptySelectedConfigIds: string[];
-  configs: ScoreConfig[];
+  configs: ValidatedScoreConfig[];
   traceId: string;
   observationId?: string;
 }) => {

web/src/features/manual-scoring/lib/helpers.ts

@@ -1,8 +1,4 @@
-import {
-  type ScoreConfig,
-  ScoreDataType,
-  type CastedConfig,
-} from "@langfuse/shared";
+import { ScoreDataType } from "@langfuse/shared";
 
 export const isNumericDataType = (dataType: ScoreDataType) =>
   dataType === ScoreDataType.NUMERIC;
@@ -13,21 +9,4 @@ export const isCategoricalDataType = (dataType: ScoreDataType) =>
 export const isBooleanDataType = (dataType: ScoreDataType) =>
   dataType === ScoreDataType.BOOLEAN;
 
-export const isPresent = <T>(value: T): value is NonNullable<T> =>
-  value !== null && value !== undefined && value !== "";
-
 export const isScoreUnsaved = (scoreId?: string): boolean => !scoreId;
-
-export const isCastedConfig = (config: ScoreConfig): config is CastedConfig => {
-  return (
-    config.categories === null ||
-    (Array.isArray(config.categories) &&
-      config.categories.every(
-        (category) =>
-          category !== null &&
-          typeof category === "object" &&
-          "label" in category &&
-          "value" in category,
-      ))
-  );
-};

web/src/features/prompts/server/routers/promptRouter.ts

@@ -635,6 +635,7 @@ export const promptRouter = createTRPCRouter({
           WHERE
               o.type = 'GENERATION'
               AND s.data_type != 'CATEGORICAL'
+              AND s.value IS NOT NULL
               AND o.prompt_id IS NOT NULL
               AND o.project_id = ${input.projectId}
               AND p.id IN (${Prisma.join(input.promptIds)})
@@ -692,6 +693,7 @@ export const promptRouter = createTRPCRouter({
           LEFT JOIN scores s ON tp.trace_id = s.trace_id AND s.observation_id IS NULL AND s.project_id = ${input.projectId}
           WHERE 
               s.data_type != 'CATEGORICAL'
+              AND s.value IS NOT NULL
         ), average_scores_by_prompt AS (
           SELECT 
               prompt_id,

web/src/features/public-api/types/score-configs.ts

@@ -0,0 +1,210 @@
+import { isPresent } from "@/src/utils/typeChecks";
+import {
+  jsonSchema,
+  paginationMetaResponseZod,
+  paginationZod,
+  type ScoreConfig as ScoreConfigDbType,
+} from "@langfuse/shared";
+import { z } from "zod";
+import * as Sentry from "@sentry/node";
+
+const validateCategories = (
+  categories: ConfigCategory[],
+  ctx: z.RefinementCtx,
+) => {
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories) {
+    if (uniqueNames.has(category.label)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category label: ${category.label}, category labels must be unique`,
+      });
+      return;
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category value: ${category.value}, category values must be unique`,
+      });
+      return;
+    }
+    uniqueValues.add(category.value);
+  }
+};
+
+/**
+ * Objects
+ */
+export const Category = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+export type ConfigCategory = z.infer<typeof Category>;
+
+const Categories = z.array(Category);
+
+const NumericScoreConfig = z.object({
+  maxValue: z.number().optional().nullish(),
+  minValue: z.number().optional().nullish(),
+  dataType: z.literal("NUMERIC"),
+  categories: z.undefined().nullish(),
+});
+
+const CategoricalScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("CATEGORICAL"),
+  categories: jsonSchema.superRefine((categories, ctx) => {
+    const parseResult = Categories.safeParse(categories);
+    if (!parseResult.success) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "Category must be an array of objects with label value pairs, where labels and values are unique.",
+      });
+      return;
+    }
+
+    validateCategories(categories as ConfigCategory[], ctx);
+  }),
+});
+
+const BooleanScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("BOOLEAN"),
+  categories: z
+    .array(Category)
+    .length(2, "Boolean data type must have exactly 2 categories.")
+    .refine((categories) => {
+      const expectedCategories = [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ];
+      return categories.every(
+        (category, index) =>
+          category.label === expectedCategories[index].label &&
+          category.value === expectedCategories[index].value,
+      );
+    }),
+});
+
+const ScoreConfigBase = z.object({
+  id: z.string(),
+  name: z.string().min(1).max(35),
+  isArchived: z.boolean(),
+  description: z.string().optional().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  projectId: z.string(),
+});
+
+const ScoreConfigPostBase = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+});
+
+const ValidatedScoreConfigSchema = z
+  .union([
+    ScoreConfigBase.merge(NumericScoreConfig),
+    ScoreConfigBase.merge(
+      z.object({
+        maxValue: z.undefined().nullish(),
+        minValue: z.undefined().nullish(),
+        dataType: z.literal("CATEGORICAL"),
+        categories: Categories.superRefine(validateCategories),
+      }),
+    ),
+    ScoreConfigBase.merge(BooleanScoreConfig),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
+
+export const filterAndValidateDbScoreConfigList = (
+  scoreConfigs: ScoreConfigDbType[],
+): ValidatedScoreConfig[] =>
+  scoreConfigs.reduce((acc, ts) => {
+    const result = ValidatedScoreConfigSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      Sentry.captureException(result.error);
+    }
+    return acc;
+  }, [] as ValidatedScoreConfig[]);
+
+export const validateDbScoreConfig = (
+  scoreConfig: ScoreConfigDbType,
+): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
+
+export const validateDbScoreConfigSafe = (scoreConfig: ScoreConfigDbType) =>
+  ValidatedScoreConfigSchema.safeParse(scoreConfig);
+
+/**
+ * Endpoints
+ */
+
+// GET /score-configs/{configId}
+export const GetScoreConfigQuery = z.object({
+  configId: z.string(),
+});
+
+export const GetScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// POST /score-configs
+export const PostScoreConfigBody = z
+  .union([
+    ScoreConfigPostBase.merge(CategoricalScoreConfig),
+    ScoreConfigPostBase.merge(NumericScoreConfig),
+    ScoreConfigPostBase.merge(
+      z.object({
+        dataType: z.literal("BOOLEAN"),
+        categories: z.undefined().nullish(),
+      }),
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// GET /score-configs
+export const GetScoreConfigsQuery = z.object({
+  ...paginationZod,
+});
+
+export const GetScoreConfigsResponse = z.object({
+  data: z.array(ValidatedScoreConfigSchema),
+  meta: paginationMetaResponseZod,
+});

web/src/features/public-api/types/scores.ts

@@ -0,0 +1,293 @@
+import * as Sentry from "@sentry/node";
+import {
+  paginationZod,
+  paginationMetaResponseZod,
+  NonEmptyString,
+  type Score,
+} from "@langfuse/shared";
+import { z } from "zod";
+import { isPresent } from "@/src/utils/typeChecks";
+import { Category as ConfigCategory } from "./score-configs";
+
+/**
+ * Objects
+ */
+
+const ScoreSource = ["API", "EVAL", "ANNOTATION"] as const;
+const ScoreDataType = ["NUMERIC", "CATEGORICAL", "BOOLEAN"] as const;
+
+const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
+
+const NumericData = z.object({
+  value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
+});
+
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
+
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
+
+const ScoreBase = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  projectId: z.string(),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  authorUserId: z.string().nullish(),
+  comment: z.string().nullish(),
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  configId: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+const BaseScoreBody = z.object({
+  id: z.string().nullish(),
+  name: NonEmptyString,
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  comment: z.string().nullish(),
+});
+
+const GetScoresDataBase = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  comment: z.string().nullish(),
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  trace: z.object({
+    userId: z.string(),
+  }),
+  configId: z.string().nullish(),
+});
+
+export const GetScoresData = z.discriminatedUnion("dataType", [
+  GetScoresDataBase.merge(NumericData),
+  GetScoresDataBase.merge(CategoricalData),
+  GetScoresDataBase.merge(BooleanData),
+]);
+
+const ValidatedScoreSchema = z.discriminatedUnion("dataType", [
+  ScoreBase.merge(NumericData),
+  ScoreBase.merge(CategoricalData),
+  ScoreBase.merge(BooleanData),
+]);
+
+export type ValidatedGetScoresData = z.infer<typeof GetScoresData>;
+
+export type ValidatedScore = z.infer<typeof ValidatedScoreSchema>;
+
+/**
+ * Validation objects
+ */
+export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number(),
+      dataType: z.literal("NUMERIC"),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.string(),
+      dataType: z.literal("CATEGORICAL"),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number().refine((val) => val === 0 || val === 1, {
+        message: "Value must be either 0 or 1",
+      }),
+      dataType: z.literal("BOOLEAN"),
+    }),
+  ),
+]);
+
+const ScorePropsAgainstConfigNumeric = z
+  .object({
+    value: z.number(),
+    maxValue: z.number().optional(),
+    minValue: z.number().optional(),
+    dataType: z.literal("NUMERIC"),
+  })
+  .superRefine((data, ctx) => {
+    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
+      });
+    }
+    if (isPresent(data.minValue) && data.value <= data.minValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value is below minimum value of ${data.minValue} defined in config`,
+      });
+    }
+  });
+
+const ScorePropsAgainstConfigCategorical = z
+  .object({
+    value: z.string(),
+    categories: z.array(ConfigCategory),
+    dataType: z.literal("CATEGORICAL"),
+  })
+  .superRefine((data, ctx) => {
+    if (!data.categories.some(({ label }) => label === data.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value ${data.value} does not map to a valid category. Pass a valid category value.`,
+      });
+    }
+  });
+
+export const ScorePropsAgainstConfig = z.union([
+  ScorePropsAgainstConfigNumeric,
+  ScorePropsAgainstConfigCategorical,
+  z.object({
+    value: z.number().refine((val) => val === 0 || val === 1, {
+      message: "Value must be either 0 or 1",
+    }),
+    dataType: z.literal("BOOLEAN"),
+  }),
+]);
+
+/**
+ * Transformations
+ */
+
+export const filterAndValidateDbScoreList = (
+  scores: Score[],
+): ValidatedScore[] =>
+  scores.reduce((acc, ts) => {
+    const result = ValidatedScoreSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      Sentry.captureException(result.error);
+    }
+    return acc;
+  }, [] as ValidatedScore[]);
+
+export const validateDbScore = (score: Score): ValidatedScore =>
+  ValidatedScoreSchema.parse(score);
+
+/**
+ * Endpoints
+ */
+
+// POST /scores
+/**
+ * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
+ */
+export const PostScoresBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      }),
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
+export const PostScoresResponse = z.void();
+
+// GET /scores
+export const GetScoresQuery = z.object({
+  ...paginationZod,
+  userId: z.string().nullish(),
+  dataType: z.enum(ScoreDataType).nullish(),
+  configId: z.string().nullish(),
+  name: z.string().nullish(),
+  fromTimestamp: z.coerce.date().nullish(),
+  source: z.enum(ScoreSource).nullish(),
+  value: z.coerce.number().nullish(),
+  operator: z.enum(operators).nullish(),
+  scoreIds: z
+    .string()
+    .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
+    .refine((arr) => arr.every((id) => typeof id === "string"), {
+      message: "Each score ID must be a string",
+    })
+    .nullish(),
+});
+
+export const GetScoresResponse = z.object({
+  data: z.array(GetScoresData),
+  meta: paginationMetaResponseZod,
+});
+
+// GET /scores/{scoreId}
+export const GetScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const GetScoreResponse = ValidatedScoreSchema;
+
+// DELETE /scores/{scoreId}
+export const DeleteScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const DeleteScoreResponse = z.object({
+  message: z.string(),
+});

web/src/pages/api/public/score-configs.ts

@@ -1,109 +1,92 @@
-import { type CastedConfig, paginationZod } from "@langfuse/shared";
-import { z } from "zod";
-import { prisma } from "@langfuse/shared/src/db";
-import { Prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
-import { isCastedConfig } from "@/src/features/manual-scoring/lib/helpers";
+import { type z } from "zod";
+import { Prisma, prisma } from "@langfuse/shared/src/db";
+import { isBooleanDataType } from "@/src/features/manual-scoring/lib/helpers";
+import { v4 } from "uuid";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+  GetScoreConfigsQuery,
+  PostScoreConfigBody,
+  validateDbScoreConfig,
+  filterAndValidateDbScoreConfigList,
+} from "@/src/features/public-api/types/score-configs";
 
-const ScoreConfigsGetSchema = z.object({
-  ...paginationZod,
-});
+const inflateConfigBody = (body: z.infer<typeof PostScoreConfigBody>) => {
+  if (isBooleanDataType(body.dataType)) {
+    return {
+      ...body,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+    };
+  }
+  return body;
+};
 
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
+export default withMiddlewares({
+  POST: createAuthedAPIRoute({
+    name: "Create Score Config",
+    bodySchema: PostScoreConfigBody,
+    responseSchema: PostScoreConfigResponse,
+    fn: async ({ body, auth }) => {
+      const inflatedConfigInput = inflateConfigBody(body);
 
-  try {
-    // CHECK AUTH
-    const authCheck = await verifyAuthHeaderAndReturnScope(
-      req.headers.authorization,
-    );
-    if (!authCheck.validKey) {
-      return res.status(401).json({
-        message: authCheck.error,
+      const config = await prisma.scoreConfig.create({
+        data: {
+          ...inflatedConfigInput,
+          categories: inflatedConfigInput.categories ?? undefined,
+          id: v4(),
+          projectId: auth.scope.projectId,
+        },
       });
-    }
-    // END CHECK AUTH
-
-    if (req.method !== "GET") {
-      return res.status(405).json({ message: "Method not allowed" });
-    }
-
-    if (req.method === "GET") {
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-
-      const obj = ScoreConfigsGetSchema.parse(req.query);
 
+      return validateDbScoreConfig(config);
+    },
+  }),
+  GET: createAuthedAPIRoute({
+    name: "Get Score Configs",
+    querySchema: GetScoreConfigsQuery,
+    responseSchema: GetScoreConfigsResponse,
+    fn: async ({ query, auth }) => {
+      const { page, limit } = query;
       const rawConfigs = await prisma.scoreConfig.findMany({
         where: {
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
         orderBy: {
           createdAt: "desc",
         },
-        take: obj.limit,
-        skip: (obj.page - 1) * obj.limit,
+        take: limit,
+        skip: (page - 1) * limit,
       });
 
-      const configs: CastedConfig[] = rawConfigs.filter(isCastedConfig);
-
-      if (configs.length !== rawConfigs.length) {
-        return res.status(500).json({
-          message: "Internal Server Error",
-          error: "Invalid config format encountered",
-        });
-      }
+      const configs = filterAndValidateDbScoreConfigList(rawConfigs);
 
       const totalItemsRes = await prisma.$queryRaw<{ count: bigint }[]>(
         Prisma.sql`
-          SELECT 
+          SELECT
             COUNT(*) as count
-          FROM 
+          FROM
             "score_configs" AS sc
-          WHERE sc.project_id = ${authCheck.scope.projectId}
+          WHERE sc.project_id = ${auth.scope.projectId}
         `,
       );
 
       const totalItems =
         totalItemsRes[0] !== undefined ? Number(totalItemsRes[0].count) : 0;
 
-      return res.status(200).json({
+      return {
         data: configs,
         meta: {
-          page: obj.page,
-          limit: obj.limit,
+          page: page,
+          limit: limit,
           totalItems,
-          totalPages: Math.ceil(totalItems / obj.limit),
+          totalPages: Math.ceil(totalItems / limit),
         },
-      });
-    }
-  } catch (error: unknown) {
-    console.error(error);
-    if (isPrismaException(error)) {
-      return res.status(500).json({
-        error: "Internal Server Error",
-      });
-    }
-    if (error instanceof z.ZodError) {
-      return res.status(400).json({
-        message: "Invalid request data",
-        error: error.errors,
-      });
-    }
-    const errorMessage =
-      error instanceof Error ? error.message : "An unknown error occurred";
-    res.status(500).json({
-      message: "Invalid request data",
-      error: errorMessage,
-    });
-  }
-}
+      };
+    },
+  }),
+});

web/src/pages/api/public/score-configs/[configId].ts

@@ -1,85 +1,39 @@
 import { prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
-import { isCastedConfig } from "@/src/features/manual-scoring/lib/helpers";
-
-const ConfigSchema = z.object({
-  configId: z.string(),
-});
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
-
-  if (req.method === "GET") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-
-      const { configId } = ConfigSchema.parse(req.query);
-
-      // CHECK ACCESS SCOPE
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-      // END CHECK ACCESS SCOPE
+import { InternalServerError, LangfuseNotFoundError } from "@langfuse/shared";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  GetScoreConfigQuery,
+  GetScoreConfigResponse,
+} from "@/src/features/public-api/types/score-configs";
+import * as Sentry from "@sentry/node";
 
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get a Score Config",
+    querySchema: GetScoreConfigQuery,
+    responseSchema: GetScoreConfigResponse,
+    fn: async ({ query, auth }) => {
       const config = await prisma.scoreConfig.findUnique({
         where: {
-          id: configId,
-          projectId: authCheck.scope.projectId,
+          id: query.configId,
+          projectId: auth.scope.projectId,
         },
       });
 
       if (!config) {
-        return res.status(404).json({
-          message: "Score config not found within authorized project",
-        });
+        throw new LangfuseNotFoundError(
+          "Score config not found within authorized project",
+        );
       }
 
-      if (isCastedConfig(config)) {
-        return res.status(200).json(config);
-      } else {
-        return res.status(500).json({
-          message: "Internal Server Error",
-          error: "Invalid config format encountered",
-        });
+      const parsedConfig = GetScoreConfigResponse.safeParse(config);
+      if (!parsedConfig.success) {
+        Sentry.captureException(parsedConfig.error);
+        throw new InternalServerError("Requested score config is corrupted");
       }
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+
+      return parsedConfig.data;
+    },
+  }),
+});

web/src/pages/api/public/scores.ts

@@ -1,151 +1,109 @@
-import { ScoreSource, prisma } from "@langfuse/shared/src/db";
-import { Prisma, type Score } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { paginationZod } from "@langfuse/shared";
-import {
-  ScoreBody,
-  eventTypes,
-  ingestionBatchEvent,
-  stringDate,
-} from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import { Prisma } from "@langfuse/shared/src/db";
+import { eventTypes, ingestionBatchEvent } from "@langfuse/shared";
+import * as Sentry from "@sentry/node";
 import { v4 } from "uuid";
 import {
   handleBatch,
   handleBatchResultLegacy,
 } from "@/src/pages/api/public/ingestion";
-import { isPrismaException } from "@/src/utils/exceptions";
-
-const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
-
-const ScoresGetSchema = z
-  .object({
-    ...paginationZod,
-    userId: z.string().nullish(),
-    name: z.string().nullish(),
-    fromTimestamp: stringDate,
-    source: z.nativeEnum(ScoreSource).nullish(),
-    value: z.coerce.number().nullish(),
-    operator: z.enum(operators).nullish(),
-    scoreIds: z
-      .string()
-      .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
-      .refine((arr) => arr.every((id) => typeof id === "string"), {
-        message: "Each score ID must be a string",
-      })
-      .nullish(),
-  })
-  .strict(); // Use strict to give 400s on typo'd query params
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
-
-  // CHECK AUTH
-  const authCheck = await verifyAuthHeaderAndReturnScope(
-    req.headers.authorization,
-  );
-  if (!authCheck.validKey)
-    return res.status(401).json({
-      message: authCheck.error,
-    });
-  // END CHECK AUTH
-
-  if (req.method === "POST") {
-    try {
-      console.log(
-        "trying to create score, project ",
-        authCheck.scope.projectId,
-        ", body:",
-        JSON.stringify(req.body, null, 2),
-      );
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  GetScoresData,
+  GetScoresQuery,
+  GetScoresResponse,
+  PostScoresBody,
+  PostScoresResponse,
+  type ValidatedGetScoresData,
+} from "@/src/features/public-api/types/scores";
 
+export default withMiddlewares({
+  POST: createAuthedAPIRoute({
+    name: "Create Score",
+    bodySchema: PostScoresBody,
+    responseSchema: PostScoresResponse,
+    fn: async ({ body, auth, req, res }) => {
       const event = {
         id: v4(),
         type: eventTypes.SCORE_CREATE,
         timestamp: new Date().toISOString(),
-        body: ScoreBody.parse(req.body),
+        body,
       };
-
       const result = await handleBatch(
         ingestionBatchEvent.parse([event]),
         {},
         req,
-        authCheck,
+        auth,
       );
-
       handleBatchResultLegacy(result.errors, result.results, res);
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else if (req.method === "GET") {
-    try {
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
+    },
+  }),
+  GET: createAuthedAPIRoute({
+    name: "Get Scores",
+    querySchema: GetScoresQuery,
+    responseSchema: GetScoresResponse,
+    fn: async ({ query, auth }) => {
+      const {
+        page,
+        limit,
+        configId,
+        userId,
+        name,
+        fromTimestamp,
+        source,
+        operator,
+        value,
+        scoreIds,
+        dataType,
+      } = query;
 
-      const obj = ScoresGetSchema.parse(req.query); // uses query and not body
-
-      const skipValue = (obj.page - 1) * obj.limit;
-      const userCondition = obj.userId
-        ? Prisma.sql`AND t."user_id" = ${obj.userId}`
+      const skipValue = (page - 1) * limit;
+      const configCondition = configId
+        ? Prisma.sql`AND s."config_id" = ${configId}`
         : Prisma.empty;
-      const nameCondition = obj.name
-        ? Prisma.sql`AND s."name" = ${obj.name}`
+      const dataTypeCondition = dataType
+        ? Prisma.sql`AND s."data_type" = ${dataType}::"ScoreDataType"`
         : Prisma.empty;
-      const fromTimestampCondition = obj.fromTimestamp
-        ? Prisma.sql`AND s."timestamp" >= ${obj.fromTimestamp}::timestamp with time zone at time zone 'UTC'`
+      const userCondition = userId
+        ? Prisma.sql`AND t."user_id" = ${userId}`
         : Prisma.empty;
-      const sourceCondition = obj.source
-        ? Prisma.sql`AND s."source" = ${obj.source}`
+      const nameCondition = name
+        ? Prisma.sql`AND s."name" = ${name}`
+        : Prisma.empty;
+      const fromTimestampCondition = fromTimestamp
+        ? Prisma.sql`AND s."timestamp" >= ${fromTimestamp}::timestamp with time zone at time zone 'UTC'`
+        : Prisma.empty;
+      const sourceCondition = source
+        ? Prisma.sql`AND s."source" = ${source}`
         : Prisma.empty;
       const valueCondition =
-        obj.operator && obj.value !== null && obj.value !== undefined
-          ? Prisma.sql`AND s."value" ${Prisma.raw(`${obj.operator}`)} ${obj.value}`
+        operator && value !== null && value !== undefined
+          ? Prisma.sql`AND s."value" ${Prisma.raw(`${operator}`)} ${value}`
           : Prisma.empty;
-      const scoreIdCondition = obj.scoreIds
-        ? Prisma.sql`AND s."id" = ANY(${obj.scoreIds})`
+      const scoreIdCondition = scoreIds
+        ? Prisma.sql`AND s."id" = ANY(${scoreIds})`
         : Prisma.empty;
 
-      const scores = await prisma.$queryRaw<
-        Array<Score & { trace: { userId: string } }>
-      >(Prisma.sql`
+      const scores = await prisma.$queryRaw<Array<unknown>>(Prisma.sql`
           SELECT
             s.id,
             s.timestamp,
             s.name,
             s.value,
+            s.string_value as "stringValue",
             s.source,
             s.comment,
+            s.data_type as "dataType",
+            s.config_id as "configId",
             s.trace_id as "traceId",
             s.observation_id as "observationId",
             json_build_object('userId', t.user_id) as "trace"
           FROM "scores" AS s
-          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${authCheck.scope.projectId}
-          WHERE s.project_id = ${authCheck.scope.projectId}
+          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${auth.scope.projectId}
+          WHERE s.project_id = ${auth.scope.projectId}
+          ${configCondition}
+          ${dataTypeCondition}
           ${userCondition}
           ${nameCondition}
           ${sourceCondition}
@@ -153,15 +111,17 @@ export default async function handler(
           ${valueCondition}
           ${scoreIdCondition}
           ORDER BY s."timestamp" DESC
-          LIMIT ${obj.limit} OFFSET ${skipValue}
+          LIMIT ${limit} OFFSET ${skipValue}
           `);
 
       const totalItemsRes = await prisma.$queryRaw<{ count: bigint }[]>(
         Prisma.sql`
           SELECT COUNT(*) as count
           FROM "scores" AS s
-          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${authCheck.scope.projectId}
-          WHERE s.project_id = ${authCheck.scope.projectId}
+          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${auth.scope.projectId}
+          WHERE s.project_id = ${auth.scope.projectId}
+          ${configCondition}
+          ${dataTypeCondition}
           ${userCondition}
           ${nameCondition}
           ${sourceCondition}
@@ -171,39 +131,31 @@ export default async function handler(
         `,
       );
 
+      const validatedScores = scores.reduce(
+        (acc: ValidatedGetScoresData[], score) => {
+          const result = GetScoresData.safeParse(score);
+          if (result.success) {
+            acc.push(result.data);
+          } else {
+            Sentry.captureException(result.error);
+          }
+          return acc;
+        },
+        [] as ValidatedGetScoresData[],
+      );
+
       const totalItems =
         totalItemsRes[0] !== undefined ? Number(totalItemsRes[0].count) : 0;
 
-      return res.status(200).json({
-        data: scores,
+      return {
+        data: validatedScores,
         meta: {
-          page: obj.page,
-          limit: obj.limit,
+          page: page,
+          limit: limit,
           totalItems,
-          totalPages: Math.ceil(totalItems / obj.limit),
+          totalPages: Math.ceil(totalItems / limit),
         },
-      });
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+      };
+    },
+  }),
+});

web/src/pages/api/public/scores/[scoreId].ts

@@ -1,38 +1,50 @@
 import { prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import * as Sentry from "@sentry/node";
+import {
+  DeleteScoreQuery,
+  DeleteScoreResponse,
+  GetScoreQuery,
+  GetScoreResponse,
+} from "@/src/features/public-api/types/scores";
+import { InternalServerError, LangfuseNotFoundError } from "@langfuse/shared";
 
-const ScoreSchema = z.object({
-  scoreId: z.string(),
-});
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get Score",
+    querySchema: GetScoreQuery,
+    responseSchema: GetScoreResponse,
+    fn: async ({ query, auth }) => {
+      const { scoreId } = query;
 
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
+      const score = await prisma.score.findUnique({
+        where: {
+          id: scoreId,
+          projectId: auth.scope.projectId,
+        },
+      });
 
-  if (req.method === "DELETE") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
+      if (!score) {
+        throw new LangfuseNotFoundError("Score not found");
       }
 
-      const { scoreId } = ScoreSchema.parse(req.query); // uses query and not body
+      const parsedScore = GetScoreResponse.safeParse(score);
+
+      if (!parsedScore.success) {
+        Sentry.captureException(parsedScore.error);
+        throw new InternalServerError("Requested score is corrupted");
+      }
+
+      return parsedScore.data;
+    },
+  }),
+  DELETE: createAuthedAPIRoute({
+    name: "Delete Score",
+    querySchema: DeleteScoreQuery,
+    responseSchema: DeleteScoreResponse,
+    fn: async ({ query, auth }) => {
+      const { scoreId } = query;
 
       const score = await prisma.score.findUnique({
         select: {
@@ -40,101 +52,24 @@ export default async function handler(
         },
         where: {
           id: scoreId,
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
       });
 
       if (!score) {
-        return res.status(404).json({
-          message: "Score not found within authorized project",
-        });
+        throw new LangfuseNotFoundError(
+          "Score not found within authorized project",
+        );
       }
 
       await prisma.score.delete({
         where: {
           id: scoreId,
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
       });
 
-      return res.status(200).json({ message: "Score deleted successfully" });
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else if (req.method === "GET") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-
-      const { scoreId } = ScoreSchema.parse(req.query);
-
-      // CHECK ACCESS SCOPE
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-      // END CHECK ACCESS SCOPE
-
-      const score = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-          projectId: authCheck.scope.projectId,
-        },
-      });
-
-      if (!score) {
-        return res.status(404).json({
-          message: "Score not found within authorized project",
-        });
-      }
-
-      return res.status(200).json(score);
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+      return { message: "Score deleted successfully" };
+    },
+  }),
+});

web/src/server/api/routers/generations/getAllQuery.ts

@@ -17,7 +17,7 @@ const getAllGenerationsInput = GenerationTableOptions.extend({
 
 export type ScoreSimplified = {
   name: string;
-  value: number;
+  value?: number | null;
   dataType: ScoreDataType;
   stringValue?: string | null;
   comment?: string | null;
@@ -61,6 +61,7 @@ export const getAllQuery = protectedProjectProcedure
                 scores."project_id" = ${input.projectId}
                 AND scores."trace_id" = t.id
                 AND scores."observation_id" = o.id
+                AND scores.value IS NOT NULL
             GROUP BY
                 name
         ) tmp

web/src/server/api/routers/scoreConfigs.ts

@@ -7,7 +7,11 @@ import { optionalPaginationZod } from "@langfuse/shared";
 
 import { ScoreDataType } from "@langfuse/shared/src/db";
 import { z } from "zod";
-import { categoriesList } from "@langfuse/shared";
+import {
+  filterAndValidateDbScoreConfigList,
+  Category,
+  validateDbScoreConfig,
+} from "@/src/features/public-api/types/score-configs";
 
 const ScoreConfigAllInput = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -27,32 +31,28 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:read",
       });
 
-      try {
-        const configs = await ctx.prisma.scoreConfig.findMany({
-          where: {
-            projectId: input.projectId,
-          },
-          ...(input.limit !== undefined && input.page !== undefined
-            ? { take: input.limit, skip: input.page * input.limit }
-            : undefined),
-          orderBy: {
-            createdAt: "desc",
-          },
-        });
+      const configs = await ctx.prisma.scoreConfig.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        ...(input.limit !== undefined && input.page !== undefined
+          ? { take: input.limit, skip: input.page * input.limit }
+          : undefined),
+        orderBy: {
+          createdAt: "desc",
+        },
+      });
 
-        const configsCount = await ctx.prisma.scoreConfig.count({
-          where: {
-            projectId: input.projectId,
-          },
-        });
+      const configsCount = await ctx.prisma.scoreConfig.count({
+        where: {
+          projectId: input.projectId,
+        },
+      });
 
-        return {
-          configs,
-          totalCount: configsCount,
-        };
-      } catch (error) {
-        console.log(error);
-      }
+      return {
+        configs: filterAndValidateDbScoreConfigList(configs),
+        totalCount: configsCount,
+      };
     }),
   create: protectedProjectProcedure
     .input(
@@ -62,7 +62,7 @@ export const scoreConfigsRouter = createTRPCRouter({
         dataType: z.nativeEnum(ScoreDataType),
         minValue: z.number().optional(),
         maxValue: z.number().optional(),
-        categories: categoriesList.optional(),
+        categories: z.array(Category).optional(),
         description: z.string().optional(),
       }),
     )
@@ -73,30 +73,13 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:CUD",
       });
 
-      try {
-        const existingConfig = await ctx.prisma.scoreConfig.findFirst({
-          where: {
-            projectId: input.projectId,
-            name: input.name,
-            dataType: input.dataType,
-          },
-        });
+      const config = await ctx.prisma.scoreConfig.create({
+        data: {
+          ...input,
+        },
+      });
 
-        if (existingConfig)
-          throw new Error(
-            "Score config with this name and data type already exists",
-          );
-
-        const config = await ctx.prisma.scoreConfig.create({
-          data: {
-            ...input,
-          },
-        });
-
-        return config;
-      } catch (error) {
-        console.log(error);
-      }
+      return validateDbScoreConfig(config);
     }),
   update: protectedProjectProcedure
     .input(
@@ -113,20 +96,16 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:CUD",
       });
 
-      try {
-        const config = await ctx.prisma.scoreConfig.update({
-          where: {
-            id: input.id,
-            projectId: input.projectId,
-          },
-          data: {
-            isArchived: input.isArchived,
-          },
-        });
+      const config = await ctx.prisma.scoreConfig.update({
+        where: {
+          id: input.id,
+          projectId: input.projectId,
+        },
+        data: {
+          isArchived: input.isArchived,
+        },
+      });
 
-        return config;
-      } catch (error) {
-        console.log(error);
-      }
+      return validateDbScoreConfig(config);
     }),
 });

web/src/server/api/routers/scores.ts

@@ -6,8 +6,12 @@ import {
 } from "@/src/server/api/trpc";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type ProjectRole, Prisma, type Score } from "@langfuse/shared/src/db";
-import { paginationZod } from "@langfuse/shared";
-import { ScoreDataType, singleFilter } from "@langfuse/shared";
+import {
+  CreateAnnotationScoreData,
+  UpdateAnnotationScoreData,
+  paginationZod,
+} from "@langfuse/shared";
+import { singleFilter } from "@langfuse/shared";
 import {
   tableColumnsToSqlFilterAndPrefix,
   orderByToPrismaSql,
@@ -18,6 +22,7 @@ import {
 } from "@/src/server/api/definitions/scoresTable";
 import { orderBy } from "@langfuse/shared";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { validateDbScore } from "@/src/features/public-api/types/scores";
 
 const ScoreFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -131,19 +136,7 @@ export const scoresRouter = createTRPCRouter({
       return res;
     }),
   createAnnotationScore: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        traceId: z.string(),
-        observationId: z.string().optional(),
-        name: z.string(),
-        value: z.number(),
-        stringValue: z.string().optional(),
-        comment: z.string().optional().nullable(),
-        configId: z.string().optional(),
-        dataType: z.nativeEnum(ScoreDataType),
-      }),
-    )
+    .input(CreateAnnotationScoreData)
     .mutation(async ({ input, ctx }) => {
       throwIfNoAccess({
         session: ctx.session,
@@ -161,76 +154,63 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No trace with this id in this project.");
       }
 
-      try {
-        const existingScore = await ctx.prisma.score.findFirst({
+      const existingScore = await ctx.prisma.score.findFirst({
+        where: {
+          projectId: input.projectId,
+          traceId: input.traceId,
+          observationId: input.observationId,
+          source: "ANNOTATION",
+          configId: input.configId,
+        },
+      });
+
+      if (existingScore) {
+        const updatedScore = await ctx.prisma.score.update({
           where: {
+            id: existingScore.id,
             projectId: input.projectId,
-            traceId: input.traceId,
-            observationId: input.observationId,
-            source: "ANNOTATION",
-            configId: input.configId,
           },
-        });
-
-        if (existingScore) {
-          return ctx.prisma.score.update({
-            where: {
-              id: existingScore.id,
-              projectId: input.projectId,
-            },
-            data: {
-              value: input.value,
-              stringValue: input.stringValue,
-              comment: input.comment,
-              authorUserId: ctx.session.user.id,
-            },
-          });
-        }
-
-        const score = await ctx.prisma.score.create({
           data: {
-            projectId: input.projectId,
-            traceId: input.traceId,
-            observationId: input.observationId,
             value: input.value,
             stringValue: input.stringValue,
-            dataType: input.dataType,
-            configId: input.configId,
-            name: input.name,
             comment: input.comment,
             authorUserId: ctx.session.user.id,
-            source: "ANNOTATION",
           },
         });
-        await auditLog({
-          projectId: input.projectId,
-          userId: ctx.session.user.id,
-          userProjectRole: ctx.session.user.projects.find(
-            (p) => p.id === input.projectId,
-          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-          resourceType: "score",
-          resourceId: score.id,
-          action: "create",
-          after: score,
-        });
-        return score;
-      } catch (error) {
-        console.log(error);
-        throw error;
+        return validateDbScore(updatedScore);
       }
+
+      const score = await ctx.prisma.score.create({
+        data: {
+          projectId: input.projectId,
+          traceId: input.traceId,
+          observationId: input.observationId,
+          value: input.value,
+          stringValue: input.stringValue,
+          dataType: input.dataType,
+          configId: input.configId,
+          name: input.name,
+          comment: input.comment,
+          authorUserId: ctx.session.user.id,
+          source: "ANNOTATION",
+        },
+      });
+
+      await auditLog({
+        projectId: input.projectId,
+        userId: ctx.session.user.id,
+        userProjectRole: ctx.session.user.projects.find(
+          (p) => p.id === input.projectId,
+        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+        resourceType: "score",
+        resourceId: score.id,
+        action: "create",
+        after: score,
+      });
+      return validateDbScore(score);
     }),
   updateAnnotationScore: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        id: z.string(),
-        value: z.number(),
-        stringValue: z.string().optional(),
-        comment: z.string().optional().nullable(),
-        configId: z.string().optional(),
-        dataType: z.nativeEnum(ScoreDataType),
-      }),
-    )
+    .input(UpdateAnnotationScoreData)
     .mutation(async ({ input, ctx }) => {
       throwIfNoAccess({
         session: ctx.session,
@@ -248,35 +228,31 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No annotation score with this id in this project.");
       }
 
-      try {
-        await auditLog({
-          projectId: input.projectId,
-          userId: ctx.session.user.id,
-          userProjectRole: ctx.session.user.projects.find(
-            (p) => p.id === input.projectId,
-          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-          resourceType: "score",
-          resourceId: score.id,
-          action: "update",
-          after: score,
-        });
+      await auditLog({
+        projectId: input.projectId,
+        userId: ctx.session.user.id,
+        userProjectRole: ctx.session.user.projects.find(
+          (p) => p.id === input.projectId,
+        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+        resourceType: "score",
+        resourceId: score.id,
+        action: "update",
+        after: score,
+      });
 
-        return ctx.prisma.score.update({
-          where: {
-            id: score.id,
-            projectId: input.projectId,
-          },
-          data: {
-            value: input.value,
-            stringValue: input.stringValue,
-            comment: input.comment,
-            authorUserId: ctx.session.user.id,
-          },
-        });
-      } catch (error) {
-        console.log(error);
-        throw error;
-      }
+      const updatedScore = await ctx.prisma.score.update({
+        where: {
+          id: score.id,
+          projectId: input.projectId,
+        },
+        data: {
+          value: input.value,
+          stringValue: input.stringValue,
+          comment: input.comment,
+          authorUserId: ctx.session.user.id,
+        },
+      });
+      return validateDbScore(updatedScore);
     }),
   deleteAnnotationScore: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), id: z.string() }))
@@ -310,7 +286,7 @@ export const scoresRouter = createTRPCRouter({
         before: score,
       });
 
-      return ctx.prisma.score.delete({
+      return await ctx.prisma.score.delete({
         where: {
           id: score.id,
           projectId: input.projectId,

web/src/server/api/routers/sessions.ts

@@ -9,6 +9,7 @@ import {
   type SessionOptions,
   getSessionTableSQL,
 } from "@langfuse/shared";
+import * as Sentry from "@sentry/node";
 import { Prisma } from "@langfuse/shared/src/db";
 import { paginationZod } from "@langfuse/shared";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
@@ -16,6 +17,7 @@ import { TRPCError } from "@trpc/server";
 import { orderBy } from "@langfuse/shared";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import type Decimal from "decimal.js";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 const SessionFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -136,6 +138,8 @@ export const sessionRouter = createTRPCRouter({
           },
         });
 
+        const validatedScores = filterAndValidateDbScoreList(scores);
+
         const totalCostQuery = Prisma.sql`
         SELECT
           SUM(COALESCE(o."calculated_total_cost", 0)) AS "totalCost"
@@ -155,7 +159,7 @@ export const sessionRouter = createTRPCRouter({
           ...session,
           traces: session.traces.map((t) => ({
             ...t,
-            scores: scores.filter((s) => s.traceId === t.id),
+            scores: validatedScores.filter((s) => s.traceId === t.id),
           })),
           totalCost: costData?.totalCost ?? 0,
           users: [

web/src/server/api/routers/traces.ts

@@ -25,6 +25,7 @@ import { orderByToPrismaSql } from "@langfuse/shared";
 import { instrumentAsync } from "@/src/utils/instrumentation";
 import type Decimal from "decimal.js";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 const TraceFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -107,7 +108,7 @@ export const traceRouter = createTRPCRouter({
         async () =>
           await ctx.prisma.$queryRaw<
             Array<
-              Omit<Trace, "input" | "output" | "metadata"> & {
+              Trace & {
                 promptTokens: number;
                 completionTokens: number;
                 totalTokens: number;
@@ -147,13 +148,17 @@ export const traceRouter = createTRPCRouter({
           },
         },
       });
+      const validatedScores = filterAndValidateDbScoreList(scores);
 
       const totalTraceCount = totalTraces[0]?.count;
       return {
-        traces: traces.map((trace) => ({
-          ...trace,
-          scores: scores.filter((s) => s.traceId === trace.id),
-        })),
+        traces: traces.map(
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          ({ input, output, metadata, ...trace }) => ({
+            ...trace,
+            scores: validatedScores.filter((s) => s.traceId === trace.id),
+          }),
+        ),
         totalCount: totalTraceCount ? Number(totalTraceCount) : undefined,
       };
     }),
@@ -298,6 +303,7 @@ export const traceRouter = createTRPCRouter({
           projectId: trace.projectId,
         },
       });
+      const validatedScores = filterAndValidateDbScoreList(scores);
 
       const obsStartTimes = observations
         .map((o) => o.startTime)
@@ -319,7 +325,7 @@ export const traceRouter = createTRPCRouter({
 
       return {
         ...trace,
-        scores,
+        scores: validatedScores,
         latency: latencyMs !== undefined ? latencyMs / 1000 : undefined,
         observations: observations as ObservationReturnType[],
       };

web/src/server/api/services/EventProcessor.ts

@@ -12,8 +12,10 @@ import {
   type legacyObservationUpdateEvent,
   type sdkLogEvent,
   type traceEvent,
+  LangfuseNotFoundError,
+  InvalidRequestError,
 } from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
+import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
 import { ResourceNotFoundError } from "@/src/utils/exceptions";
 import { mergeJson } from "@langfuse/shared";
 import {
@@ -30,6 +32,14 @@ import { sendToBetterstack } from "@/src/features/betterstack/server/betterstack
 import { ForbiddenError } from "@langfuse/shared";
 import { instrument } from "@/src/utils/instrumentation";
 import Decimal from "decimal.js";
+import {
+  ScoreBodyWithoutConfig,
+  ScorePropsAgainstConfig,
+} from "@/src/features/public-api/types/scores";
+import {
+  validateDbScoreConfigSafe,
+  type ValidatedScoreConfig,
+} from "@/src/features/public-api/types/score-configs";
 
 export interface EventProcessor {
   process(
@@ -606,6 +616,145 @@ export class ScoreProcessor implements EventProcessor {
     this.event = event;
   }
 
+  static inferDataType(value: string | number): ScoreDataType {
+    return typeof value === "number"
+      ? ScoreDataType.NUMERIC
+      : ScoreDataType.CATEGORICAL;
+  }
+
+  static mapStringValueToNumericValue(
+    config: ValidatedScoreConfig,
+    label: string,
+  ): number | null {
+    return (
+      config.categories?.find((category) => category.label === label)?.value ??
+      null
+    );
+  }
+
+  static inflateScoreBody(
+    body: any,
+    id: string,
+    projectId: string,
+    config?: ValidatedScoreConfig,
+  ): Score {
+    const relevantDataType = config?.dataType ?? body.dataType;
+    const scoreProps = { ...body, id, projectId, source: "API" };
+
+    if (typeof body.value === "number") {
+      if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {
+        return {
+          ...scoreProps,
+          value: body.value,
+          stringValue: body.value === 1 ? "True" : "False",
+          dataType: ScoreDataType.BOOLEAN,
+        };
+      }
+
+      return {
+        ...scoreProps,
+        value: body.value,
+        dataType: ScoreDataType.NUMERIC,
+      };
+    }
+    return {
+      ...scoreProps,
+      value: config
+        ? ScoreProcessor.mapStringValueToNumericValue(config, body.value)
+        : null,
+      stringValue: body.value,
+      dataType: ScoreDataType.CATEGORICAL,
+    };
+  }
+
+  validateConfigAgainstBody(body: any, config: ValidatedScoreConfig): void {
+    const { maxValue, minValue, categories, dataType: configDataType } = config;
+    if (body.dataType && body.dataType !== configDataType) {
+      throw new InvalidRequestError(
+        `Data type mismatch based on config: expected ${configDataType}, got ${body.dataType}`,
+      );
+    }
+
+    if (config.isArchived) {
+      throw new InvalidRequestError(
+        "Config is archived and cannot be used to create new scores. Please restore the config first.",
+      );
+    }
+
+    if (config.name !== body.name) {
+      throw new InvalidRequestError(
+        `Name mismatch based on config: expected ${config.name}, got ${body.name}`,
+      );
+    }
+
+    const relevantDataType = configDataType ?? body.dataType;
+
+    const dataTypeValidation = ScoreBodyWithoutConfig.safeParse({
+      ...body,
+      dataType: relevantDataType,
+    });
+    if (!dataTypeValidation.success) {
+      throw new InvalidRequestError(
+        `Ingested score body not valid against provided config data type.`,
+      );
+    }
+
+    const rangeValidation = ScorePropsAgainstConfig.safeParse({
+      value: body.value,
+      dataType: relevantDataType,
+      ...(maxValue !== null && maxValue !== undefined && { maxValue }),
+      ...(minValue !== null && minValue !== undefined && { minValue }),
+      ...(categories && { categories }),
+    });
+    if (!rangeValidation.success) {
+      const errorDetails = rangeValidation.error.errors
+        .map((error) => `${error.path.join(".")} - ${error.message}`)
+        .join(", ");
+      throw new InvalidRequestError(
+        `Ingested score body not valid against provided config: ${errorDetails}`,
+      );
+    }
+  }
+
+  async validateAndInflate(
+    body: any,
+    id: string,
+    projectId: string,
+  ): Promise<Score> {
+    if (body.configId) {
+      const config = await prisma.scoreConfig.findFirst({
+        where: {
+          projectId,
+          id: body.configId,
+        },
+      });
+
+      if (!config || !validateDbScoreConfigSafe(config).success)
+        throw new LangfuseNotFoundError(
+          "The configId you provided does not match a valid config in this project",
+        );
+
+      this.validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+      return ScoreProcessor.inflateScoreBody(
+        body,
+        id,
+        projectId,
+        config as ValidatedScoreConfig,
+      );
+    } else {
+      const validation = ScoreBodyWithoutConfig.safeParse({
+        ...body,
+        dataType: body.dataType ?? ScoreProcessor.inferDataType(body.value),
+      });
+      if (!validation.success) {
+        throw new InvalidRequestError(
+          `Ingested score value type not valid against provided data type. Provide numeric values for numeric and boolean scores, and string values for categorical scores.`,
+        );
+      }
+      return ScoreProcessor.inflateScoreBody(body, id, projectId);
+    }
+  }
+
   async process(
     apiScope: ApiAccessScope,
   ): Promise<Trace | Observation | Score> {
@@ -632,6 +781,12 @@ export class ScoreProcessor implements EventProcessor {
       );
     }
 
+    const validatedScore = await this.validateAndInflate(
+      body,
+      id,
+      apiScope.projectId,
+    );
+
     return await prisma.score.upsert({
       where: {
         id_projectId: {
@@ -640,24 +795,10 @@ export class ScoreProcessor implements EventProcessor {
         },
       },
       create: {
-        id,
-        projectId: apiScope.projectId,
-        traceId: body.traceId,
-        observationId: body.observationId ?? undefined,
-        timestamp: new Date(),
-        value: body.value,
-        name: body.name,
-        comment: body.comment,
-        source: "API",
+        ...validatedScore,
       },
       update: {
-        traceId: body.traceId,
-        observationId: body.observationId ?? undefined,
-        timestamp: new Date(),
-        value: body.value,
-        name: body.name,
-        comment: body.comment,
-        source: "API",
+        ...validatedScore,
       },
     });
   }

web/src/server/api/services/tableDefinitions.ts

@@ -126,6 +126,7 @@ const tracesObservationsColumns: ColumnDefinition[] = [
   observationsProjectId,
   duration,
   totalTokens,
+  calculatedTotalCost,
   model,
   traceTimestamp,
   traceUser,

web/src/utils/typeChecks.ts

@@ -0,0 +1,2 @@
+export const isPresent = <T>(value: T | null | undefined): value is T =>
+  value !== null && value !== undefined && value !== "";

worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "worker",
-  "version": "2.59.1",
+  "version": "2.60.0",
   "description": "",
   "license": "MIT",
   "private": true,

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.59.1";
+export const VERSION = "v2.60.0";