chore: release v2.60.2

push

.env.prod.example

@@ -134,6 +134,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Used to determine the Sentry sample rate
 # LANGFUSE_TRACING_SAMPLE_RATE=
 
+# NewRelic
+# NEW_RELIC_API_KEY= 
+# OTLP_ENDPOINT=
+
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
@@ -153,4 +157,5 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Admin API
 # ADMIN_API_KEY=
 
+
 ### END Langfuse Cloud Config

.gitignore

@@ -1,4 +1,5 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+**/newrelic_agent.log
 
 # dependencies
 /node_modules

README.md

@@ -159,7 +159,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

fern/apis/client/definition/score.yml

@@ -17,15 +17,131 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -120,25 +120,93 @@ types:
       updatedAt: datetime
       projectId: string
       dataType: ScoreDataType
-      isArchived: boolean
-      minValue: optional<double>
-      maxValue: optional<double>
-      categories: optional<list<ConfigCategory>>
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
       description: optional<string>
   ConfigCategory:
     properties:
       value: double
       label: string
-  Score:
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
   Dataset:
     properties:
       id: string
@@ -185,6 +253,39 @@ types:
     extends: DatasetRun
     properties:
       datasetRunItems: list<DatasetRunItem>
+  Model:
+    docs: Model definition used for transforming usage into USD cost and/or tokenization.
+    properties:
+      id: string
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>
+      isLangfuseManaged:
+        type: boolean
 
   # Utilities
   ModelUsageUnit:

fern/apis/server/definition/datasets.yml

@@ -22,7 +22,7 @@ service:
       response: PaginatedDatasets
     get:
       method: GET
-      docs: Get a dataset and its items
+      docs: Get a dataset
       path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
@@ -43,7 +43,7 @@ service:
       response: commons.DatasetRunWithItems
     getRuns:
       method: GET
-      docs: Get a dataset runs
+      docs: Get dataset runs
       path: /datasets/{datasetName}/runs
       path-parameters:
         datasetName: string

fern/apis/server/definition/ingestion.yml

@@ -185,9 +185,54 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/models.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      method: POST
+      docs: Create a model
+      path: /models
+      request: CreateModelRequest
+      response: commons.Model
+    list:
+      method: GET
+      docs: Get all models
+      path: /models
+      request:
+        name: GetModelsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedModels
+    get:
+      method: GET
+      docs: Get a model
+      path: /models/{id}
+      path-parameters:
+        id: string
+      response: commons.Model
+    delete:
+      method: DELETE
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      path: /models/{id}
+      path-parameters:
+        id: string
+
+types:
+  PaginatedModels:
+    properties:
+      data: list<commons.Model>
+      meta: pagination.MetaResponse
+  CreateModelRequest:
+    properties:
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: commons.ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total units. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>

fern/apis/server/definition/score-configs.yml

@@ -6,6 +6,12 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
     get:
       docs: Get all score configs
       method: GET
@@ -18,7 +24,7 @@ service:
             docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
       response: ScoreConfigs
     get-by-id:
       docs: Get a score config
@@ -34,3 +40,19 @@ types:
     properties:
       data: list<commons.ScoreConfig>
       meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -25,8 +25,12 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
@@ -42,6 +46,12 @@ service:
           scoreIds:
             type: optional<string>
             docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -66,9 +76,55 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+
   Scores:
     properties:
       data: list<commons.Score>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.58.0",
+  "version": "2.60.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -79,5 +79,8 @@
         "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
       }
     }
+  },
+  "dependencies": {
+    "newrelic": "^11.22.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -286,7 +286,7 @@ export type ObservationView = {
     trace_id: string | null;
     project_id: string;
     type: ObservationType;
-    start_time: Generated<Timestamp>;
+    start_time: Timestamp;
     end_time: Timestamp | null;
     name: string | null;
     metadata: unknown | null;
@@ -294,7 +294,8 @@ export type ObservationView = {
     level: Generated<ObservationLevel>;
     status_message: string | null;
     version: string | null;
-    created_at: Generated<Timestamp>;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     model: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
@@ -357,7 +358,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
     author_user_id: string | null;
     comment: string | null;
@@ -438,6 +439,8 @@ export type TraceView = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     duration: number | null;
 };
 export type User = {

packages/shared/prisma/migrations/20240703214747_models_anthropic_aws_bedrock/migration.sql

@@ -0,0 +1,9 @@
+-- Add AWS Bedrock model names for Anthropic models
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku-20240307|anthropic\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet-20240229|anthropic\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus-20240229|anthropic\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-5-sonnet-20240620|anthropic\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$' WHERE "id" = 'clxt0n0m60000pumz1j5b7zsf';

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240704103901_scores_make_value_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "value" DROP NOT NULL;

packages/shared/prisma/migrations/20240705152639_traces_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,20 @@
+-- Drop and create to be able to change columns, otherwise new t.* cols cannot be added
+
+DROP VIEW IF EXISTS traces_view;
+CREATE VIEW traces_view AS
+WITH observations_metrics AS (
+    SELECT
+        trace_id,
+        project_id,
+        EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+    FROM
+        observations o
+    GROUP BY
+        project_id, trace_id
+)
+SELECT
+    t.*,
+    o.duration
+FROM
+    traces t
+    LEFT JOIN observations_metrics o ON t.id = o.trace_id and t.project_id = o.project_id

packages/shared/prisma/migrations/20240705154048_observation_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,89 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240704103900_observations_view_read_from_calculated and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/schema.prisma

@@ -272,6 +272,8 @@ view TraceView {
     input      Json?
     output     Json?
     sessionId  String?  @map("session_id")
+    createdAt  DateTime @map("created_at")
+    updatedAt  DateTime @map("updated_at")
 
     // calculated fields
     duration Float? @map("duration") // can be null if no observations in trace
@@ -352,7 +354,7 @@ view ObservationView {
     traceId             String?          @map("trace_id")
     projectId           String           @map("project_id")
     type                ObservationType
-    startTime           DateTime         @default(now()) @map("start_time")
+    startTime           DateTime         @map("start_time")
     endTime             DateTime?        @map("end_time")
     name                String?
     metadata            Json?
@@ -360,7 +362,8 @@ view ObservationView {
     level               ObservationLevel @default(DEFAULT)
     statusMessage       String?          @map("status_message")
     version             String?
-    createdAt           DateTime         @default(now()) @map("created_at")
+    createdAt           DateTime         @map("created_at")
+    updateAt            DateTime         @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -410,14 +413,14 @@ model Score {
     projectId     String         @map("project_id")
     project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
-    value         Float
+    value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
     source        ScoreSource
     authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
-    stringValue   String?        @map("string_value")
+    stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")

packages/shared/prisma/seed.ts

@@ -17,6 +17,11 @@ import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
 const options = {
   environment: { type: "string" },
 } as const;
@@ -459,7 +464,15 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
-  configIdsAndNames: Map<string, { name: string; id: string }[]>
+  configParams: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -518,13 +531,34 @@ function createObjects(
 
     traces.push(trace);
 
-    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const configArray = configParams.get(projectId) ?? [];
     const randomIndex = Math.floor(Math.random() * 3);
     const config =
       configArray.length >= randomIndex - 1 && configArray[randomIndex];
-    const { name: annotationScoreName, id: configId } = config || {
+    const {
+      name: annotationScoreName,
+      id: configId,
+      dataType,
+      categories,
+    } = config || {
       name: "manual-score",
       id: undefined,
+      dataType: ScoreDataType.NUMERIC,
+      categories: null,
+    };
+
+    const value = Math.floor(Math.random() * 2);
+    const scoreNumericAndStringValue = {
+      ...(dataType === ScoreDataType.NUMERIC && { value }),
+      ...(dataType === ScoreDataType.CATEGORICAL && {
+        value,
+        stringValue: categories?.find((category) => category.value === value)
+          ?.label,
+      }),
+      ...(dataType === ScoreDataType.BOOLEAN && {
+        value,
+        stringValue: value === 1 ? "True" : "False",
+      }),
     };
 
     const traceScores = [
@@ -533,12 +567,12 @@ function createObjects(
             {
               traceId: trace.id,
               name: annotationScoreName,
-              value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
-              dataType: ScoreDataType.NUMERIC,
+              dataType,
+              ...scoreNumericAndStringValue,
               ...(configId ? { configId } : {}),
             },
           ]
@@ -556,6 +590,20 @@ function createObjects(
             },
           ]
         : []),
+      ...(Math.random() < 0.8
+        ? [
+            {
+              traceId: trace.id,
+              name: "Completeness",
+              timestamp: traceTs,
+              source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.CATEGORICAL,
+              stringValue:
+                Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+            },
+          ]
+        : []),
     ];
 
     scores.push(...traceScores);
@@ -976,8 +1024,15 @@ async function generatePrompts(project: Project) {
 }
 
 async function generateConfigsForProject(projects: Project[]) {
-  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
-    new Map();
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
 
   await Promise.all(
     projects.map(async (project) => {
@@ -989,7 +1044,12 @@ async function generateConfigsForProject(projects: Project[]) {
 }
 
 async function generateConfigs(project: Project) {
-  const configNameAndId: { name: string; id: string }[] = [];
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
 
   const configs = [
     {
@@ -1046,7 +1106,12 @@ async function generateConfigs(project: Project) {
         id: config.id,
       },
     });
-    configNameAndId.push({ name: config.name, id: config.id });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
   }
 
   return configNameAndId;

packages/shared/src/errors/InternalServerError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InternalServerError extends BaseError {
+  constructor(description = "Internal Server Error") {
+    super("InternalServerError", 500, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -5,3 +5,4 @@ export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
+export { InternalServerError } from "./InternalServerError";

packages/shared/src/features/annotation/types.ts

@@ -1,15 +1,59 @@
 import z from "zod";
-import { ScoreConfig } from "../../db";
+import { type ScoreDataType } from "../../db";
 
-const configCategory = z.object({
-  label: z.string().min(1),
+const NUMERIC: ScoreDataType = "NUMERIC";
+const CATEGORICAL: ScoreDataType = "CATEGORICAL";
+const BOOLEAN: ScoreDataType = "BOOLEAN";
+
+export const availableDataTypes = [NUMERIC, CATEGORICAL, BOOLEAN] as const;
+
+const NumericData = z.object({
   value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
 });
 
-export const categoriesList = z.array(configCategory);
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
 
-export type ConfigCategory = z.infer<typeof configCategory>;
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
 
-export type CastedConfig = Omit<ScoreConfig, "categories"> & {
-  categories: ConfigCategory[] | null;
-};
+const CreateAnnotationScoreBase = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  traceId: z.string(),
+  configId: z.string().optional(),
+  observationId: z.string().optional(),
+  comment: z.string().optional().nullish(),
+});
+
+const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
+  id: z.string(),
+});
+
+/**
+ * CreateAnnotationScoreData is only used for annotation scores created via the UI.
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const CreateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  CreateAnnotationScoreBase.merge(NumericData),
+  CreateAnnotationScoreBase.merge(CategoricalData),
+  CreateAnnotationScoreBase.merge(BooleanData),
+]);
+
+/**
+ * UpdateAnnotationScoreData is only used for annotation scores updated via the UI
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  UpdateAnnotationScoreBase.merge(NumericData),
+  UpdateAnnotationScoreBase.merge(CategoricalData),
+  UpdateAnnotationScoreBase.merge(BooleanData),
+]);

packages/shared/src/features/ingestion/types.ts

@@ -145,15 +145,75 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
   return false;
 });
 
-export const ScoreBody = z.object({
+const BaseScoreBody = z.object({
   id: z.string().nullish(),
   name: NonEmptyString,
-  value: z.number(),
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
 });
 
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      })
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
   id: z.string().nullish(),

web/Dockerfile

@@ -1,10 +1,10 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} alpine AS base
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
 RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
@@ -12,7 +12,7 @@ RUN corepack enable
 RUN corepack prepare pnpm@8.15.5 --activate
 
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS pruner
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS pruner
 
 WORKDIR /app
 
@@ -20,7 +20,7 @@ COPY . .
 RUN turbo prune --scope=web --docker
 
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS builder
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS builder
 
 WORKDIR /app
 

web/next.config.mjs

@@ -80,22 +80,22 @@ const nextConfig = {
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-          {
-            source: "/api/auth/session",
-            headers: [
-              {
-                key: "Access-Control-Allow-Origin",
-                value: "https://langfuse.com",
-              },
-              { key: "Access-Control-Allow-Credentials", value: "true" },
-              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-              {
-                key: "Access-Control-Allow-Headers",
-                value: "Content-Type, Authorization",
-              },
-            ],
-          },
-        ]
+            {
+              source: "/api/auth/session",
+              headers: [
+                {
+                  key: "Access-Control-Allow-Origin",
+                  value: "https://langfuse.com",
+                },
+                { key: "Access-Control-Allow-Credentials", value: "true" },
+                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+                {
+                  key: "Access-Control-Allow-Headers",
+                  value: "Content-Type, Authorization",
+                },
+              ],
+            },
+          ]
         : []),
     ];
   },

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.58.0",
+  "version": "2.60.2",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -36,6 +36,18 @@
     "@marsidev/react-turnstile": "^0.5.4",
     "@mui/x-tree-view": "^7.6.2",
     "@next-auth/prisma-adapter": "^1.0.7",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.48.0",
+    "@opentelemetry/exporter-jaeger": "^1.25.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.52.1",
+    "@opentelemetry/resource-detector-aws": "^1.5.2",
+    "@opentelemetry/resource-detector-container": "^0.3.11",
+    "@opentelemetry/resources": "^1.25.1",
+    "@opentelemetry/sdk-node": "^0.52.1",
+    "@opentelemetry/sdk-trace-node": "^1.25.1",
+    "@opentelemetry/semantic-conventions": "^1.25.1",
+    "@opentelemetry/winston-transport": "^0.5.0",
+    "@prisma/instrumentation": "^5.16.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-avatar": "^1.0.4",

web/public/generated/api-client/openapi.yml

@@ -59,21 +59,39 @@ components:
           type: string
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
         comment:
           type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
         - value
-    Score:
-      title: Score
+    NumericScore:
+      title: NumericScore
       type: object
       properties:
         id:
@@ -85,6 +103,9 @@ components:
         value:
           type: number
           format: double
+          description: The numeric value of the score
+        source:
+          $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
         timestamp:
@@ -92,12 +113,163 @@ components:
           format: date-time
         comment:
           type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
         - name
         - value
+        - source
         - timestamp
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal
+      required:
+        - id
+        - traceId
+        - name
+        - value
+        - stringValue
+        - source
+        - timestamp
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and stringValue must map to a config category
+      required:
+        - id
+        - traceId
+        - name
+        - stringValue
+        - source
+        - timestamp
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    ScoreSource:
+      title: ScoreSource
+      type: string
+      enum:
+        - ANNOTATION
+        - API
+        - EVAL
+    ScoreDataType:
+      title: ScoreDataType
+      type: string
+      enum:
+        - NUMERIC
+        - CATEGORICAL
+        - BOOLEAN
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: string
+        - type: number
+          format: double
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
   securitySchemes:
     BearerAuth:
       type: http

web/public/generated/api/openapi.yml

@@ -334,7 +334,7 @@ paths:
               $ref: '#/components/schemas/CreateDatasetRequest'
   /api/public/v2/datasets/{datasetName}:
     get:
-      description: Get a dataset and its items
+      description: Get a dataset
       operationId: datasets_get
       tags:
         - Datasets
@@ -429,7 +429,7 @@ paths:
       security: *ref_0
   /api/public/datasets/{datasetName}/runs:
     get:
-      description: Get a dataset runs
+      description: Get dataset runs
       operationId: datasets_getRuns
       tags:
         - Datasets
@@ -699,6 +699,193 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
+  /api/public/models:
+    post:
+      description: Create a model
+      operationId: models_create
+      tags:
+        - Models
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Model'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateModelRequest'
+    get:
+      description: Get all models
+      operationId: models_list
+      tags:
+        - Models
+      parameters:
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedModels'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/models/{id}:
+    get:
+      description: Get a model
+      operationId: models_get
+      tags:
+        - Models
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Model'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+    delete:
+      description: >-
+        Delete a model. Cannot delete models managed by Langfuse. You can create
+        your own definition with the same modelName to override the definition
+        though.
+      operationId: models_delete
+      tags:
+        - Models
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '204':
+          description: ''
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/observations/{observationId}:
     get:
       description: Get a observation
@@ -1063,6 +1250,53 @@ paths:
             schema:
               $ref: '#/components/schemas/CreatePromptRequest'
   /api/public/score-configs:
+    post:
+      description: >-
+        Create a score configuration (config). Score configs are used to define
+        the structure of scores
+      operationId: scoreConfigs_create
+      tags:
+        - ScoreConfigs
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreConfig'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateScoreConfigRequest'
     get:
       description: Get all score configs
       operationId: scoreConfigs_get
@@ -1080,7 +1314,7 @@ paths:
           in: query
           description: >-
             Limit of items per page. If you encounter api issues due to too
-            large page sizes, try to reduce the limit.
+            large page sizes, try to reduce the limit
           required: false
           schema:
             type: integer
@@ -1234,12 +1468,14 @@ paths:
             nullable: true
         - name: userId
           in: query
+          description: Retrieve only scores with this userId associated to the trace.
           required: false
           schema:
             type: string
             nullable: true
         - name: name
           in: query
+          description: Retrieve only scores with this name.
           required: false
           schema:
             type: string
@@ -1281,6 +1517,20 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: configId
+          in: query
+          description: Retrieve only scores with a specific configId.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: dataType
+          in: query
+          description: Retrieve only scores with a specific dataType.
+          required: false
+          schema:
+            $ref: '#/components/schemas/ScoreDataType'
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1882,19 +2132,27 @@ components:
           $ref: '#/components/schemas/ScoreDataType'
         isArchived:
           type: boolean
+          description: Whether the score config is archived. Defaults to false
         minValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets minimum value for numerical scores. If not set, the minimum
+            value defaults to -∞
         maxValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets maximum value for numerical scores. If not set, the maximum
+            value defaults to +∞
         categories:
           type: array
           items:
             $ref: '#/components/schemas/ConfigCategory'
           nullable: true
+          description: Configures custom categories for categorical scores
         description:
           type: string
           nullable: true
@@ -1918,8 +2176,8 @@ components:
       required:
         - value
         - label
-    Score:
-      title: Score
+    NumericScore:
+      title: NumericScore
       type: object
       properties:
         id:
@@ -1931,6 +2189,7 @@ components:
         value:
           type: number
           format: double
+          description: The numeric value of the score
         source:
           $ref: '#/components/schemas/ScoreSource'
         observationId:
@@ -1942,6 +2201,13 @@ components:
         comment:
           type: string
           nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
@@ -1949,6 +2215,143 @@ components:
         - value
         - source
         - timestamp
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+          nullable: true
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+          nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal
+      required:
+        - id
+        - traceId
+        - name
+        - value
+        - stringValue
+        - source
+        - timestamp
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        id:
+          type: string
+        traceId:
+          type: string
+        name:
+          type: string
+        value:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+        source:
+          $ref: '#/components/schemas/ScoreSource'
+        observationId:
+          type: string
+          nullable: true
+        timestamp:
+          type: string
+          format: date-time
+        comment:
+          type: string
+          nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and stringValue must map to a config category
+      required:
+        - id
+        - traceId
+        - name
+        - stringValue
+        - source
+        - timestamp
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: string
+        - type: number
+          format: double
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
     Dataset:
       title: Dataset
       type: object
@@ -2086,6 +2489,71 @@ components:
         - datasetRunItems
       allOf:
         - $ref: '#/components/schemas/DatasetRun'
+    Model:
+      title: Model
+      type: object
+      description: >-
+        Model definition used for transforming usage into USD cost and/or
+        tokenization.
+      properties:
+        id:
+          type: string
+        modelName:
+          type: string
+          description: >-
+            Name of the model definition. If multiple with the same name exist,
+            they are applied in the following order: (1) custom over built-in,
+            (2) newest according to startTime where
+            model.startTime<observation.startTime
+        matchPattern:
+          type: string
+          description: >-
+            Regex pattern which matches this model definition to
+            generation.model. Useful in case of fine-tuned models. If you want
+            to exact match, use `(?i)^modelname$`
+        startDate:
+          type: string
+          nullable: true
+          description: Apply only to generations which are newer than this ISO date.
+        unit:
+          $ref: '#/components/schemas/ModelUsageUnit'
+          description: Unit used by this model.
+        inputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per input unit
+        outputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per output unit
+        totalPrice:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Price (USD) per total unit. Cannot be set if input or output price
+            is set.
+        tokenizerId:
+          type: string
+          nullable: true
+          description: >-
+            Optional. Tokenizer to be applied to observations which match to
+            this model. See docs for more details.
+        tokenizerConfig:
+          nullable: true
+          description: >-
+            Optional. Configuration for the selected tokenizer. Needs to be
+            JSON. See docs for more details.
+        isLangfuseManaged:
+          type: boolean
+      required:
+        - id
+        - modelName
+        - matchPattern
+        - unit
+        - isLangfuseManaged
     ModelUsageUnit:
       title: ModelUsageUnit
       type: string
@@ -2639,17 +3107,37 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
@@ -2888,6 +3376,75 @@ components:
         - countTraces
         - countObservations
         - totalCost
+    PaginatedModels:
+      title: PaginatedModels
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/Model'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    CreateModelRequest:
+      title: CreateModelRequest
+      type: object
+      properties:
+        modelName:
+          type: string
+          description: >-
+            Name of the model definition. If multiple with the same name exist,
+            they are applied in the following order: (1) custom over built-in,
+            (2) newest according to startTime where
+            model.startTime<observation.startTime
+        matchPattern:
+          type: string
+          description: >-
+            Regex pattern which matches this model definition to
+            generation.model. Useful in case of fine-tuned models. If you want
+            to exact match, use `(?i)^modelname$`
+        startDate:
+          type: string
+          nullable: true
+          description: Apply only to generations which are newer than this ISO date.
+        unit:
+          $ref: '#/components/schemas/ModelUsageUnit'
+          description: Unit used by this model.
+        inputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per input unit
+        outputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per output unit
+        totalPrice:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Price (USD) per total units. Cannot be set if input or output price
+            is set.
+        tokenizerId:
+          type: string
+          nullable: true
+          description: >-
+            Optional. Tokenizer to be applied to observations which match to
+            this model. See docs for more details.
+        tokenizerConfig:
+          nullable: true
+          description: >-
+            Optional. Configuration for the selected tokenizer. Needs to be
+            JSON. See docs for more details.
+      required:
+        - modelName
+        - matchPattern
+        - unit
     Observations:
       title: Observations
       type: object
@@ -3146,6 +3703,47 @@ components:
       required:
         - data
         - meta
+    CreateScoreConfigRequest:
+      title: CreateScoreConfigRequest
+      type: object
+      properties:
+        name:
+          type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+        categories:
+          type: array
+          items:
+            $ref: '#/components/schemas/ConfigCategory'
+          nullable: true
+          description: >-
+            Configure custom categories for categorical scores. Pass a list of
+            objects with `label` and `value` properties. Categories are
+            autogenerated for boolean configs and cannot be passed
+        minValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a minimum value for numerical scores. If not set, the
+            minimum value defaults to -∞
+        maxValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a maximum value for numerical scores. If not set, the
+            maximum value defaults to +∞
+        description:
+          type: string
+          nullable: true
+          description: >-
+            Description is shown across the Langfuse UI and can be used to e.g.
+            explain the config categories in detail, why a numeric range was
+            set, or provide additional context on config name or usage
+      required:
+        - name
+        - dataType
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object
@@ -3155,17 +3753,36 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            The data type of the score. When passing a configId this field is
+            inferred. Otherwise, this field must be passed or will default to
+            numeric.
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. The unique langfuse identifier
+            of a score config. When passing this field, the dataType and
+            stringValue fields are automatically populated.
       required:
         - traceId
         - name

web/public/generated/postman/collection.json

@@ -246,7 +246,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a dataset and its items",
+            "description": "Get a dataset",
             "url": {
               "raw": "{{baseUrl}}/api/public/v2/datasets/:datasetName",
               "host": [
@@ -352,7 +352,7 @@
           "_type": "endpoint",
           "name": "Get Runs",
           "request": {
-            "description": "Get a dataset runs",
+            "description": "Get dataset runs",
             "url": {
               "raw": "{{baseUrl}}/api/public/datasets/:datasetName/runs?page=&limit=",
               "host": [
@@ -535,6 +535,146 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Models",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"modelName\": \"example\",\n    \"matchPattern\": \"example\",\n    \"startDate\": \"1994-11-05\",\n    \"unit\": \"CHARACTERS\",\n    \"inputPrice\": 0,\n    \"outputPrice\": 0,\n    \"totalPrice\": 0,\n    \"tokenizerId\": \"example\",\n    \"tokenizerConfig\": \"UNKNOWN\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all models",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -814,6 +954,39 @@
       "description": null,
       "name": "Score Configs",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a score configuration (config). Score configs are used to define the structure of scores",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"dataType\": \"NUMERIC\",\n    \"categories\": [\n        {\n            \"value\": 0,\n            \"label\": \"example\"\n        }\n    ],\n    \"minValue\": 0,\n    \"maxValue\": 0,\n    \"description\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -838,7 +1011,7 @@
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit"
                 }
               ],
               "variable": []
@@ -912,7 +1085,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"value\": 0,\n    \"observationId\": \"example\",\n    \"comment\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"novelty\",\n    \"value\": 0.9,\n    \"traceId\": \"cdef-1234-5678-90ab\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -928,7 +1101,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=&configId=&dataType=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -951,12 +1124,12 @@
                 {
                   "key": "userId",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this userId associated to the trace."
                 },
                 {
                   "key": "name",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this name."
                 },
                 {
                   "key": "fromTimestamp",
@@ -982,6 +1155,16 @@
                   "key": "scoreIds",
                   "value": "",
                   "description": "Comma-separated list of score IDs to limit the results to."
+                },
+                {
+                  "key": "configId",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific configId."
+                },
+                {
+                  "key": "dataType",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific dataType."
                 }
               ],
               "variable": []

web/scripts/observations-backfill-calculated-cost.ts

@@ -77,15 +77,28 @@ const backfillCalculatedGenerationCost = async () => {
         ),
         updated_batch AS (
             UPDATE observations o
-            SET calculated_input_cost = COALESCE(batch.input_cost, batch.prompt_tokens::numeric * batch.input_price),
-                calculated_output_cost = COALESCE(batch.output_cost, batch.completion_tokens::numeric * batch.output_price),
-                calculated_total_cost = COALESCE(
-                    batch.total_cost,
+            SET calculated_input_cost = 
                     CASE
-                        WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
-                        ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
-                    END
-                ),
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.prompt_tokens::numeric * batch.input_price
+                        ELSE batch.input_cost
+                    END,
+                calculated_output_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.completion_tokens::numeric * batch.output_price
+                        ELSE batch.output_cost
+                    END,
+                calculated_total_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN
+                            CASE
+                                WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
+                                ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
+                            END
+                        ELSE batch.total_cost
+                    END,
                 internal_model_id = batch.model_id,
                 tmp_has_calculated_cost = TRUE
             FROM batch

web/src/__tests__/model-definitions.servertest.ts

@@ -0,0 +1,221 @@
+/** @jest-environment node */
+
+import { prisma } from "@langfuse/shared/src/db";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  DeleteModelV1Response,
+  GetModelV1Response,
+  GetModelsV1Response,
+  PostModelsV1Response,
+} from "@/src/features/public-api/types/models";
+
+describe("/models API Endpoints", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+    // create some default models that do not belong to a project
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0020",
+        outputPrice: "0.0040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("GET /models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(2);
+    expect(models.body.data[0]).toMatchObject({
+      isLangfuseManaged: true,
+      modelName: "gpt-3.5-turbo",
+    });
+  });
+
+  it("GET /models pagination", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models?page=2&limit=1",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(1);
+    expect(models.body.meta).toMatchObject({
+      page: 2,
+      totalPages: 2,
+      limit: 1,
+      totalItems: 2,
+    });
+  });
+
+  it("Create and get custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+    expect(customModel.body.isLangfuseManaged).toBe(false);
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    const getModel = await makeZodVerifiedAPICall(
+      GetModelV1Response,
+      "GET",
+      `/api/public/models/${customModel.body.id}`,
+    );
+    expect(getModel.body.id).toBe(customModel.body.id);
+    expect(getModel.body).toMatchObject({
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      startDate: new Date("2023-12-01").toISOString(),
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      isLangfuseManaged: false,
+    });
+  });
+
+  it("Post model with invalid matchPattern", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Post model without prices or tokenizer", async () => {
+    await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        unit: "TOKENS",
+      },
+    );
+  });
+
+  it("Post model with missing fields", async () => {
+    const { status } = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      // missing unit
+    });
+    expect(status).toBe(400);
+  });
+
+  it("Post model with invalid price (input and total cost)", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      totalPrice: 0.1,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Cannot delete built-in models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(2);
+
+    const deleteModel = await makeAPICall(
+      "DELETE",
+      `/api/public/models/${models.body.data[0].id}`,
+    );
+    expect(deleteModel.status).toBe(404);
+  });
+
+  it("Delete custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    await makeZodVerifiedAPICall(
+      DeleteModelV1Response,
+      "DELETE",
+      `/api/public/models/${customModel.body.id}`,
+    );
+
+    const modelsAfterDelete = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(modelsAfterDelete.body.data.length).toBe(2);
+  });
+});

web/src/__tests__/score-configs.servertest.ts

@@ -1,24 +1,30 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { v4 as uuidv4 } from "uuid";
-import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
-import { type CastedConfig } from "@langfuse/shared";
-
-const CONFIG_ID_ONE = uuidv4();
-const CONFIG_ID_TWO = uuidv4();
-const CONFIG_ID_THREE = uuidv4();
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  type ScoreConfig,
+  prisma,
+  type ScoreDataType,
+} from "@langfuse/shared/src/db";
+import {
+  GetScoreConfigResponse,
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+} from "@/src/features/public-api/types/score-configs";
 
 const configOne = [
   {
-    id: CONFIG_ID_ONE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Boolean Config",
     description: "Test Description",
-    dataType: ScoreDataType.BOOLEAN,
+    dataType: "BOOLEAN" as ScoreDataType,
     categories: [
-      { label: "False", value: 0 },
       { label: "True", value: 1 },
+      { label: "False", value: 0 },
     ],
     createdAt: new Date("2024-05-10T00:00:00.000Z"),
     updatedAt: new Date("2024-05-10T00:00:00.000Z"),
@@ -26,11 +32,10 @@ const configOne = [
 ];
 const configTwo = [
   {
-    id: CONFIG_ID_TWO,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Numeric Config",
     description: "Test Description",
-    dataType: ScoreDataType.NUMERIC,
+    dataType: "NUMERIC" as ScoreDataType,
     minValue: 0,
     createdAt: new Date("2024-05-11T00:00:00.000Z"),
     updatedAt: new Date("2024-05-11T00:00:00.000Z"),
@@ -39,11 +44,10 @@ const configTwo = [
 
 const configThree = [
   {
-    id: CONFIG_ID_THREE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Categorical Config",
     description: "Test Description",
-    dataType: ScoreDataType.CATEGORICAL,
+    dataType: "CATEGORICAL" as ScoreDataType,
     categories: [
       { label: "A", value: 0 },
       { label: "B", value: 1 },
@@ -69,11 +73,18 @@ describe("/api/public/score-configs API Endpoint", () => {
   );
 
   it("should GET a score config", async () => {
-    const configId = CONFIG_ID_ONE;
+    const { id: configId } = (await prisma.scoreConfig.findFirst({
+      where: {
+        projectId: configOne[0].projectId,
+        name: configOne[0].name,
+      },
+    })) as ScoreConfig;
 
-    const getScoreConfig = await makeAPICall<{
-      id: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(200);
     expect(getScoreConfig.body).toMatchObject({
@@ -84,24 +95,12 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
-  it("test invalid config id input", async () => {
-    const configId = "invalid-config-id";
-
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
-
-    expect(getScoreConfig.status).toBe(404);
-    expect(getScoreConfig.body).toMatchObject({
-      message: "Score config not found within authorized project",
-    });
-  });
-
   it("should GET all score configs", async () => {
-    const fetchedConfigs = await makeAPICall<{
-      data: CastedConfig[];
-      meta: object;
-    }>("GET", `/api/public/score-configs?limit=50&page=1`);
+    const fetchedConfigs = await makeZodVerifiedAPICall(
+      GetScoreConfigsResponse,
+      "GET",
+      `/api/public/score-configs?limit=50&page=1`,
+    );
 
     expect(fetchedConfigs.status).toBe(200);
     expect(fetchedConfigs.body.meta).toMatchObject({
@@ -120,6 +119,20 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
+  it("test invalid config id input", async () => {
+    const configId = "invalid-config-id";
+
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
+
+    expect(getScoreConfig.status).toBe(404);
+    expect(getScoreConfig.body).toMatchObject({
+      message: "Score config not found within authorized project",
+    });
+  });
+
   it("should return 500 when hitting corrupted score config", async () => {
     const configId = "corrupted-config-id";
 
@@ -131,13 +144,234 @@ describe("/api/public/score-configs API Endpoint", () => {
       },
     });
 
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(500);
     expect(getScoreConfig.body).toMatchObject({
-      message: "Internal Server Error",
+      message: "Requested score config is corrupted",
+    });
+  });
+
+  it("should POST a numeric score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "numeric-config-name",
+        dataType: "NUMERIC",
+        maxValue: 0,
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("numeric-config-name");
+    expect(scoreConfig.body.dataType).toBe("NUMERIC");
+    expect(scoreConfig.body.maxValue).toBe(0);
+  });
+
+  it("should POST a boolean score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "boolean-config-name",
+        dataType: "BOOLEAN",
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("boolean-config-name");
+    expect(scoreConfig.body.dataType).toBe("BOOLEAN");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "True", value: 1 },
+      { label: "False", value: 0 },
+    ]);
+  });
+
+  it("should POST a categorical score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("categorical-config-name");
+    expect(scoreConfig.body.dataType).toBe("CATEGORICAL");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "Good", value: 1 },
+      { label: "Bad", value: 0 },
+    ]);
+  });
+
+  it("should fail POST of numeric score config with invalid range", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        PostScoreConfigResponse,
+        "POST",
+        "/api/public/score-configs",
+        {
+          name: "invalid-numeric-config-name",
+          dataType: "NUMERIC",
+          maxValue: 0,
+          minValue: 1,
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"custom\",\"message\":\"Maximum value must be greater than Minimum value\",\"path\":[]}]}`,
+      );
+    }
+  });
+
+  it("should fail POST of boolean score config with custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-boolean-config-name",
+        dataType: "BOOLEAN",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with NO custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with invalid custom categories format", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { key: "first", value: 1 },
+          { key: "second", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Category must be an array of objects with label value pairs, where labels and values are unique.",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category label", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "first", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category label: first, category labels must be unique",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category value", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "second", value: 1 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category value: 1, category values must be unique",
+          path: ["categories"],
+        },
+      ],
     });
   });
 });

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -19,11 +19,11 @@ describe("Traces TRPC Router", () => {
           id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           role: "ADMIN",
           name: "test",
+          cloudConfig: { defaultLookBackDays: null },
         },
       ],
       featureFlags: {
         templateFlag: true,
-        evals: true,
       },
       admin: true,
     },
@@ -36,7 +36,6 @@ describe("Traces TRPC Router", () => {
     const trace = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };
@@ -57,11 +56,42 @@ describe("Traces TRPC Router", () => {
     expect(traces).toMatchObject({ traces: [trace] });
   });
 
+  test("traces.all RPC must not return input, output, metadata", async () => {
+    const trace = {
+      name: "trace-name",
+      userId: "user-1",
+      input: { a: 1 },
+      output: { b: 2 },
+      metadata: { c: 3 },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    await prisma.trace.create({
+      data: { ...trace, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const traces = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: null,
+    });
+    expect(traces.traces).toBeDefined();
+    expect(traces.traces).toHaveLength(1);
+
+    const returnedTrace = traces.traces[0];
+    expect(returnedTrace).not.toHaveProperty("input");
+    expect(returnedTrace).not.toHaveProperty("output");
+    expect(returnedTrace).not.toHaveProperty("metadata");
+  });
+
   test("traces.all RPC orders traces by userId", async () => {
     const traceTmpl = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };

web/src/__tests__/test-utils.ts

@@ -82,10 +82,22 @@ export async function makeZodVerifiedAPICall<T extends z.ZodTypeAny>(
   auth?: string,
 ): Promise<{ body: z.infer<T>; status: number }> {
   const { body: resBody, status } = await makeAPICall(method, url, body, auth);
-  if (responseZodSchema instanceof ZodObject) {
-    responseZodSchema.strict().parse(resBody);
-  } else {
-    responseZodSchema.parse(resBody);
+  if (status !== 200) {
+    throw new Error(
+      `API call did not return 200, returned status ${status}, body ${JSON.stringify(resBody)}`,
+    );
+  }
+  try {
+    if (responseZodSchema instanceof ZodObject) {
+      responseZodSchema.strict().parse(resBody);
+    } else {
+      responseZodSchema.parse(resBody);
+    }
+  } catch (e) {
+    console.error(e);
+    throw new Error(
+      `API call (${method} ${url}) did not return valid response, returned status ${status}, body ${JSON.stringify(resBody)}, error ${e}`,
+    );
   }
   return { body: resBody, status };
 }

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
         <span key={i} className="group/score ml-1 first:ml-0">
           {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
             ? s.stringValue
-            : s.value.toFixed(2)}
+            : s.value?.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">

web/src/components/table/use-cases/generations.tsx

@@ -25,14 +25,13 @@ import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import {
-  type Prisma,
   type ObservationLevel,
   type FilterState,
   type ObservationOptions,
 } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type BatchExportFileFormat,
@@ -58,14 +57,14 @@ export type GenerationsTableRow = {
   timeToFirstToken?: number;
   name?: string;
   model?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
+  metadata?: unknown;
   inputCost?: Decimal;
   outputCost?: Decimal;
   totalCost?: Decimal;
   traceName?: string;
-  metadata?: Prisma.JsonValue;
   scores?: ScoreSimplified[];
   usage: {
     promptTokens: number;
@@ -454,7 +453,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
     },
     {
@@ -470,7 +469,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
     },
     {
@@ -486,7 +485,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
     },
     {
@@ -519,10 +518,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -538,10 +537,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -553,12 +552,16 @@ export default function GenerationsTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values = row.getValue(
-          "metadata",
-        ) as GenerationsTableRow["metadata"];
-        return !!values ? (
-          <IOTableCell data={values} singleLine={rowHeight === "s"} />
-        ) : null;
+        const observationId: string = row.getValue("id");
+        const traceId: string = row.getValue("traceId");
+        return (
+          <GenerationsDynamicCell
+            observationId={observationId}
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
       defaultHidden: true,
@@ -617,7 +620,6 @@ export default function GenerationsTable({
           model: generation.model ?? "",
           scores: generation.scores,
           level: generation.level,
-          metadata: generation.metadata,
           statusMessage: generation.statusMessage ?? undefined,
           usage: {
             promptTokens: generation.promptTokens,
@@ -712,15 +714,15 @@ export default function GenerationsTable({
   );
 }
 
-const GenerationsIOCell = ({
+const GenerationsDynamicCell = ({
   traceId,
   observationId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
   observationId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine: boolean;
 }) => {
   const observation = api.observations.byId.useQuery(
@@ -742,9 +744,13 @@ const GenerationsIOCell = ({
     <IOTableCell
       isLoading={observation.isLoading}
       data={
-        io === "output" ? observation.data?.output : observation.data?.input
+        col === "output"
+          ? observation.data?.output
+          : col === "input"
+            ? observation.data?.input
+            : observation.data?.metadata
       }
-      className={cn(io === "output" && "bg-accent-light-green")}
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/table/use-cases/score-configs.tsx

@@ -6,11 +6,7 @@ import { api } from "@/src/utils/api";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import { DataTable } from "@/src/components/table/data-table";
-import {
-  type ScoreDataType,
-  type Prisma,
-  type ConfigCategory,
-} from "@langfuse/shared";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 import {
@@ -28,6 +24,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { type ConfigCategory } from "@/src/features/public-api/types/score-configs";
 
 type ScoreConfigTableRow = {
   id: string;
@@ -38,7 +35,7 @@ type ScoreConfigTableRow = {
   range: {
     maxValue?: number | null;
     minValue?: number | null;
-    categories?: Prisma.JsonValue | null;
+    categories?: ConfigCategory[] | null;
   };
   description?: string | null;
   isArchived: boolean;
@@ -46,7 +43,7 @@ type ScoreConfigTableRow = {
 
 function getConfigRange(
   originalRow: ScoreConfigTableRow,
-): Prisma.JsonValue | undefined {
+): undefined | Prisma.JsonValue {
   const { range, dataType } = originalRow;
 
   if (isNumericDataType(dataType)) {
@@ -57,7 +54,7 @@ function getConfigRange(
   }
 
   if (isCategoricalDataType(dataType) || isBooleanDataType(dataType)) {
-    const configCategories = (range.categories as ConfigCategory[]) ?? [];
+    const configCategories = range.categories ?? [];
 
     return configCategories.reduce(
       (acc, category) => {

web/src/components/table/use-cases/scores.tsx

@@ -16,6 +16,7 @@ import {
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
 import { utcDateOffsetByDays } from "@/src/utils/dates";
+import { isPresent } from "@/src/utils/typeChecks";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
@@ -116,9 +117,18 @@ export default function ScoresTable({
   });
   const totalCount = scores.data?.totalCount ?? 0;
 
-  const filterOptions = api.scores.filterOptions.useQuery({
-    projectId,
-  });
+  const filterOptions = api.scores.filterOptions.useQuery(
+    {
+      projectId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
 
   const rawColumns: LangfuseColumnDef<ScoresTableRow>[] = [
     {
@@ -310,11 +320,12 @@ export default function ScoresTable({
       source: score.source,
       name: score.name,
       dataType: score.dataType,
-      value: isNumericDataType(score.dataType)
-        ? score.value % 1 === 0
-          ? String(score.value)
-          : score.value.toFixed(4)
-        : score.stringValue ?? "",
+      value:
+        isNumericDataType(score.dataType) && isPresent(score.value)
+          ? score.value % 1 === 0
+            ? String(score.value)
+            : score.value.toFixed(4)
+          : score.stringValue ?? "",
       author: {
         image: score.authorUserImage ?? undefined,
         name: score.authorUserName ?? undefined,

web/src/components/table/use-cases/sessions.tsx

@@ -15,7 +15,7 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { api } from "@/src/utils/api";
 import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
 import type Decimal from "decimal.js";
 import { useEffect } from "react";
@@ -276,7 +276,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value: number | undefined = row.getValue("inputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -289,7 +291,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value = row.getValue("outputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -301,7 +305,9 @@ export default function SessionsTable({
       enableSorting: true,
       cell: ({ row }) => {
         const value = row.getValue("totalTokens");
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {

web/src/components/table/use-cases/traces.tsx

@@ -23,7 +23,7 @@ import {
   withDefault,
 } from "use-query-params";
 import type Decimal from "decimal.js";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { DeleteButton } from "@/src/components/deleteButton";
 import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
@@ -34,11 +34,11 @@ import {
   type TraceOptions,
   tracesTableColsWithOptions,
   type ObservationLevel,
-  type Score,
 } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -46,17 +46,17 @@ export type TracesTableRow = {
   timestamp: string;
   name: string;
   userId: string;
-  metadata?: string;
   level: ObservationLevel;
   observationCount: number;
   latency?: number;
   release?: string;
   version?: string;
   sessionId?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
-  scores: Score[];
+  metadata?: unknown;
+  scores: ValidatedScore[];
   tags: string[];
   usage: {
     promptTokens: number;
@@ -128,7 +128,6 @@ export default function TracesTable({
     filter: filterState,
     searchQuery,
     orderBy: orderByState,
-    returnIO: false,
   };
   const traces = api.traces.all.useQuery(tracesAllQueryFilter);
 
@@ -180,7 +179,6 @@ export default function TracesTable({
       name: trace.name ?? "",
       level: trace.level,
       observationCount: trace.observationCount,
-      metadata: JSON.stringify(trace.metadata),
       release: trace.release ?? undefined,
       version: trace.version ?? undefined,
       userId: trace.userId ?? "",
@@ -339,7 +337,7 @@ export default function TracesTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -355,7 +353,7 @@ export default function TracesTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -371,7 +369,7 @@ export default function TracesTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -465,7 +463,7 @@ export default function TracesTable({
       header: "Scores",
       enableColumnFilter: !omittedFilter.find((f) => f === "scores"),
       cell: ({ row }) => {
-        const values: Score[] = row.getValue("scores");
+        const values: ValidatedScore[] = row.getValue("scores");
         return <GroupedScoreBadges scores={values} variant="headings" />;
       },
       enableHiding: true,
@@ -477,9 +475,9 @@ export default function TracesTable({
       cell: ({ row }) => {
         const traceId: string = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -494,9 +492,9 @@ export default function TracesTable({
       cell: ({ row }) => {
         const traceId: string = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -508,10 +506,17 @@ export default function TracesTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values: string = row.getValue("metadata");
-        return <IOTableCell data={values} singleLine={rowHeight === "s"} />;
+        const traceId: string = row.getValue("id");
+        return (
+          <TracesDynamicCell
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "level",
@@ -668,13 +673,13 @@ export default function TracesTable({
   );
 }
 
-const TracesIOCell = ({
+const TracesDynamicCell = ({
   traceId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine?: boolean;
 }) => {
   const trace = api.traces.byId.useQuery(
@@ -692,8 +697,14 @@ const TracesIOCell = ({
   return (
     <IOTableCell
       isLoading={trace.isLoading}
-      data={io === "output" ? trace.data?.output : trace.data?.input}
-      className={cn(io === "output" && "bg-accent-light-green")}
+      data={
+        col === "output"
+          ? trace.data?.output
+          : col === "input"
+            ? trace.data?.input
+            : trace.data?.metadata
+      }
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/token-usage-badge.tsx

@@ -1,5 +1,6 @@
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+import { numberFormatter } from "@/src/utils/numbers";
 import { type Observation } from "@langfuse/shared";
 
 export const TraceAggUsageBadge = (props: {
@@ -52,13 +53,13 @@ export const TokenUsageBadge = (
   if (props.inline)
     return (
       <span>
-        {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+        {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
       </span>
     );
 
   return (
     <Badge variant="outline">
-      {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+      {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
     </Badge>
   );
 };

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type ScoreSource, type Score } from "@langfuse/shared";
+import { type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -22,11 +22,12 @@ import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string;
   traceId: string;
 }) => {
@@ -59,7 +60,7 @@ export const ObservationPreview = (props: {
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, ValidatedScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,7 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, type $Enums } from "@langfuse/shared";
+import { type Trace, type $Enums } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -22,7 +23,7 @@ export const ObservationTree = (props: {
   collapseAll: () => void;
   expandAll: () => void;
   trace: Trace;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics: boolean;
@@ -61,7 +62,7 @@ const ObservationTreeTraceNode = (props: {
   trace: Trace & { latency?: number };
   expandAll: () => void;
   collapseAll: () => void;
-  scores: Score[];
+  scores: ValidatedScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics?: boolean;
@@ -119,7 +120,7 @@ const ObservationTreeNode = (props: {
   observations: NestedObservation[];
   collapsedObservations: string[];
   toggleCollapsedObservation: (id: string) => void;
-  scores: Score[];
+  scores: ValidatedScore[];
   indentationLevel: number;
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;

web/src/components/trace/ScoresPreview.tsx

@@ -1,10 +1,10 @@
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { type Score } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const ScoresPreview = ({
   itemScoresBySource,
 }: {
-  itemScoresBySource: Map<string, Score[]>;
+  itemScoresBySource: Map<string, ValidatedScore[]>;
 }) => {
   if (!Boolean(itemScoresBySource.size)) return null;
 

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Trace, type Score, type ScoreSource } from "@langfuse/shared";
+import { type Trace, type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -19,6 +19,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export const TracePreview = ({
   trace,
@@ -27,7 +28,7 @@ export const TracePreview = ({
 }: {
   trace: Trace & { latency?: number };
   observations: ObservationReturnType[];
-  scores: Score[];
+  scores: ValidatedScore[];
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -41,7 +42,7 @@ export const TracePreview = ({
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, ValidatedScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">

web/src/components/trace/TraceTimelineView.tsx

@@ -1,12 +1,19 @@
 import { Card } from "@/src/components/ui/card";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Score, type Trace } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
-import React, { useEffect, useRef, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";
 import { SimpleTreeView } from "@mui/x-tree-view/SimpleTreeView";
 import { TreeItem } from "@mui/x-tree-view/TreeItem";
 
-import { MinusIcon, PlusIcon, PanelRightOpen } from "lucide-react";
+import {
+  MinusIcon,
+  PlusIcon,
+  PanelRightOpen,
+  PlusSquareIcon,
+  MinusSquare,
+} from "lucide-react";
 import { nestObservations } from "@/src/components/trace/lib/helpers";
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
@@ -36,6 +43,24 @@ const PREDEFINED_STEP_SIZES = [
   0.25, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10,
 ];
 
+const getNestedObservationKeys = (
+  observations: NestedObservation[],
+): string[] => {
+  const keys: string[] = [];
+
+  const collectKeys = (obs: NestedObservation[]) => {
+    obs.forEach((observation) => {
+      keys.push(`observation-${observation.id}`);
+      if (observation.children) {
+        collectKeys(observation.children);
+      }
+    });
+  };
+
+  collectKeys(observations);
+  return keys;
+};
+
 const calculateStepSize = (latency: number, scaleWidth: number) => {
   const calculatedStepSize = latency / (scaleWidth / STEP_SIZE);
   return (
@@ -163,7 +188,7 @@ function TraceTreeItem({
   traceStartTime: Date;
   totalScaleSpan: number;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   observations: Array<ObservationReturnType>;
   cardWidth: number;
 }) {
@@ -242,7 +267,7 @@ export function TraceTimelineView({
   trace: Trace & { latency?: number };
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
 }) {
   const { latency, name, id } = trace;
   const [backgroundColor, setBackgroundColor] = useState("");
@@ -270,9 +295,17 @@ export function TraceTimelineView({
     };
   }, [parentRef]);
 
+  const nestedObservations = useMemo(
+    () => nestObservations(observations),
+    [observations],
+  );
+  const nestedObservationKeys = useMemo(
+    () => getNestedObservationKeys(nestedObservations),
+    [nestedObservations],
+  );
+
   if (!latency) return null;
 
-  const nestedObservations = nestObservations(observations);
   const stepSize = calculateStepSize(latency, SCALE_WIDTH);
   const totalScaleSpan = stepSize * (SCALE_WIDTH / STEP_SIZE);
 
@@ -283,14 +316,39 @@ export function TraceTimelineView({
         style={{ width: cardWidth }}
       >
         <div className="grid w-full grid-cols-[1fr,auto] items-center p-2">
-          <h3
-            className="p-2 text-2xl font-semibold tracking-tight"
+          <div
+            className="flex flex-row items-center gap-2"
             style={{
               minWidth: `${MIN_LABEL_WIDTH}px`,
             }}
           >
-            Trace Timeline
-          </h3>
+            <h3 className="text-2xl font-semibold tracking-tight">
+              Trace Timeline
+            </h3>
+            <div className="flex h-full items-center">
+              <Button
+                onClick={() =>
+                  setExpandedItems([
+                    `trace-${trace.id}`,
+                    ...nestedObservationKeys,
+                  ])
+                }
+                size="xs"
+                variant="ghost"
+                title="Expand all"
+              >
+                <PlusSquareIcon className="h-4 w-4" />
+              </Button>
+              <Button
+                onClick={() => setExpandedItems([])}
+                size="xs"
+                variant="ghost"
+                title="Collapse all"
+              >
+                <MinusSquare className="h-4 w-4" />
+              </Button>
+            </div>
+          </div>
           <div
             className="relative mr-2 h-4"
             style={{ width: `${SCALE_WIDTH}px` }}

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";
@@ -34,11 +34,12 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePos
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
 import { Alert, AlertDescription, AlertTitle } from "@/src/components/ui/alert";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
   trace: Trace;
-  scores: Score[];
+  scores: ValidatedScore[];
   projectId: string;
 }) {
   const capture = usePostHogClientCapture();
@@ -293,7 +294,7 @@ export function TracePage({ traceId }: { traceId: string }) {
           </Badge>
         ) : undefined}
       </div>
-      <div className="mt-5 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
+      <div className="mt-4 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
         <div className="flex flex-row items-center gap-3 p-2.5">
           Tags
           <TagTraceDetailsPopover
@@ -311,19 +312,19 @@ export function TracePage({ traceId }: { traceId: string }) {
           setSelectedTab(tab);
           capture("trace_detail:display_mode_switch", { view: tab });
         }}
-        className="flex w-full justify-end border-b bg-background"
+        className="mt-2 flex w-full justify-end border-b bg-transparent"
       >
-        <TabsList className="bg-background py-0">
+        <TabsList className="bg-transparent py-0">
           <TabsTrigger
             value="details"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <Network className="mr-1 h-4 w-4"></Network>
             Tree
           </TabsTrigger>
           <TabsTrigger
             value="timeline"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <ListTree className="mr-1 h-4 w-4"></ListTree>
             Timeline
@@ -332,7 +333,7 @@ export function TracePage({ traceId }: { traceId: string }) {
         </TabsList>
       </Tabs>
       {selectedTab === "details" && (
-        <div className="mt-5 flex-1 overflow-hidden border-t pt-5">
+        <div className="mt-5 flex-1 overflow-hidden">
           <Trace
             key={trace.data.id}
             trace={trace.data}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.58.0";
+export const VERSION = "v2.60.2";

web/src/features/dashboard/components/BaseTimeSeriesChart.tsx

@@ -42,7 +42,13 @@ export function BaseTimeSeriesChart(props: {
   }
 
   const convertDate = (date: number, agg: DateTimeAggregationOption) => {
-    if (agg === "24 hours" || agg === "1 hour" || agg === "30 minutes") {
+    const showMinutes: DateTimeAggregationOption[] = [
+      "5 minutes",
+      "30 minutes",
+      "1 hour",
+      "3 hours",
+    ];
+    if (showMinutes.includes(agg)) {
       return new Date(date).toLocaleTimeString("en-US", {
         year: "2-digit",
         month: "numeric",

web/src/features/dashboard/components/MetricTable.tsx

@@ -8,6 +8,8 @@ import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";
 import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 export const MetricTable = ({
   className,
   projectId,
@@ -20,7 +22,9 @@ export const MetricTable = ({
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "totalTokens", agg: "SUM" },

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -20,6 +20,8 @@ import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 export const ModelUsageChart = ({
   className,
   projectId,
@@ -34,7 +36,9 @@ export const ModelUsageChart = ({
   const tokens = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "totalTokens", agg: "SUM" },
         { column: "calculatedTotalCost", agg: "SUM" },

web/src/features/dashboard/components/UserChart.tsx

@@ -15,6 +15,8 @@ import {
   totalCostDashboardFormatted,
 } from "@/src/features/dashboard/lib/dashboard-utils";
 
+import { env } from "@/src/env.mjs";
+
 type BarChartDataPoint = {
   name: string;
   value: number;
@@ -34,7 +36,9 @@ export const UserChart = ({
   const user = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observationsview",
+      from: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION // Langfuse Cloud has already completed the cost backfill job, thus cost can be pulled directly from obs. table
+        ? "traces_observations"
+        : "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "user" },

web/src/features/dashboard/lib/timeseries-aggregation.ts

@@ -6,8 +6,10 @@ export const dateTimeAggregationOptions = [
   "1 month",
   "7 days",
   "24 hours",
+  "3 hours",
   "1 hour",
   "30 minutes",
+  "5 minutes",
 ] as const;
 
 export type DateTimeAggregationOption =
@@ -51,6 +53,12 @@ export const dateTimeAggregationSettings: Record<
       date.toLocaleTimeString("en-US", { hour: "numeric" }),
     minutes: 24 * 60,
   },
+  "3 hours": {
+    date_trunc: "minute",
+    date_formatter: (date) =>
+      date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
+    minutes: 3 * 60,
+  },
   "1 hour": {
     date_trunc: "minute",
     date_formatter: (date) =>
@@ -63,6 +71,12 @@ export const dateTimeAggregationSettings: Record<
       date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
     minutes: 30,
   },
+  "5 minutes": {
+    date_trunc: "minute",
+    date_formatter: (date) =>
+      date.toLocaleTimeString("en-US", { hour: "numeric", minute: "numeric" }),
+    minutes: 5,
+  },
 };
 
 export const findClosestInterval = (

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -7,7 +7,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
-import { type Score } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
 import { usdFormatter } from "../../../utils/numbers";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
@@ -30,7 +30,7 @@ type RowData = {
   output?: unknown;
   expectedOutput?: unknown;
 
-  scores: Score[];
+  scores: ValidatedScore[];
   latency?: number;
   totalCost?: string;
 };

web/src/features/datasets/server/dataset-router.ts

@@ -13,6 +13,7 @@ import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { DB } from "@/src/server/db";
 import { paginationZod } from "@langfuse/shared";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 export const datasetRouter = createTRPCRouter({
   allDatasetMeta: protectedProjectProcedure
@@ -168,6 +169,7 @@ export const datasetRouter = createTRPCRouter({
               WHERE 
                 t.project_id = ${input.projectId}
                 AND s.data_type != 'CATEGORICAL'
+                AND s.value IS NOT NULL
                 AND ri.dataset_run_id = runs.id
               GROUP BY s.name
             ) s
@@ -620,6 +622,10 @@ export const datasetRouter = createTRPCRouter({
         `,
       );
 
+      const validatedTraceScores = filterAndValidateDbScoreList(traceScores);
+      const validatedObservationScores =
+        filterAndValidateDbScoreList(observationScores);
+
       const items = runItems.map((ri) => {
         return {
           id: ri.id,
@@ -628,8 +634,8 @@ export const datasetRouter = createTRPCRouter({
           observation: observations.find((o) => o.id === ri.observationId),
           trace: traces.find((t) => t.id === ri.traceId),
           scores: [
-            ...traceScores.filter((s) => s.traceId === ri.traceId),
-            ...observationScores.filter(
+            ...validatedTraceScores.filter((s) => s.traceId === ri.traceId),
+            ...validatedObservationScores.filter(
               (s) =>
                 s.observationId === ri.observationId &&
                 s.traceId === ri.traceId,

web/src/features/filters/components/filter-builder.tsx

@@ -274,10 +274,11 @@ function FilterBuilderForm({
                           column: col?.name,
                           type: col?.type,
                           operator:
-                            col?.type !== undefined &&
-                            filterOperators[col.type]?.length > 0
-                              ? (filterOperators[col.type][0] as any) // operator matches type
-                              : undefined,
+                            // does not work as expected on eval-template form when embedded into form via InlineFilterBuilder
+                            // col?.type !== undefined &&
+                            // filterOperators[col.type]?.length > 0
+                            //   ? (filterOperators[col.type][0] as any) // operator matches type
+                            undefined,
                           value: undefined,
                           key: undefined,
                         },

web/src/features/manual-scoring/components/AnnotateDrawer.tsx

@@ -31,11 +31,15 @@ import {
   DrawerTrigger,
 } from "@/src/components/ui/drawer";
 import {
-  type ConfigCategory,
   ScoreDataType,
-  type Score,
-  type ScoreConfig,
+  CreateAnnotationScoreData,
+  UpdateAnnotationScoreData,
 } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
+import {
+  type ValidatedScoreConfig,
+  type ConfigCategory,
+} from "@/src/features/public-api/types/score-configs";
 import { z } from "zod";
 import { Input } from "@/src/components/ui/input";
 import {
@@ -56,10 +60,8 @@ import { Textarea } from "@/src/components/ui/textarea";
 import { HoverCardContent } from "@radix-ui/react-hover-card";
 import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
 import { ScoreConfigDetails } from "@/src/features/manual-scoring/components/ScoreConfigDetails";
-import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 import {
   isNumericDataType,
-  isPresent,
   isScoreUnsaved,
 } from "@/src/features/manual-scoring/lib/helpers";
 import { getDefaultScoreData } from "@/src/features/manual-scoring/lib/getDefaultScoreData";
@@ -71,6 +73,7 @@ import { useRouter } from "next/router";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { cn } from "@/src/utils/tailwind";
+import { isPresent } from "@/src/utils/typeChecks";
 
 const AnnotationScoreDataSchema = z.object({
   name: z.string(),
@@ -134,7 +137,7 @@ export function AnnotateDrawer({
   source = "TraceDetail",
 }: {
   traceId: string;
-  scores: Score[];
+  scores: ValidatedScore[];
   observationId?: string;
   projectId: string;
   variant?: "button" | "badge";
@@ -178,9 +181,6 @@ export function AnnotateDrawer({
   });
 
   const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: async (data, error) => {
       if (!data || error) return;
 
@@ -214,7 +214,7 @@ export function AnnotateDrawer({
 
   const utils = api.useUtils();
 
-  const onSettledUpsert = async (data?: Score, error?: unknown) => {
+  const onSettledUpsert = async (data?: ValidatedScore, error?: unknown) => {
     if (!data || error) return;
 
     const { id, value, stringValue, name, dataType, configId, comment } = data;
@@ -240,16 +240,10 @@ export function AnnotateDrawer({
   };
 
   const mutCreateScores = api.scores.createAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: onSettledUpsert,
   });
 
   const mutUpdateScores = api.scores.updateAnnotationScore.useMutation({
-    onError: (error) => {
-      trpcErrorToast(error);
-    },
     onSettled: onSettledUpsert,
   });
 
@@ -314,27 +308,43 @@ export function AnnotateDrawer({
 
         if (!!stringValue) {
           if (!!score.scoreId) {
-            await mutUpdateScores.mutateAsync({
-              projectId,
-              ...score,
+            const validatedScore = UpdateAnnotationScoreData.parse({
               id: score.scoreId,
+              projectId,
+              traceId,
+              name: score.name,
+              dataType: score.dataType,
+              configId: score.configId,
+              comment: score.comment,
+              observationId,
               value: newValue,
               stringValue,
             });
+
+            await mutUpdateScores.mutateAsync({
+              ...validatedScore,
+            });
             capture("score:update", {
               type: type,
               source: source,
               dataType: score.dataType,
             });
           } else {
-            await mutCreateScores.mutateAsync({
+            const validatedScore = CreateAnnotationScoreData.parse({
               projectId,
               traceId,
-              ...score,
+              name: score.name,
+              dataType: score.dataType,
+              configId: score.configId,
+              comment: score.comment,
               observationId,
               value: newValue,
               stringValue,
             });
+
+            await mutCreateScores.mutateAsync({
+              ...validatedScore,
+            });
             capture("score:create", {
               type: type,
               source: source,
@@ -361,13 +371,23 @@ export function AnnotateDrawer({
     return async () => {
       const { value, scoreId } = score;
       if (!!field.value && !!scoreId && isPresent(value)) {
-        await mutUpdateScores.mutateAsync({
-          projectId,
-          ...score,
-          value,
+        const validatedScore = UpdateAnnotationScoreData.parse({
           id: scoreId,
+          projectId,
+          traceId,
+          name: score.name,
+          dataType: score.dataType,
+          configId: score.configId,
+          stringValue: score.stringValue,
+          observationId,
+          value,
           comment,
         });
+
+        await mutUpdateScores.mutateAsync({
+          ...validatedScore,
+        });
+
         capture(comment ? "score:update_comment" : "score:delete_comment", {
           type: type,
           source: source,
@@ -382,7 +402,7 @@ export function AnnotateDrawer({
     index,
     score,
   }: {
-    config: ScoreConfig;
+    config: ValidatedScoreConfig;
     field: ControllerRenderProps<
       AnnotateFormSchemaType,
       `scoreData.${number}.value`
@@ -409,25 +429,45 @@ export function AnnotateDrawer({
 
       if (isPresent(field.value)) {
         if (!!score.scoreId) {
-          await mutUpdateScores.mutateAsync({
-            projectId,
-            ...score,
-            value: Number(field.value),
+          const validatedScore = UpdateAnnotationScoreData.parse({
             id: score.scoreId,
+            projectId,
+            traceId,
+            name: score.name,
+            dataType: score.dataType,
+            configId: score.configId,
+            stringValue: score.stringValue,
+            comment: score.comment,
+            observationId,
+            value: Number(field.value),
           });
+
+          await mutUpdateScores.mutateAsync({
+            ...validatedScore,
+          });
+
           capture("score:update", {
             type: type,
             source: source,
             dataType: score.dataType,
           });
         } else {
-          await mutCreateScores.mutateAsync({
+          const validatedScore = CreateAnnotationScoreData.parse({
             projectId,
             traceId,
-            ...score,
+            name: score.name,
+            dataType: score.dataType,
+            configId: score.configId,
+            stringValue: score.stringValue,
+            comment: score.comment,
             observationId,
             value: Number(field.value),
           });
+
+          await mutCreateScores.mutateAsync({
+            ...validatedScore,
+          });
+
           capture("score:create", {
             type: type,
             source: source,

web/src/features/manual-scoring/components/CreateScoreConfigButton.tsx

@@ -1,7 +1,6 @@
 import React, { useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { z } from "zod";
 import {
   Dialog,
   DialogContent,
@@ -21,7 +20,7 @@ import {
   FormMessage,
 } from "@/src/components/ui/form";
 import { Input } from "@/src/components/ui/input";
-import { ScoreDataType } from "@langfuse/shared";
+import { ScoreDataType, availableDataTypes } from "@langfuse/shared";
 import {
   Select,
   SelectContent,
@@ -36,29 +35,66 @@ import {
   isCategoricalDataType,
   isNumericDataType,
 } from "@/src/features/manual-scoring/lib/helpers";
+import { isPresent } from "@/src/utils/typeChecks";
 import DocPopup from "@/src/components/layouts/doc-popup";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { z } from "zod";
 
-const availableDataTypes = [
-  ScoreDataType.NUMERIC,
-  ScoreDataType.CATEGORICAL,
-  ScoreDataType.BOOLEAN,
-] as const;
-
-const category = z.object({
+const Category = z.object({
   label: z.string().min(1),
-  value: z.coerce.number(),
+  value: z.number(),
 });
 
-const formSchema = z.object({
+const createConfigSchema = z.object({
   name: z.string().min(1).max(35),
   dataType: z.enum(availableDataTypes),
   minValue: z.coerce.number().optional(),
   maxValue: z.coerce.number().optional(),
-  categories: z.array(category).optional(),
+  categories: z.array(Category).optional(),
   description: z.string().optional(),
 });
 
+type CreateConfig = z.infer<typeof createConfigSchema>;
+
+const validateScoreConfig = (values: CreateConfig): string | null => {
+  const { dataType, maxValue, minValue, categories } = values;
+
+  if (isNumericDataType(dataType)) {
+    if (isPresent(maxValue) && isPresent(minValue) && maxValue <= minValue) {
+      return "Maximum value must be greater than Minimum value.";
+    }
+  } else if (isCategoricalDataType(dataType)) {
+    if (!categories || categories.length === 0) {
+      return "At least one category is required for categorical data types.";
+    }
+  } else if (isBooleanDataType(dataType)) {
+    if (categories?.length !== 2)
+      return "Boolean data type must have exactly 2 categories.";
+    const isBooleanCategoryInvalid = categories?.some(
+      (category) => category.value !== 0 && category.value !== 1,
+    );
+    if (isBooleanCategoryInvalid)
+      return "Boolean data type must have categories with values 0 and 1.";
+  }
+
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories || []) {
+    if (uniqueNames.has(category.label)) {
+      return "Category names must be unique.";
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      return "Category values must be unique.";
+    }
+    uniqueValues.add(category.value);
+  }
+
+  return null;
+};
+
 export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
   const [open, setOpen] = useState(false);
   const [formError, setFormError] = useState<string | null>(null);
@@ -76,8 +112,8 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
       setFormError(error.message ?? "An error occurred while creating config."),
   });
 
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
+  const form = useForm<CreateConfig>({
+    resolver: zodResolver(createConfigSchema),
     defaultValues: {
       dataType: ScoreDataType.NUMERIC,
       minValue: undefined,
@@ -93,8 +129,8 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
 
   if (!hasAccess) return null;
 
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    const error = validateForm(values);
+  function onSubmit(values: CreateConfig) {
+    const error = validateScoreConfig(values);
     setFormError(error);
     if (error) return;
 
@@ -380,44 +416,3 @@ export function CreateScoreConfigButton({ projectId }: { projectId: string }) {
     </>
   );
 }
-
-function validateForm(values: z.infer<typeof formSchema>): string | null {
-  if (isNumericDataType(values.dataType)) {
-    if (
-      !!values.maxValue &&
-      !!values.minValue &&
-      values.maxValue <= values.minValue
-    ) {
-      return "Maximum value must be greater than Minimum value.";
-    }
-  } else if (isCategoricalDataType(values.dataType)) {
-    if (!values.categories || values.categories.length === 0) {
-      return "At least one category is required for categorical data types.";
-    }
-  } else if (isBooleanDataType(values.dataType)) {
-    if (values.categories?.length !== 2)
-      return "Boolean data type must have exactly 2 categories.";
-    const isBooleanCategoryInvalid = values.categories?.some(
-      (category) => category.value !== 0 && category.value !== 1,
-    );
-    if (isBooleanCategoryInvalid)
-      return "Boolean data type must have categories with values 0 and 1.";
-  }
-
-  const uniqueNames = new Set<string>();
-  const uniqueValues = new Set<number>();
-
-  for (const category of values.categories || []) {
-    if (uniqueNames.has(category.label)) {
-      return "Category names must be unique.";
-    }
-    uniqueNames.add(category.label);
-
-    if (uniqueValues.has(category.value)) {
-      return "Category values must be unique.";
-    }
-    uniqueValues.add(category.value);
-  }
-
-  return null;
-}

web/src/features/manual-scoring/components/ScoreConfigDetails.tsx

@@ -1,11 +1,13 @@
-import {
-  isNumericDataType,
-  isPresent,
-} from "@/src/features/manual-scoring/lib/helpers";
-import { type ScoreConfig } from "@langfuse/shared";
+import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
+import { type ValidatedScoreConfig } from "@/src/features/public-api/types/score-configs";
+import { isPresent } from "@/src/utils/typeChecks";
 import React from "react";
 
-export function ScoreConfigDetails({ config }: { config: ScoreConfig }) {
+export function ScoreConfigDetails({
+  config,
+}: {
+  config: ValidatedScoreConfig;
+}) {
   const { name, description, minValue, maxValue, dataType } = config;
   if (!description && !isPresent(minValue) && !isPresent(maxValue)) return null;
   const isNameTruncated = name.length > 20;

web/src/features/manual-scoring/lib/getDefaultScoreData.ts

@@ -1,4 +1,6 @@
-import { type Score, ScoreSource, type ScoreConfig } from "@langfuse/shared";
+import { ScoreSource } from "@langfuse/shared";
+import { type ValidatedScore } from "@/src/features/public-api/types/scores";
+import { type ValidatedScoreConfig } from "@/src/features/public-api/types/score-configs";
 
 export const getDefaultScoreData = ({
   scores,
@@ -7,9 +9,9 @@ export const getDefaultScoreData = ({
   traceId,
   observationId,
 }: {
-  scores: Score[];
+  scores: ValidatedScore[];
   emptySelectedConfigIds: string[];
-  configs: ScoreConfig[];
+  configs: ValidatedScoreConfig[];
   traceId: string;
   observationId?: string;
 }) => {

web/src/features/manual-scoring/lib/helpers.ts

@@ -1,8 +1,4 @@
-import {
-  type ScoreConfig,
-  ScoreDataType,
-  type CastedConfig,
-} from "@langfuse/shared";
+import { ScoreDataType } from "@langfuse/shared";
 
 export const isNumericDataType = (dataType: ScoreDataType) =>
   dataType === ScoreDataType.NUMERIC;
@@ -13,21 +9,4 @@ export const isCategoricalDataType = (dataType: ScoreDataType) =>
 export const isBooleanDataType = (dataType: ScoreDataType) =>
   dataType === ScoreDataType.BOOLEAN;
 
-export const isPresent = <T>(value: T): value is NonNullable<T> =>
-  value !== null && value !== undefined && value !== "";
-
 export const isScoreUnsaved = (scoreId?: string): boolean => !scoreId;
-
-export const isCastedConfig = (config: ScoreConfig): config is CastedConfig => {
-  return (
-    config.categories === null ||
-    (Array.isArray(config.categories) &&
-      config.categories.every(
-        (category) =>
-          category !== null &&
-          typeof category === "object" &&
-          "label" in category &&
-          "value" in category,
-      ))
-  );
-};

web/src/features/models/components/NewModelForm.tsx

@@ -29,6 +29,7 @@ import { zodResolver } from "@hookform/resolvers/zod";
 import { JsonEditor } from "@/src/components/json-editor";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import Link from "next/link";
+import { utcDate } from "@/src/utils/dates";
 
 const formSchema = z.object({
   modelName: z.string().min(1),
@@ -124,6 +125,7 @@ export const NewModelForm = (props: {
           typeof JSON.parse(values.tokenizerConfig) === "object"
             ? (JSON.parse(values.tokenizerConfig) as Record<string, number>)
             : undefined,
+        startDate: values.startDate ? utcDate(values.startDate) : undefined,
       })
       .then(() => {
         props.onFormSuccess?.();

web/src/features/models/server/isValidPostgresRegex.ts

@@ -0,0 +1,13 @@
+import { Prisma, type prisma as _prisma } from "@langfuse/shared/src/db";
+
+export const isValidPostgresRegex = async (
+  regex: string,
+  prisma: typeof _prisma,
+): Promise<boolean> => {
+  try {
+    await prisma.$queryRaw(Prisma.sql`SELECT 'test_string' ~ ${regex}`);
+    return true;
+  } catch {
+    return false;
+  }
+};

web/src/features/prompts/server/routers/promptRouter.ts

@@ -635,6 +635,7 @@ export const promptRouter = createTRPCRouter({
           WHERE
               o.type = 'GENERATION'
               AND s.data_type != 'CATEGORICAL'
+              AND s.value IS NOT NULL
               AND o.prompt_id IS NOT NULL
               AND o.project_id = ${input.projectId}
               AND p.id IN (${Prisma.join(input.promptIds)})
@@ -692,6 +693,7 @@ export const promptRouter = createTRPCRouter({
           LEFT JOIN scores s ON tp.trace_id = s.trace_id AND s.observation_id IS NULL AND s.project_id = ${input.projectId}
           WHERE 
               s.data_type != 'CATEGORICAL'
+              AND s.value IS NOT NULL
         ), average_scores_by_prompt AS (
           SELECT 
               prompt_id,

web/src/features/public-api/types/models.ts

@@ -0,0 +1,105 @@
+import {
+  type ModelUsageUnit,
+  paginationMetaResponseZod,
+  paginationZod,
+  type Model as PrismaModel,
+  jsonSchema,
+} from "@langfuse/shared";
+import { z } from "zod";
+
+/**
+ * Objects
+ */
+
+const ModelDefinition = z.object({
+  id: z.string(),
+  modelName: z.string(),
+  matchPattern: z.string(),
+  startDate: z.coerce.date().nullable(),
+  inputPrice: z.number().nonnegative().nullable(),
+  outputPrice: z.number().nonnegative().nullable(),
+  totalPrice: z.number().nonnegative().nullable(),
+  unit: z.enum(["TOKENS", "CHARACTERS", "MILLISECONDS", "SECONDS", "IMAGES"]),
+  tokenizerId: z.string().nullable(),
+  tokenizerConfig: z.any(), // Assuming Prisma.JsonValue is any type
+  isLangfuseManaged: z.boolean(),
+  createdAt: z.coerce.date(),
+});
+
+/**
+ * Transforms
+ */
+
+export function prismaToApiModelDefinition({
+  projectId,
+  inputPrice,
+  outputPrice,
+  totalPrice,
+  unit,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  updatedAt,
+  ...model
+}: PrismaModel): z.infer<typeof ModelDefinition> {
+  return {
+    ...model,
+    unit: unit as ModelUsageUnit,
+    inputPrice: inputPrice?.toNumber() ?? null,
+    outputPrice: outputPrice?.toNumber() ?? null,
+    totalPrice: totalPrice?.toNumber() ?? null,
+    isLangfuseManaged: !Boolean(projectId),
+  };
+}
+
+/**
+ * Endpoints
+ */
+
+// GET /models
+export const GetModelsV1Query = z.object({
+  ...paginationZod,
+});
+export const GetModelsV1Response = z.object({
+  data: z.array(ModelDefinition),
+  meta: paginationMetaResponseZod,
+});
+
+// POST /models
+export const PostModelsV1Body = z
+  .object({
+    modelName: z.string(),
+    matchPattern: z.string(),
+    startDate: z.coerce.date().nullish(),
+    inputPrice: z.number().nonnegative().nullish(),
+    outputPrice: z.number().nonnegative().nullish(),
+    totalPrice: z.number().nonnegative().nullish(),
+    unit: z.enum(["TOKENS", "CHARACTERS", "MILLISECONDS", "SECONDS", "IMAGES"]),
+    tokenizerId: z.enum(["openai", "claude"]).nullish(),
+    tokenizerConfig: jsonSchema.nullish(), // Assuming Prisma.JsonValue is any type
+  })
+  .refine(
+    ({ inputPrice, outputPrice, totalPrice }) => {
+      if (inputPrice || outputPrice) {
+        return !totalPrice;
+      }
+      return true;
+    },
+    {
+      path: ["totalPrice"],
+      message: "If input and/or output price is set, total price must be null",
+    },
+  );
+export const PostModelsV1Response = ModelDefinition;
+
+// GET /models/{modelId}
+export const GetModelV1Query = z.object({
+  modelId: z.string(),
+});
+export const GetModelV1Response = ModelDefinition;
+
+// DELETE /models/{modelId}
+export const DeleteModelV1Query = z.object({
+  modelId: z.string(),
+});
+export const DeleteModelV1Response = z.object({
+  message: z.literal("Model successfully deleted"),
+});

web/src/features/public-api/types/score-configs.ts

@@ -0,0 +1,210 @@
+import { isPresent } from "@/src/utils/typeChecks";
+import {
+  jsonSchema,
+  paginationMetaResponseZod,
+  paginationZod,
+  type ScoreConfig as ScoreConfigDbType,
+} from "@langfuse/shared";
+import { z } from "zod";
+import * as Sentry from "@sentry/node";
+
+const validateCategories = (
+  categories: ConfigCategory[],
+  ctx: z.RefinementCtx,
+) => {
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories) {
+    if (uniqueNames.has(category.label)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category label: ${category.label}, category labels must be unique`,
+      });
+      return;
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category value: ${category.value}, category values must be unique`,
+      });
+      return;
+    }
+    uniqueValues.add(category.value);
+  }
+};
+
+/**
+ * Objects
+ */
+export const Category = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+export type ConfigCategory = z.infer<typeof Category>;
+
+const Categories = z.array(Category);
+
+const NumericScoreConfig = z.object({
+  maxValue: z.number().optional().nullish(),
+  minValue: z.number().optional().nullish(),
+  dataType: z.literal("NUMERIC"),
+  categories: z.undefined().nullish(),
+});
+
+const CategoricalScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("CATEGORICAL"),
+  categories: jsonSchema.superRefine((categories, ctx) => {
+    const parseResult = Categories.safeParse(categories);
+    if (!parseResult.success) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "Category must be an array of objects with label value pairs, where labels and values are unique.",
+      });
+      return;
+    }
+
+    validateCategories(categories as ConfigCategory[], ctx);
+  }),
+});
+
+const BooleanScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("BOOLEAN"),
+  categories: z
+    .array(Category)
+    .length(2, "Boolean data type must have exactly 2 categories.")
+    .refine((categories) => {
+      const expectedCategories = [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ];
+      return categories.every(
+        (category, index) =>
+          category.label === expectedCategories[index].label &&
+          category.value === expectedCategories[index].value,
+      );
+    }),
+});
+
+const ScoreConfigBase = z.object({
+  id: z.string(),
+  name: z.string().min(1).max(35),
+  isArchived: z.boolean(),
+  description: z.string().optional().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  projectId: z.string(),
+});
+
+const ScoreConfigPostBase = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+});
+
+const ValidatedScoreConfigSchema = z
+  .union([
+    ScoreConfigBase.merge(NumericScoreConfig),
+    ScoreConfigBase.merge(
+      z.object({
+        maxValue: z.undefined().nullish(),
+        minValue: z.undefined().nullish(),
+        dataType: z.literal("CATEGORICAL"),
+        categories: Categories.superRefine(validateCategories),
+      }),
+    ),
+    ScoreConfigBase.merge(BooleanScoreConfig),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
+
+export const filterAndValidateDbScoreConfigList = (
+  scoreConfigs: ScoreConfigDbType[],
+): ValidatedScoreConfig[] =>
+  scoreConfigs.reduce((acc, ts) => {
+    const result = ValidatedScoreConfigSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      Sentry.captureException(result.error);
+    }
+    return acc;
+  }, [] as ValidatedScoreConfig[]);
+
+export const validateDbScoreConfig = (
+  scoreConfig: ScoreConfigDbType,
+): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
+
+export const validateDbScoreConfigSafe = (scoreConfig: ScoreConfigDbType) =>
+  ValidatedScoreConfigSchema.safeParse(scoreConfig);
+
+/**
+ * Endpoints
+ */
+
+// GET /score-configs/{configId}
+export const GetScoreConfigQuery = z.object({
+  configId: z.string(),
+});
+
+export const GetScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// POST /score-configs
+export const PostScoreConfigBody = z
+  .union([
+    ScoreConfigPostBase.merge(CategoricalScoreConfig),
+    ScoreConfigPostBase.merge(NumericScoreConfig),
+    ScoreConfigPostBase.merge(
+      z.object({
+        dataType: z.literal("BOOLEAN"),
+        categories: z.undefined().nullish(),
+      }),
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// GET /score-configs
+export const GetScoreConfigsQuery = z.object({
+  ...paginationZod,
+});
+
+export const GetScoreConfigsResponse = z.object({
+  data: z.array(ValidatedScoreConfigSchema),
+  meta: paginationMetaResponseZod,
+});

web/src/features/public-api/types/scores.ts

@@ -0,0 +1,293 @@
+import * as Sentry from "@sentry/node";
+import {
+  paginationZod,
+  paginationMetaResponseZod,
+  NonEmptyString,
+  type Score,
+} from "@langfuse/shared";
+import { z } from "zod";
+import { isPresent } from "@/src/utils/typeChecks";
+import { Category as ConfigCategory } from "./score-configs";
+
+/**
+ * Objects
+ */
+
+const ScoreSource = ["API", "EVAL", "ANNOTATION"] as const;
+const ScoreDataType = ["NUMERIC", "CATEGORICAL", "BOOLEAN"] as const;
+
+const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
+
+const NumericData = z.object({
+  value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
+});
+
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
+
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
+
+const ScoreBase = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  projectId: z.string(),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  authorUserId: z.string().nullish(),
+  comment: z.string().nullish(),
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  configId: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+const BaseScoreBody = z.object({
+  id: z.string().nullish(),
+  name: NonEmptyString,
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  comment: z.string().nullish(),
+});
+
+const GetScoresDataBase = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  comment: z.string().nullish(),
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  trace: z.object({
+    userId: z.string(),
+  }),
+  configId: z.string().nullish(),
+});
+
+export const GetScoresData = z.discriminatedUnion("dataType", [
+  GetScoresDataBase.merge(NumericData),
+  GetScoresDataBase.merge(CategoricalData),
+  GetScoresDataBase.merge(BooleanData),
+]);
+
+const ValidatedScoreSchema = z.discriminatedUnion("dataType", [
+  ScoreBase.merge(NumericData),
+  ScoreBase.merge(CategoricalData),
+  ScoreBase.merge(BooleanData),
+]);
+
+export type ValidatedGetScoresData = z.infer<typeof GetScoresData>;
+
+export type ValidatedScore = z.infer<typeof ValidatedScoreSchema>;
+
+/**
+ * Validation objects
+ */
+export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number(),
+      dataType: z.literal("NUMERIC"),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.string(),
+      dataType: z.literal("CATEGORICAL"),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number().refine((val) => val === 0 || val === 1, {
+        message: "Value must be either 0 or 1",
+      }),
+      dataType: z.literal("BOOLEAN"),
+    }),
+  ),
+]);
+
+const ScorePropsAgainstConfigNumeric = z
+  .object({
+    value: z.number(),
+    maxValue: z.number().optional(),
+    minValue: z.number().optional(),
+    dataType: z.literal("NUMERIC"),
+  })
+  .superRefine((data, ctx) => {
+    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
+      });
+    }
+    if (isPresent(data.minValue) && data.value <= data.minValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value is below minimum value of ${data.minValue} defined in config`,
+      });
+    }
+  });
+
+const ScorePropsAgainstConfigCategorical = z
+  .object({
+    value: z.string(),
+    categories: z.array(ConfigCategory),
+    dataType: z.literal("CATEGORICAL"),
+  })
+  .superRefine((data, ctx) => {
+    if (!data.categories.some(({ label }) => label === data.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value ${data.value} does not map to a valid category. Pass a valid category value.`,
+      });
+    }
+  });
+
+export const ScorePropsAgainstConfig = z.union([
+  ScorePropsAgainstConfigNumeric,
+  ScorePropsAgainstConfigCategorical,
+  z.object({
+    value: z.number().refine((val) => val === 0 || val === 1, {
+      message: "Value must be either 0 or 1",
+    }),
+    dataType: z.literal("BOOLEAN"),
+  }),
+]);
+
+/**
+ * Transformations
+ */
+
+export const filterAndValidateDbScoreList = (
+  scores: Score[],
+): ValidatedScore[] =>
+  scores.reduce((acc, ts) => {
+    const result = ValidatedScoreSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      Sentry.captureException(result.error);
+    }
+    return acc;
+  }, [] as ValidatedScore[]);
+
+export const validateDbScore = (score: Score): ValidatedScore =>
+  ValidatedScoreSchema.parse(score);
+
+/**
+ * Endpoints
+ */
+
+// POST /scores
+/**
+ * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
+ */
+export const PostScoresBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      }),
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
+export const PostScoresResponse = z.void();
+
+// GET /scores
+export const GetScoresQuery = z.object({
+  ...paginationZod,
+  userId: z.string().nullish(),
+  dataType: z.enum(ScoreDataType).nullish(),
+  configId: z.string().nullish(),
+  name: z.string().nullish(),
+  fromTimestamp: z.coerce.date().nullish(),
+  source: z.enum(ScoreSource).nullish(),
+  value: z.coerce.number().nullish(),
+  operator: z.enum(operators).nullish(),
+  scoreIds: z
+    .string()
+    .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
+    .refine((arr) => arr.every((id) => typeof id === "string"), {
+      message: "Each score ID must be a string",
+    })
+    .nullish(),
+});
+
+export const GetScoresResponse = z.object({
+  data: z.array(GetScoresData),
+  meta: paginationMetaResponseZod,
+});
+
+// GET /scores/{scoreId}
+export const GetScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const GetScoreResponse = ValidatedScoreSchema;
+
+// DELETE /scores/{scoreId}
+export const DeleteScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const DeleteScoreResponse = z.object({
+  message: z.string(),
+});

web/src/new-relic-instrumentation.ts

@@ -0,0 +1,51 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { Resource } from "@opentelemetry/resources";
+import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import {
+  awsEksDetector,
+  awsEc2Detector,
+} from "@opentelemetry/resource-detector-aws";
+import {
+  hostDetector,
+  osDetector,
+  processDetector,
+} from "@opentelemetry/resources/build/src/detectors/platform";
+import { envDetector } from "@opentelemetry/resources";
+import { containerDetector } from "@opentelemetry/resource-detector-container";
+
+const sdk = new NodeSDK({
+  resource: new Resource({
+    [SEMRESATTRS_SERVICE_NAME]: "web",
+  }),
+  spanProcessors: [
+    new BatchSpanProcessor(
+      new OTLPTraceExporter({
+        url:
+          process.env.OTLP_ENDPOINT ||
+          "https://otlp.eu01.nr-data.net/v1/traces",
+        headers: {
+          "api-key": process.env.NEW_RELIC_API_KEY,
+        },
+      }),
+    ),
+  ],
+  resourceDetectors: [
+    containerDetector,
+    envDetector,
+    hostDetector,
+    osDetector,
+    processDetector,
+    awsEksDetector,
+    awsEc2Detector,
+  ],
+  instrumentations: [
+    getNodeAutoInstrumentations(),
+    new PrismaInstrumentation(),
+  ],
+});
+
+sdk.start();

web/src/pages/api/public/ingestion.ts

@@ -134,7 +134,7 @@ export default async function handler(
       res,
     );
   } catch (error: unknown) {
-    console.error("error handling ingestion event", error);
+    console.error("error_handling_ingestion_event", error);
 
     if (!(error instanceof UnauthorizedError)) {
       Sentry.captureException(error);
@@ -494,6 +494,7 @@ export const sendToWorkerIfEnvironmentConfigured = async (
               ).toString("base64"),
           },
           body: JSON.stringify(body),
+          signal: AbortSignal.timeout(2 * 1000),
         });
       }
     }

web/src/pages/api/public/models/[modelId]/index.ts

@@ -0,0 +1,71 @@
+import { prisma } from "@langfuse/shared/src/db";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import {
+  DeleteModelV1Query,
+  DeleteModelV1Response,
+  GetModelV1Query,
+  GetModelV1Response,
+  prismaToApiModelDefinition,
+} from "@/src/features/public-api/types/models";
+import { LangfuseNotFoundError } from "@langfuse/shared";
+
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get model definitions",
+    querySchema: GetModelV1Query,
+    responseSchema: GetModelV1Response,
+    fn: async ({ query, auth }) => {
+      const model = await prisma.model.findFirst({
+        where: {
+          AND: [
+            {
+              id: query.modelId,
+            },
+            {
+              OR: [
+                {
+                  projectId: auth.scope.projectId,
+                },
+                {
+                  projectId: null,
+                },
+              ],
+            },
+          ],
+        },
+      });
+      if (!model) {
+        throw new LangfuseNotFoundError("No model with this id found.");
+      }
+      return prismaToApiModelDefinition(model);
+    },
+  }),
+  DELETE: createAuthedAPIRoute({
+    name: "Delete model",
+    querySchema: DeleteModelV1Query,
+    responseSchema: DeleteModelV1Response,
+    fn: async ({ query, auth }) => {
+      const model = await prisma.model.findFirst({
+        where: {
+          id: query.modelId,
+          projectId: auth.scope.projectId,
+        },
+      });
+      if (!model) {
+        throw new LangfuseNotFoundError(
+          "No model with this id found. Note: You cannot delete built-in models, override them with a model with the same name.",
+        );
+      }
+      await prisma.model.delete({
+        where: {
+          id: query.modelId,
+          projectId: auth.scope.projectId,
+        },
+      });
+      return {
+        message: "Model successfully deleted" as const,
+      };
+    },
+  }),
+});

web/src/pages/api/public/models/index.ts

@@ -0,0 +1,91 @@
+import { prisma } from "@langfuse/shared/src/db";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import {
+  GetModelsV1Query,
+  GetModelsV1Response,
+  PostModelsV1Body,
+  PostModelsV1Response,
+  prismaToApiModelDefinition,
+} from "@/src/features/public-api/types/models";
+import { InvalidRequestError } from "@langfuse/shared";
+import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";
+
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get model definitions",
+    querySchema: GetModelsV1Query,
+    responseSchema: GetModelsV1Response,
+    fn: async ({ query, auth }) => {
+      const models = await prisma.model.findMany({
+        where: {
+          OR: [
+            {
+              projectId: auth.scope.projectId,
+            },
+            {
+              projectId: null,
+            },
+          ],
+        },
+        orderBy: [
+          { modelName: "asc" },
+          { unit: "asc" },
+          {
+            startDate: {
+              sort: "desc",
+              nulls: "last",
+            },
+          },
+        ],
+        take: query.limit,
+        skip: (query.page - 1) * query.limit,
+      });
+
+      const totalItems = await prisma.model.count({
+        where: {
+          OR: [
+            {
+              projectId: auth.scope.projectId,
+            },
+            {
+              projectId: null,
+            },
+          ],
+        },
+      });
+
+      return {
+        data: models.map(prismaToApiModelDefinition),
+        meta: {
+          page: query.page,
+          limit: query.limit,
+          totalItems,
+          totalPages: Math.ceil(totalItems / query.limit),
+        },
+      };
+    },
+  }),
+  POST: createAuthedAPIRoute({
+    name: "Create custom model definition",
+    bodySchema: PostModelsV1Body,
+    responseSchema: PostModelsV1Response,
+    fn: async ({ body, auth }) => {
+      const validRegex = await isValidPostgresRegex(body.matchPattern, prisma);
+      if (!validRegex) {
+        throw new InvalidRequestError(
+          "matchPattern is not a valid regex pattern (Postgres)",
+        );
+      }
+      const { tokenizerConfig, ...rest } = body;
+      const model = await prisma.model.create({
+        data: {
+          ...rest,
+          tokenizerConfig: tokenizerConfig ?? undefined,
+          projectId: auth.scope.projectId,
+        },
+      });
+      return prismaToApiModelDefinition(model);
+    },
+  }),
+});

web/src/pages/api/public/score-configs.ts

@@ -1,109 +1,92 @@
-import { type CastedConfig, paginationZod } from "@langfuse/shared";
-import { z } from "zod";
-import { prisma } from "@langfuse/shared/src/db";
-import { Prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
-import { isCastedConfig } from "@/src/features/manual-scoring/lib/helpers";
+import { type z } from "zod";
+import { Prisma, prisma } from "@langfuse/shared/src/db";
+import { isBooleanDataType } from "@/src/features/manual-scoring/lib/helpers";
+import { v4 } from "uuid";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+  GetScoreConfigsQuery,
+  PostScoreConfigBody,
+  validateDbScoreConfig,
+  filterAndValidateDbScoreConfigList,
+} from "@/src/features/public-api/types/score-configs";
 
-const ScoreConfigsGetSchema = z.object({
-  ...paginationZod,
-});
+const inflateConfigBody = (body: z.infer<typeof PostScoreConfigBody>) => {
+  if (isBooleanDataType(body.dataType)) {
+    return {
+      ...body,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+    };
+  }
+  return body;
+};
 
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
+export default withMiddlewares({
+  POST: createAuthedAPIRoute({
+    name: "Create Score Config",
+    bodySchema: PostScoreConfigBody,
+    responseSchema: PostScoreConfigResponse,
+    fn: async ({ body, auth }) => {
+      const inflatedConfigInput = inflateConfigBody(body);
 
-  try {
-    // CHECK AUTH
-    const authCheck = await verifyAuthHeaderAndReturnScope(
-      req.headers.authorization,
-    );
-    if (!authCheck.validKey) {
-      return res.status(401).json({
-        message: authCheck.error,
+      const config = await prisma.scoreConfig.create({
+        data: {
+          ...inflatedConfigInput,
+          categories: inflatedConfigInput.categories ?? undefined,
+          id: v4(),
+          projectId: auth.scope.projectId,
+        },
       });
-    }
-    // END CHECK AUTH
-
-    if (req.method !== "GET") {
-      return res.status(405).json({ message: "Method not allowed" });
-    }
-
-    if (req.method === "GET") {
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-
-      const obj = ScoreConfigsGetSchema.parse(req.query);
 
+      return validateDbScoreConfig(config);
+    },
+  }),
+  GET: createAuthedAPIRoute({
+    name: "Get Score Configs",
+    querySchema: GetScoreConfigsQuery,
+    responseSchema: GetScoreConfigsResponse,
+    fn: async ({ query, auth }) => {
+      const { page, limit } = query;
       const rawConfigs = await prisma.scoreConfig.findMany({
         where: {
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
         orderBy: {
           createdAt: "desc",
         },
-        take: obj.limit,
-        skip: (obj.page - 1) * obj.limit,
+        take: limit,
+        skip: (page - 1) * limit,
       });
 
-      const configs: CastedConfig[] = rawConfigs.filter(isCastedConfig);
-
-      if (configs.length !== rawConfigs.length) {
-        return res.status(500).json({
-          message: "Internal Server Error",
-          error: "Invalid config format encountered",
-        });
-      }
+      const configs = filterAndValidateDbScoreConfigList(rawConfigs);
 
       const totalItemsRes = await prisma.$queryRaw<{ count: bigint }[]>(
         Prisma.sql`
-          SELECT 
+          SELECT
             COUNT(*) as count
-          FROM 
+          FROM
             "score_configs" AS sc
-          WHERE sc.project_id = ${authCheck.scope.projectId}
+          WHERE sc.project_id = ${auth.scope.projectId}
         `,
       );
 
       const totalItems =
         totalItemsRes[0] !== undefined ? Number(totalItemsRes[0].count) : 0;
 
-      return res.status(200).json({
+      return {
         data: configs,
         meta: {
-          page: obj.page,
-          limit: obj.limit,
+          page: page,
+          limit: limit,
           totalItems,
-          totalPages: Math.ceil(totalItems / obj.limit),
+          totalPages: Math.ceil(totalItems / limit),
         },
-      });
-    }
-  } catch (error: unknown) {
-    console.error(error);
-    if (isPrismaException(error)) {
-      return res.status(500).json({
-        error: "Internal Server Error",
-      });
-    }
-    if (error instanceof z.ZodError) {
-      return res.status(400).json({
-        message: "Invalid request data",
-        error: error.errors,
-      });
-    }
-    const errorMessage =
-      error instanceof Error ? error.message : "An unknown error occurred";
-    res.status(500).json({
-      message: "Invalid request data",
-      error: errorMessage,
-    });
-  }
-}
+      };
+    },
+  }),
+});

web/src/pages/api/public/score-configs/[configId].ts

@@ -1,85 +1,39 @@
 import { prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
-import { isCastedConfig } from "@/src/features/manual-scoring/lib/helpers";
-
-const ConfigSchema = z.object({
-  configId: z.string(),
-});
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
-
-  if (req.method === "GET") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-
-      const { configId } = ConfigSchema.parse(req.query);
-
-      // CHECK ACCESS SCOPE
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-      // END CHECK ACCESS SCOPE
+import { InternalServerError, LangfuseNotFoundError } from "@langfuse/shared";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  GetScoreConfigQuery,
+  GetScoreConfigResponse,
+} from "@/src/features/public-api/types/score-configs";
+import * as Sentry from "@sentry/node";
 
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get a Score Config",
+    querySchema: GetScoreConfigQuery,
+    responseSchema: GetScoreConfigResponse,
+    fn: async ({ query, auth }) => {
       const config = await prisma.scoreConfig.findUnique({
         where: {
-          id: configId,
-          projectId: authCheck.scope.projectId,
+          id: query.configId,
+          projectId: auth.scope.projectId,
         },
       });
 
       if (!config) {
-        return res.status(404).json({
-          message: "Score config not found within authorized project",
-        });
+        throw new LangfuseNotFoundError(
+          "Score config not found within authorized project",
+        );
       }
 
-      if (isCastedConfig(config)) {
-        return res.status(200).json(config);
-      } else {
-        return res.status(500).json({
-          message: "Internal Server Error",
-          error: "Invalid config format encountered",
-        });
+      const parsedConfig = GetScoreConfigResponse.safeParse(config);
+      if (!parsedConfig.success) {
+        Sentry.captureException(parsedConfig.error);
+        throw new InternalServerError("Requested score config is corrupted");
       }
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+
+      return parsedConfig.data;
+    },
+  }),
+});

web/src/pages/api/public/scores.ts

@@ -1,151 +1,109 @@
-import { ScoreSource, prisma } from "@langfuse/shared/src/db";
-import { Prisma, type Score } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { paginationZod } from "@langfuse/shared";
-import {
-  ScoreBody,
-  eventTypes,
-  ingestionBatchEvent,
-  stringDate,
-} from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import { Prisma } from "@langfuse/shared/src/db";
+import { eventTypes, ingestionBatchEvent } from "@langfuse/shared";
+import * as Sentry from "@sentry/node";
 import { v4 } from "uuid";
 import {
   handleBatch,
   handleBatchResultLegacy,
 } from "@/src/pages/api/public/ingestion";
-import { isPrismaException } from "@/src/utils/exceptions";
-
-const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
-
-const ScoresGetSchema = z
-  .object({
-    ...paginationZod,
-    userId: z.string().nullish(),
-    name: z.string().nullish(),
-    fromTimestamp: stringDate,
-    source: z.nativeEnum(ScoreSource).nullish(),
-    value: z.coerce.number().nullish(),
-    operator: z.enum(operators).nullish(),
-    scoreIds: z
-      .string()
-      .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
-      .refine((arr) => arr.every((id) => typeof id === "string"), {
-        message: "Each score ID must be a string",
-      })
-      .nullish(),
-  })
-  .strict(); // Use strict to give 400s on typo'd query params
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
-
-  // CHECK AUTH
-  const authCheck = await verifyAuthHeaderAndReturnScope(
-    req.headers.authorization,
-  );
-  if (!authCheck.validKey)
-    return res.status(401).json({
-      message: authCheck.error,
-    });
-  // END CHECK AUTH
-
-  if (req.method === "POST") {
-    try {
-      console.log(
-        "trying to create score, project ",
-        authCheck.scope.projectId,
-        ", body:",
-        JSON.stringify(req.body, null, 2),
-      );
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import {
+  GetScoresData,
+  GetScoresQuery,
+  GetScoresResponse,
+  PostScoresBody,
+  PostScoresResponse,
+  type ValidatedGetScoresData,
+} from "@/src/features/public-api/types/scores";
 
+export default withMiddlewares({
+  POST: createAuthedAPIRoute({
+    name: "Create Score",
+    bodySchema: PostScoresBody,
+    responseSchema: PostScoresResponse,
+    fn: async ({ body, auth, req, res }) => {
       const event = {
         id: v4(),
         type: eventTypes.SCORE_CREATE,
         timestamp: new Date().toISOString(),
-        body: ScoreBody.parse(req.body),
+        body,
       };
-
       const result = await handleBatch(
         ingestionBatchEvent.parse([event]),
         {},
         req,
-        authCheck,
+        auth,
       );
-
       handleBatchResultLegacy(result.errors, result.results, res);
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else if (req.method === "GET") {
-    try {
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
+    },
+  }),
+  GET: createAuthedAPIRoute({
+    name: "Get Scores",
+    querySchema: GetScoresQuery,
+    responseSchema: GetScoresResponse,
+    fn: async ({ query, auth }) => {
+      const {
+        page,
+        limit,
+        configId,
+        userId,
+        name,
+        fromTimestamp,
+        source,
+        operator,
+        value,
+        scoreIds,
+        dataType,
+      } = query;
 
-      const obj = ScoresGetSchema.parse(req.query); // uses query and not body
-
-      const skipValue = (obj.page - 1) * obj.limit;
-      const userCondition = obj.userId
-        ? Prisma.sql`AND t."user_id" = ${obj.userId}`
+      const skipValue = (page - 1) * limit;
+      const configCondition = configId
+        ? Prisma.sql`AND s."config_id" = ${configId}`
         : Prisma.empty;
-      const nameCondition = obj.name
-        ? Prisma.sql`AND s."name" = ${obj.name}`
+      const dataTypeCondition = dataType
+        ? Prisma.sql`AND s."data_type" = ${dataType}::"ScoreDataType"`
         : Prisma.empty;
-      const fromTimestampCondition = obj.fromTimestamp
-        ? Prisma.sql`AND s."timestamp" >= ${obj.fromTimestamp}::timestamp with time zone at time zone 'UTC'`
+      const userCondition = userId
+        ? Prisma.sql`AND t."user_id" = ${userId}`
         : Prisma.empty;
-      const sourceCondition = obj.source
-        ? Prisma.sql`AND s."source" = ${obj.source}`
+      const nameCondition = name
+        ? Prisma.sql`AND s."name" = ${name}`
+        : Prisma.empty;
+      const fromTimestampCondition = fromTimestamp
+        ? Prisma.sql`AND s."timestamp" >= ${fromTimestamp}::timestamp with time zone at time zone 'UTC'`
+        : Prisma.empty;
+      const sourceCondition = source
+        ? Prisma.sql`AND s."source" = ${source}`
         : Prisma.empty;
       const valueCondition =
-        obj.operator && obj.value !== null && obj.value !== undefined
-          ? Prisma.sql`AND s."value" ${Prisma.raw(`${obj.operator}`)} ${obj.value}`
+        operator && value !== null && value !== undefined
+          ? Prisma.sql`AND s."value" ${Prisma.raw(`${operator}`)} ${value}`
           : Prisma.empty;
-      const scoreIdCondition = obj.scoreIds
-        ? Prisma.sql`AND s."id" = ANY(${obj.scoreIds})`
+      const scoreIdCondition = scoreIds
+        ? Prisma.sql`AND s."id" = ANY(${scoreIds})`
         : Prisma.empty;
 
-      const scores = await prisma.$queryRaw<
-        Array<Score & { trace: { userId: string } }>
-      >(Prisma.sql`
+      const scores = await prisma.$queryRaw<Array<unknown>>(Prisma.sql`
           SELECT
             s.id,
             s.timestamp,
             s.name,
             s.value,
+            s.string_value as "stringValue",
             s.source,
             s.comment,
+            s.data_type as "dataType",
+            s.config_id as "configId",
             s.trace_id as "traceId",
             s.observation_id as "observationId",
             json_build_object('userId', t.user_id) as "trace"
           FROM "scores" AS s
-          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${authCheck.scope.projectId}
-          WHERE s.project_id = ${authCheck.scope.projectId}
+          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${auth.scope.projectId}
+          WHERE s.project_id = ${auth.scope.projectId}
+          ${configCondition}
+          ${dataTypeCondition}
           ${userCondition}
           ${nameCondition}
           ${sourceCondition}
@@ -153,15 +111,17 @@ export default async function handler(
           ${valueCondition}
           ${scoreIdCondition}
           ORDER BY s."timestamp" DESC
-          LIMIT ${obj.limit} OFFSET ${skipValue}
+          LIMIT ${limit} OFFSET ${skipValue}
           `);
 
       const totalItemsRes = await prisma.$queryRaw<{ count: bigint }[]>(
         Prisma.sql`
           SELECT COUNT(*) as count
           FROM "scores" AS s
-          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${authCheck.scope.projectId}
-          WHERE s.project_id = ${authCheck.scope.projectId}
+          LEFT JOIN "traces" AS t ON t.id = s.trace_id AND t.project_id = ${auth.scope.projectId}
+          WHERE s.project_id = ${auth.scope.projectId}
+          ${configCondition}
+          ${dataTypeCondition}
           ${userCondition}
           ${nameCondition}
           ${sourceCondition}
@@ -171,39 +131,31 @@ export default async function handler(
         `,
       );
 
+      const validatedScores = scores.reduce(
+        (acc: ValidatedGetScoresData[], score) => {
+          const result = GetScoresData.safeParse(score);
+          if (result.success) {
+            acc.push(result.data);
+          } else {
+            Sentry.captureException(result.error);
+          }
+          return acc;
+        },
+        [] as ValidatedGetScoresData[],
+      );
+
       const totalItems =
         totalItemsRes[0] !== undefined ? Number(totalItemsRes[0].count) : 0;
 
-      return res.status(200).json({
-        data: scores,
+      return {
+        data: validatedScores,
         meta: {
-          page: obj.page,
-          limit: obj.limit,
+          page: page,
+          limit: limit,
           totalItems,
-          totalPages: Math.ceil(totalItems / obj.limit),
+          totalPages: Math.ceil(totalItems / limit),
         },
-      });
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+      };
+    },
+  }),
+});

web/src/pages/api/public/scores/[scoreId].ts

@@ -1,38 +1,50 @@
 import { prisma } from "@langfuse/shared/src/db";
-import { type NextApiRequest, type NextApiResponse } from "next";
-import { z } from "zod";
-import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
-import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
-import { isPrismaException } from "@/src/utils/exceptions";
+import { withMiddlewares } from "@/src/features/public-api/server/withMiddlewares";
+import { createAuthedAPIRoute } from "@/src/features/public-api/server/createAuthedAPIRoute";
+import * as Sentry from "@sentry/node";
+import {
+  DeleteScoreQuery,
+  DeleteScoreResponse,
+  GetScoreQuery,
+  GetScoreResponse,
+} from "@/src/features/public-api/types/scores";
+import { InternalServerError, LangfuseNotFoundError } from "@langfuse/shared";
 
-const ScoreSchema = z.object({
-  scoreId: z.string(),
-});
+export default withMiddlewares({
+  GET: createAuthedAPIRoute({
+    name: "Get Score",
+    querySchema: GetScoreQuery,
+    responseSchema: GetScoreResponse,
+    fn: async ({ query, auth }) => {
+      const { scoreId } = query;
 
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse,
-) {
-  await runMiddleware(req, res, cors);
+      const score = await prisma.score.findUnique({
+        where: {
+          id: scoreId,
+          projectId: auth.scope.projectId,
+        },
+      });
 
-  if (req.method === "DELETE") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
+      if (!score) {
+        throw new LangfuseNotFoundError("Score not found");
       }
 
-      const { scoreId } = ScoreSchema.parse(req.query); // uses query and not body
+      const parsedScore = GetScoreResponse.safeParse(score);
+
+      if (!parsedScore.success) {
+        Sentry.captureException(parsedScore.error);
+        throw new InternalServerError("Requested score is corrupted");
+      }
+
+      return parsedScore.data;
+    },
+  }),
+  DELETE: createAuthedAPIRoute({
+    name: "Delete Score",
+    querySchema: DeleteScoreQuery,
+    responseSchema: DeleteScoreResponse,
+    fn: async ({ query, auth }) => {
+      const { scoreId } = query;
 
       const score = await prisma.score.findUnique({
         select: {
@@ -40,101 +52,24 @@ export default async function handler(
         },
         where: {
           id: scoreId,
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
       });
 
       if (!score) {
-        return res.status(404).json({
-          message: "Score not found within authorized project",
-        });
+        throw new LangfuseNotFoundError(
+          "Score not found within authorized project",
+        );
       }
 
       await prisma.score.delete({
         where: {
           id: scoreId,
-          projectId: authCheck.scope.projectId,
+          projectId: auth.scope.projectId,
         },
       });
 
-      return res.status(200).json({ message: "Score deleted successfully" });
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else if (req.method === "GET") {
-    try {
-      // CHECK AUTH
-      const authCheck = await verifyAuthHeaderAndReturnScope(
-        req.headers.authorization,
-      );
-      if (!authCheck.validKey)
-        return res.status(401).json({
-          message: authCheck.error,
-        });
-      // END CHECK AUTH
-
-      const { scoreId } = ScoreSchema.parse(req.query);
-
-      // CHECK ACCESS SCOPE
-      if (authCheck.scope.accessLevel !== "all") {
-        return res.status(401).json({
-          message: "Access denied - need to use basic auth with secret key",
-        });
-      }
-      // END CHECK ACCESS SCOPE
-
-      const score = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-          projectId: authCheck.scope.projectId,
-        },
-      });
-
-      if (!score) {
-        return res.status(404).json({
-          message: "Score not found within authorized project",
-        });
-      }
-
-      return res.status(200).json(score);
-    } catch (error: unknown) {
-      console.error(error);
-      if (isPrismaException(error)) {
-        return res.status(500).json({
-          error: "Internal Server Error",
-        });
-      }
-      if (error instanceof z.ZodError) {
-        return res.status(400).json({
-          message: "Invalid request data",
-          error: error.errors,
-        });
-      }
-      const errorMessage =
-        error instanceof Error ? error.message : "An unknown error occurred";
-      res.status(500).json({
-        message: "Invalid request data",
-        error: errorMessage,
-      });
-    }
-  } else {
-    return res.status(405).json({ message: "Method not allowed" });
-  }
-}
+      return { message: "Score deleted successfully" };
+    },
+  }),
+});

web/src/sentry.server.config.ts

@@ -12,6 +12,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
     tracesSampleRate: env.LANGFUSE_TRACING_SAMPLE_RATE,
+
     profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations

web/src/server/api/routers/generations/db/getAllGenerationsSqlQuery.ts

@@ -18,16 +18,17 @@ export type FullObservations = Array<
   AdditionalObservationFields & ObservationView
 >;
 
-export type IOOmittedObservations = Array<
-  Omit<ObservationView, "input" | "output"> & AdditionalObservationFields
+export type IOAndMetadataOmittedObservations = Array<
+  Omit<ObservationView, "input" | "output" | "metadata"> &
+    AdditionalObservationFields
 >;
 
 export async function getAllGenerations({
   input,
-  selectIO,
+  selectIOAndMetadata,
 }: {
   input: GetAllGenerationsInput;
-  selectIO: boolean;
+  selectIOAndMetadata: boolean;
 }) {
   const searchCondition = input.searchQuery
     ? Prisma.sql`AND (
@@ -96,8 +97,7 @@ export async function getAllGenerations({
         o."modelParameters",
         o.start_time as "startTime",
         o.end_time as "endTime",
-        ${selectIO ? Prisma.sql`o.input, o.output,` : Prisma.empty} 
-        o.metadata,
+        ${selectIOAndMetadata ? Prisma.sql`o.input, o.output, o.metadata,` : Prisma.empty} 
         o.trace_id as "traceId",
         t.name as "traceName",
         o.completion_start_time as "completionStartTime",
@@ -134,9 +134,10 @@ export async function getAllGenerations({
       LIMIT ${input.limit} OFFSET ${input.page * input.limit}
     `;
 
-  const generations: FullObservations | IOOmittedObservations = selectIO
-    ? await prisma.$queryRaw(query)
-    : await prisma.$queryRaw(query);
+  const generations: FullObservations | IOAndMetadataOmittedObservations =
+    selectIOAndMetadata
+      ? ((await prisma.$queryRaw(query)) as FullObservations)
+      : ((await prisma.$queryRaw(query)) as IOAndMetadataOmittedObservations);
 
   const scores = await prisma.score.findMany({
     where: {

web/src/server/api/routers/generations/exportQuery/index.ts

@@ -52,7 +52,7 @@ export const generationsExportQuery = protectedProjectProcedure
             page: offset / pageSize,
             limit: pageSize,
           },
-          selectIO: true, // selecting input/output data
+          selectIOAndMetadata: true, // selecting input/output and metadata
         });
         return generations as unknown as FullObservations;
       },

web/src/server/api/routers/generations/getAllQuery.ts

@@ -17,7 +17,7 @@ const getAllGenerationsInput = GenerationTableOptions.extend({
 
 export type ScoreSimplified = {
   name: string;
-  value: number;
+  value?: number | null;
   dataType: ScoreDataType;
   stringValue?: string | null;
   comment?: string | null;
@@ -37,7 +37,7 @@ export const getAllQuery = protectedProjectProcedure
   .input(getAllGenerationsInput)
   .query(async ({ input, ctx }) => {
     const { generations, datetimeFilter, filterCondition, searchCondition } =
-      await getAllGenerations({ input, selectIO: false });
+      await getAllGenerations({ input, selectIOAndMetadata: false });
 
     const totalGenerations = await ctx.prisma.$queryRaw<
       Array<{ count: bigint }>
@@ -61,6 +61,7 @@ export const getAllQuery = protectedProjectProcedure
                 scores."project_id" = ${input.projectId}
                 AND scores."trace_id" = t.id
                 AND scores."observation_id" = o.id
+                AND scores.value IS NOT NULL
             GROUP BY
                 name
         ) tmp

web/src/server/api/routers/models.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-import { ModelUsageUnit, Prisma } from "@langfuse/shared";
+import { ModelUsageUnit } from "@langfuse/shared";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import {
   createTRPCRouter,
@@ -9,6 +9,7 @@ import {
 import { paginationZod } from "@langfuse/shared";
 import { TRPCError } from "@trpc/server";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";
 
 const ModelAllOptions = z.object({
   projectId: z.string(),
@@ -119,11 +120,12 @@ export const modelRouter = createTRPCRouter({
 
       // Check if regex is valid POSIX regex
       // Use DB to check, because JS regex is not POSIX compliant
-      try {
-        await ctx.prisma.$queryRaw(
-          Prisma.sql`SELECT 'test_string' ~ ${input.matchPattern}`,
-        );
-      } catch (error) {
+
+      const isValidRegex = await isValidPostgresRegex(
+        input.matchPattern,
+        ctx.prisma,
+      );
+      if (!isValidRegex) {
         throw new TRPCError({
           code: "BAD_REQUEST",
           message: "Invalid regex, needs to be Postgres syntax",

web/src/server/api/routers/scoreConfigs.ts

@@ -7,7 +7,11 @@ import { optionalPaginationZod } from "@langfuse/shared";
 
 import { ScoreDataType } from "@langfuse/shared/src/db";
 import { z } from "zod";
-import { categoriesList } from "@langfuse/shared";
+import {
+  filterAndValidateDbScoreConfigList,
+  Category,
+  validateDbScoreConfig,
+} from "@/src/features/public-api/types/score-configs";
 
 const ScoreConfigAllInput = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -27,32 +31,28 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:read",
       });
 
-      try {
-        const configs = await ctx.prisma.scoreConfig.findMany({
-          where: {
-            projectId: input.projectId,
-          },
-          ...(input.limit !== undefined && input.page !== undefined
-            ? { take: input.limit, skip: input.page * input.limit }
-            : undefined),
-          orderBy: {
-            createdAt: "desc",
-          },
-        });
+      const configs = await ctx.prisma.scoreConfig.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        ...(input.limit !== undefined && input.page !== undefined
+          ? { take: input.limit, skip: input.page * input.limit }
+          : undefined),
+        orderBy: {
+          createdAt: "desc",
+        },
+      });
 
-        const configsCount = await ctx.prisma.scoreConfig.count({
-          where: {
-            projectId: input.projectId,
-          },
-        });
+      const configsCount = await ctx.prisma.scoreConfig.count({
+        where: {
+          projectId: input.projectId,
+        },
+      });
 
-        return {
-          configs,
-          totalCount: configsCount,
-        };
-      } catch (error) {
-        console.log(error);
-      }
+      return {
+        configs: filterAndValidateDbScoreConfigList(configs),
+        totalCount: configsCount,
+      };
     }),
   create: protectedProjectProcedure
     .input(
@@ -62,7 +62,7 @@ export const scoreConfigsRouter = createTRPCRouter({
         dataType: z.nativeEnum(ScoreDataType),
         minValue: z.number().optional(),
         maxValue: z.number().optional(),
-        categories: categoriesList.optional(),
+        categories: z.array(Category).optional(),
         description: z.string().optional(),
       }),
     )
@@ -73,30 +73,13 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:CUD",
       });
 
-      try {
-        const existingConfig = await ctx.prisma.scoreConfig.findFirst({
-          where: {
-            projectId: input.projectId,
-            name: input.name,
-            dataType: input.dataType,
-          },
-        });
+      const config = await ctx.prisma.scoreConfig.create({
+        data: {
+          ...input,
+        },
+      });
 
-        if (existingConfig)
-          throw new Error(
-            "Score config with this name and data type already exists",
-          );
-
-        const config = await ctx.prisma.scoreConfig.create({
-          data: {
-            ...input,
-          },
-        });
-
-        return config;
-      } catch (error) {
-        console.log(error);
-      }
+      return validateDbScoreConfig(config);
     }),
   update: protectedProjectProcedure
     .input(
@@ -113,20 +96,16 @@ export const scoreConfigsRouter = createTRPCRouter({
         scope: "scoreConfigs:CUD",
       });
 
-      try {
-        const config = await ctx.prisma.scoreConfig.update({
-          where: {
-            id: input.id,
-            projectId: input.projectId,
-          },
-          data: {
-            isArchived: input.isArchived,
-          },
-        });
+      const config = await ctx.prisma.scoreConfig.update({
+        where: {
+          id: input.id,
+          projectId: input.projectId,
+        },
+        data: {
+          isArchived: input.isArchived,
+        },
+      });
 
-        return config;
-      } catch (error) {
-        console.log(error);
-      }
+      return validateDbScoreConfig(config);
     }),
 });

web/src/server/api/routers/scores.ts

@@ -6,8 +6,12 @@ import {
 } from "@/src/server/api/trpc";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type ProjectRole, Prisma, type Score } from "@langfuse/shared/src/db";
-import { paginationZod } from "@langfuse/shared";
-import { ScoreDataType, singleFilter } from "@langfuse/shared";
+import {
+  CreateAnnotationScoreData,
+  UpdateAnnotationScoreData,
+  paginationZod,
+} from "@langfuse/shared";
+import { singleFilter } from "@langfuse/shared";
 import {
   tableColumnsToSqlFilterAndPrefix,
   orderByToPrismaSql,
@@ -18,6 +22,7 @@ import {
 } from "@/src/server/api/definitions/scoresTable";
 import { orderBy } from "@langfuse/shared";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { validateDbScore } from "@/src/features/public-api/types/scores";
 
 const ScoreFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -131,19 +136,7 @@ export const scoresRouter = createTRPCRouter({
       return res;
     }),
   createAnnotationScore: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        traceId: z.string(),
-        observationId: z.string().optional(),
-        name: z.string(),
-        value: z.number(),
-        stringValue: z.string().optional(),
-        comment: z.string().optional().nullable(),
-        configId: z.string().optional(),
-        dataType: z.nativeEnum(ScoreDataType),
-      }),
-    )
+    .input(CreateAnnotationScoreData)
     .mutation(async ({ input, ctx }) => {
       throwIfNoAccess({
         session: ctx.session,
@@ -161,76 +154,63 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No trace with this id in this project.");
       }
 
-      try {
-        const existingScore = await ctx.prisma.score.findFirst({
+      const existingScore = await ctx.prisma.score.findFirst({
+        where: {
+          projectId: input.projectId,
+          traceId: input.traceId,
+          observationId: input.observationId,
+          source: "ANNOTATION",
+          configId: input.configId,
+        },
+      });
+
+      if (existingScore) {
+        const updatedScore = await ctx.prisma.score.update({
           where: {
+            id: existingScore.id,
             projectId: input.projectId,
-            traceId: input.traceId,
-            observationId: input.observationId,
-            source: "ANNOTATION",
-            configId: input.configId,
           },
-        });
-
-        if (existingScore) {
-          return ctx.prisma.score.update({
-            where: {
-              id: existingScore.id,
-              projectId: input.projectId,
-            },
-            data: {
-              value: input.value,
-              stringValue: input.stringValue,
-              comment: input.comment,
-              authorUserId: ctx.session.user.id,
-            },
-          });
-        }
-
-        const score = await ctx.prisma.score.create({
           data: {
-            projectId: input.projectId,
-            traceId: input.traceId,
-            observationId: input.observationId,
             value: input.value,
             stringValue: input.stringValue,
-            dataType: input.dataType,
-            configId: input.configId,
-            name: input.name,
             comment: input.comment,
             authorUserId: ctx.session.user.id,
-            source: "ANNOTATION",
           },
         });
-        await auditLog({
-          projectId: input.projectId,
-          userId: ctx.session.user.id,
-          userProjectRole: ctx.session.user.projects.find(
-            (p) => p.id === input.projectId,
-          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-          resourceType: "score",
-          resourceId: score.id,
-          action: "create",
-          after: score,
-        });
-        return score;
-      } catch (error) {
-        console.log(error);
-        throw error;
+        return validateDbScore(updatedScore);
       }
+
+      const score = await ctx.prisma.score.create({
+        data: {
+          projectId: input.projectId,
+          traceId: input.traceId,
+          observationId: input.observationId,
+          value: input.value,
+          stringValue: input.stringValue,
+          dataType: input.dataType,
+          configId: input.configId,
+          name: input.name,
+          comment: input.comment,
+          authorUserId: ctx.session.user.id,
+          source: "ANNOTATION",
+        },
+      });
+
+      await auditLog({
+        projectId: input.projectId,
+        userId: ctx.session.user.id,
+        userProjectRole: ctx.session.user.projects.find(
+          (p) => p.id === input.projectId,
+        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+        resourceType: "score",
+        resourceId: score.id,
+        action: "create",
+        after: score,
+      });
+      return validateDbScore(score);
     }),
   updateAnnotationScore: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        id: z.string(),
-        value: z.number(),
-        stringValue: z.string().optional(),
-        comment: z.string().optional().nullable(),
-        configId: z.string().optional(),
-        dataType: z.nativeEnum(ScoreDataType),
-      }),
-    )
+    .input(UpdateAnnotationScoreData)
     .mutation(async ({ input, ctx }) => {
       throwIfNoAccess({
         session: ctx.session,
@@ -248,35 +228,31 @@ export const scoresRouter = createTRPCRouter({
         throw new Error("No annotation score with this id in this project.");
       }
 
-      try {
-        await auditLog({
-          projectId: input.projectId,
-          userId: ctx.session.user.id,
-          userProjectRole: ctx.session.user.projects.find(
-            (p) => p.id === input.projectId,
-          )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
-          resourceType: "score",
-          resourceId: score.id,
-          action: "update",
-          after: score,
-        });
+      await auditLog({
+        projectId: input.projectId,
+        userId: ctx.session.user.id,
+        userProjectRole: ctx.session.user.projects.find(
+          (p) => p.id === input.projectId,
+        )?.role as ProjectRole, // throwIfNoAccess ensures this is defined
+        resourceType: "score",
+        resourceId: score.id,
+        action: "update",
+        after: score,
+      });
 
-        return ctx.prisma.score.update({
-          where: {
-            id: score.id,
-            projectId: input.projectId,
-          },
-          data: {
-            value: input.value,
-            stringValue: input.stringValue,
-            comment: input.comment,
-            authorUserId: ctx.session.user.id,
-          },
-        });
-      } catch (error) {
-        console.log(error);
-        throw error;
-      }
+      const updatedScore = await ctx.prisma.score.update({
+        where: {
+          id: score.id,
+          projectId: input.projectId,
+        },
+        data: {
+          value: input.value,
+          stringValue: input.stringValue,
+          comment: input.comment,
+          authorUserId: ctx.session.user.id,
+        },
+      });
+      return validateDbScore(updatedScore);
     }),
   deleteAnnotationScore: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), id: z.string() }))
@@ -310,7 +286,7 @@ export const scoresRouter = createTRPCRouter({
         before: score,
       });
 
-      return ctx.prisma.score.delete({
+      return await ctx.prisma.score.delete({
         where: {
           id: score.id,
           projectId: input.projectId,

web/src/server/api/routers/sessions.ts

@@ -9,6 +9,7 @@ import {
   type SessionOptions,
   getSessionTableSQL,
 } from "@langfuse/shared";
+import * as Sentry from "@sentry/node";
 import { Prisma } from "@langfuse/shared/src/db";
 import { paginationZod } from "@langfuse/shared";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
@@ -16,6 +17,7 @@ import { TRPCError } from "@trpc/server";
 import { orderBy } from "@langfuse/shared";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 import type Decimal from "decimal.js";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 const SessionFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
@@ -136,6 +138,8 @@ export const sessionRouter = createTRPCRouter({
           },
         });
 
+        const validatedScores = filterAndValidateDbScoreList(scores);
+
         const totalCostQuery = Prisma.sql`
         SELECT
           SUM(COALESCE(o."calculated_total_cost", 0)) AS "totalCost"
@@ -155,7 +159,7 @@ export const sessionRouter = createTRPCRouter({
           ...session,
           traces: session.traces.map((t) => ({
             ...t,
-            scores: scores.filter((s) => s.traceId === t.id),
+            scores: validatedScores.filter((s) => s.traceId === t.id),
           })),
           totalCost: costData?.totalCost ?? 0,
           users: [

web/src/server/api/routers/traces.ts

@@ -25,13 +25,13 @@ import { orderByToPrismaSql } from "@langfuse/shared";
 import { instrumentAsync } from "@/src/utils/instrumentation";
 import type Decimal from "decimal.js";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { filterAndValidateDbScoreList } from "@/src/features/public-api/types/scores";
 
 const TraceFilterOptions = z.object({
   projectId: z.string(), // Required for protectedProjectProcedure
   searchQuery: z.string().nullable(),
   filter: z.array(singleFilter).nullable(),
   orderBy: orderBy,
-  returnIO: z.boolean().default(true),
   ...paginationZod,
 });
 
@@ -46,7 +46,6 @@ export const traceRouter = createTRPCRouter({
   all: protectedProjectProcedure
     .input(TraceFilterOptions)
     .query(async ({ input, ctx }) => {
-      const returnIO = input.returnIO;
       const filterCondition = tableColumnsToSqlFilterAndPrefix(
         input.filter ?? [],
         tracesTableCols,
@@ -82,7 +81,6 @@ export const traceRouter = createTRPCRouter({
       const tracesQuery = createTracesQuery(
         Prisma.sql`t.*,
           t."user_id" AS "userId",
-          t."metadata" AS "metadata",
           t.session_id AS "sessionId",
           t."bookmarked" AS "bookmarked",
           COALESCE(tm."promptTokens", 0)::int AS "promptTokens",
@@ -150,24 +148,17 @@ export const traceRouter = createTRPCRouter({
           },
         },
       });
+      const validatedScores = filterAndValidateDbScoreList(scores);
 
       const totalTraceCount = totalTraces[0]?.count;
       return {
-        traces: traces.map((trace) => {
-          const filteredScores = scores.filter((s) => s.traceId === trace.id);
-
-          const { input, output, ...rest } = trace;
-          if (returnIO) {
-            return { ...rest, input, output, scores: filteredScores };
-          } else {
-            return {
-              ...rest,
-              input: undefined,
-              output: undefined,
-              scores: filteredScores,
-            };
-          }
-        }),
+        traces: traces.map(
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          ({ input, output, metadata, ...trace }) => ({
+            ...trace,
+            scores: validatedScores.filter((s) => s.traceId === trace.id),
+          }),
+        ),
         totalCount: totalTraceCount ? Number(totalTraceCount) : undefined,
       };
     }),
@@ -312,6 +303,7 @@ export const traceRouter = createTRPCRouter({
           projectId: trace.projectId,
         },
       });
+      const validatedScores = filterAndValidateDbScoreList(scores);
 
       const obsStartTimes = observations
         .map((o) => o.startTime)
@@ -333,7 +325,7 @@ export const traceRouter = createTRPCRouter({
 
       return {
         ...trace,
-        scores,
+        scores: validatedScores,
         latency: latencyMs !== undefined ? latencyMs / 1000 : undefined,
         observations: observations as ObservationReturnType[],
       };

web/src/server/api/services/EventProcessor.ts

@@ -12,8 +12,10 @@ import {
   type legacyObservationUpdateEvent,
   type sdkLogEvent,
   type traceEvent,
+  LangfuseNotFoundError,
+  InvalidRequestError,
 } from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
+import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
 import { ResourceNotFoundError } from "@/src/utils/exceptions";
 import { mergeJson } from "@langfuse/shared";
 import {
@@ -30,6 +32,14 @@ import { sendToBetterstack } from "@/src/features/betterstack/server/betterstack
 import { ForbiddenError } from "@langfuse/shared";
 import { instrument } from "@/src/utils/instrumentation";
 import Decimal from "decimal.js";
+import {
+  ScoreBodyWithoutConfig,
+  ScorePropsAgainstConfig,
+} from "@/src/features/public-api/types/scores";
+import {
+  validateDbScoreConfigSafe,
+  type ValidatedScoreConfig,
+} from "@/src/features/public-api/types/score-configs";
 
 export interface EventProcessor {
   process(
@@ -606,6 +616,145 @@ export class ScoreProcessor implements EventProcessor {
     this.event = event;
   }
 
+  static inferDataType(value: string | number): ScoreDataType {
+    return typeof value === "number"
+      ? ScoreDataType.NUMERIC
+      : ScoreDataType.CATEGORICAL;
+  }
+
+  static mapStringValueToNumericValue(
+    config: ValidatedScoreConfig,
+    label: string,
+  ): number | null {
+    return (
+      config.categories?.find((category) => category.label === label)?.value ??
+      null
+    );
+  }
+
+  static inflateScoreBody(
+    body: any,
+    id: string,
+    projectId: string,
+    config?: ValidatedScoreConfig,
+  ): Score {
+    const relevantDataType = config?.dataType ?? body.dataType;
+    const scoreProps = { ...body, id, projectId, source: "API" };
+
+    if (typeof body.value === "number") {
+      if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {
+        return {
+          ...scoreProps,
+          value: body.value,
+          stringValue: body.value === 1 ? "True" : "False",
+          dataType: ScoreDataType.BOOLEAN,
+        };
+      }
+
+      return {
+        ...scoreProps,
+        value: body.value,
+        dataType: ScoreDataType.NUMERIC,
+      };
+    }
+    return {
+      ...scoreProps,
+      value: config
+        ? ScoreProcessor.mapStringValueToNumericValue(config, body.value)
+        : null,
+      stringValue: body.value,
+      dataType: ScoreDataType.CATEGORICAL,
+    };
+  }
+
+  validateConfigAgainstBody(body: any, config: ValidatedScoreConfig): void {
+    const { maxValue, minValue, categories, dataType: configDataType } = config;
+    if (body.dataType && body.dataType !== configDataType) {
+      throw new InvalidRequestError(
+        `Data type mismatch based on config: expected ${configDataType}, got ${body.dataType}`,
+      );
+    }
+
+    if (config.isArchived) {
+      throw new InvalidRequestError(
+        "Config is archived and cannot be used to create new scores. Please restore the config first.",
+      );
+    }
+
+    if (config.name !== body.name) {
+      throw new InvalidRequestError(
+        `Name mismatch based on config: expected ${config.name}, got ${body.name}`,
+      );
+    }
+
+    const relevantDataType = configDataType ?? body.dataType;
+
+    const dataTypeValidation = ScoreBodyWithoutConfig.safeParse({
+      ...body,
+      dataType: relevantDataType,
+    });
+    if (!dataTypeValidation.success) {
+      throw new InvalidRequestError(
+        `Ingested score body not valid against provided config data type.`,
+      );
+    }
+
+    const rangeValidation = ScorePropsAgainstConfig.safeParse({
+      value: body.value,
+      dataType: relevantDataType,
+      ...(maxValue !== null && maxValue !== undefined && { maxValue }),
+      ...(minValue !== null && minValue !== undefined && { minValue }),
+      ...(categories && { categories }),
+    });
+    if (!rangeValidation.success) {
+      const errorDetails = rangeValidation.error.errors
+        .map((error) => `${error.path.join(".")} - ${error.message}`)
+        .join(", ");
+      throw new InvalidRequestError(
+        `Ingested score body not valid against provided config: ${errorDetails}`,
+      );
+    }
+  }
+
+  async validateAndInflate(
+    body: any,
+    id: string,
+    projectId: string,
+  ): Promise<Score> {
+    if (body.configId) {
+      const config = await prisma.scoreConfig.findFirst({
+        where: {
+          projectId,
+          id: body.configId,
+        },
+      });
+
+      if (!config || !validateDbScoreConfigSafe(config).success)
+        throw new LangfuseNotFoundError(
+          "The configId you provided does not match a valid config in this project",
+        );
+
+      this.validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+      return ScoreProcessor.inflateScoreBody(
+        body,
+        id,
+        projectId,
+        config as ValidatedScoreConfig,
+      );
+    } else {
+      const validation = ScoreBodyWithoutConfig.safeParse({
+        ...body,
+        dataType: body.dataType ?? ScoreProcessor.inferDataType(body.value),
+      });
+      if (!validation.success) {
+        throw new InvalidRequestError(
+          `Ingested score value type not valid against provided data type. Provide numeric values for numeric and boolean scores, and string values for categorical scores.`,
+        );
+      }
+      return ScoreProcessor.inflateScoreBody(body, id, projectId);
+    }
+  }
+
   async process(
     apiScope: ApiAccessScope,
   ): Promise<Trace | Observation | Score> {
@@ -632,6 +781,12 @@ export class ScoreProcessor implements EventProcessor {
       );
     }
 
+    const validatedScore = await this.validateAndInflate(
+      body,
+      id,
+      apiScope.projectId,
+    );
+
     return await prisma.score.upsert({
       where: {
         id_projectId: {
@@ -640,24 +795,10 @@ export class ScoreProcessor implements EventProcessor {
         },
       },
       create: {
-        id,
-        projectId: apiScope.projectId,
-        traceId: body.traceId,
-        observationId: body.observationId ?? undefined,
-        timestamp: new Date(),
-        value: body.value,
-        name: body.name,
-        comment: body.comment,
-        source: "API",
+        ...validatedScore,
       },
       update: {
-        traceId: body.traceId,
-        observationId: body.observationId ?? undefined,
-        timestamp: new Date(),
-        value: body.value,
-        name: body.name,
-        comment: body.comment,
-        source: "API",
+        ...validatedScore,
       },
     });
   }

web/src/server/api/services/tableDefinitions.ts

@@ -126,6 +126,7 @@ const tracesObservationsColumns: ColumnDefinition[] = [
   observationsProjectId,
   duration,
   totalTokens,
+  calculatedTotalCost,
   model,
   traceTimestamp,
   traceUser,

web/src/utils/dates.ts

@@ -5,6 +5,15 @@ export const utcDateOffsetByDays = (days: number) => {
   return date;
 };
 
+export const utcDate = (localDateTime: Date) =>
+  new Date(
+    Date.UTC(
+      localDateTime.getFullYear(),
+      localDateTime.getMonth(),
+      localDateTime.getDate(),
+    ),
+  );
+
 export const setBeginningOfDay = (date: Date) => {
   date.setHours(0, 0, 0, 0);
   return date;

web/src/utils/typeChecks.ts

@@ -0,0 +1,2 @@
+export const isPresent = <T>(value: T | null | undefined): value is T =>
+  value !== null && value !== undefined && value !== "";

worker/Dockerfile

@@ -1,10 +1,11 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} alpine AS base
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
 RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
@@ -12,7 +13,7 @@ RUN corepack enable
 RUN corepack prepare pnpm@8.15.5 --activate
 
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS pruner
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS pruner
 
 WORKDIR /app
 
@@ -21,7 +22,7 @@ RUN turbo prune --scope=worker --docker
 
 
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS builder
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS builder
 
 WORKDIR /app
 
@@ -57,8 +58,10 @@ RUN addgroup --system --gid 1001 expressjs
 RUN adduser --system --uid 1001 expressjs
 USER expressjs
 COPY --from=builder --chown=expressjs:expressjs /app .
+COPY --chown=expressjs:expressjs ./worker/newrelic.js ./newrelic.js
 
 EXPOSE 3030
 ENV PORT=3030
 
-CMD ["node", "worker/dist/index.js"]
+CMD ["node", "--experimental-loader=newrelic/esm-loader.mjs", "worker/dist/index.js"]
+

worker/newrelic.js

@@ -0,0 +1,26 @@
+"use strict";
+
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: ["worker"],
+  /**
+   * Your New Relic license key.
+   */
+  license_key: process.env.NEW_RELIC_API_KEY,
+  logging: {
+    enabled: false,
+  },
+  application_logging: {
+    forwarding: {
+      enabled: false,
+    },
+  },
+};

worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "worker",
-  "version": "2.58.0",
+  "version": "2.60.2",
   "description": "",
   "license": "MIT",
   "private": true,
@@ -8,9 +8,9 @@
     "node": ">=20"
   },
   "scripts": {
-    "test": "dotenv -e ../.env -- vitest run --reporter=basic ",
+    "test": "dotenv -e ../.env -- vitest run --reporter=basic --pool=forks",
     "coverage": "vitest run --coverage",
-    "start": "dotenv -e ../.env -- node dist/index.js",
+    "start": "dotenv -e ../.env -- node --experimental-loader=newrelic/esm-loader.mjs dist/index.js",
     "build": "tsc",
     "dev": "dotenv -e ../.env -- nodemon src/index.ts",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
@@ -35,6 +35,7 @@
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
     "lodash": "^4.17.21",
+    "newrelic": "^11.22.0",
     "pg": "^8.11.5",
     "pino": "^9.2.0",
     "pino-http": "^9.0.0",
@@ -50,6 +51,7 @@
     "@types/express": "^4.17.21",
     "@types/express-serve-static-core": "^4.19.3",
     "@types/lodash": "^4.17.5",
+    "@types/newrelic": "^9.14.4",
     "@types/node": "^20.11.19",
     "@types/pg": "^8.11.6",
     "@typescript-eslint/parser": "^7.12.0",

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.58.0";
+export const VERSION = "v2.60.2";

worker/src/index.ts

@@ -1,3 +1,4 @@
+import "newrelic";
 import app from "./app";
 import { env } from "./env";
 import logger from "./logger";

worker/src/instrumentation.ts

@@ -1,37 +1,12 @@
-import { nodeProfilingIntegration } from "@sentry/profiling-node";
-import { env } from "./env";
-import * as Sentry from "@sentry/node";
+import newrelic from "newrelic";
 
-Sentry.init({
-  dsn: String(env.SENTRY_DSN),
-  integrations: [
-    Sentry.httpIntegration(),
-    Sentry.expressIntegration(),
-    nodeProfilingIntegration(),
-    Sentry.redisIntegration(),
-    Sentry.prismaIntegration(),
-  ],
-
-  // Add Tracing by setting tracesSampleRate
-  // We recommend adjusting this value in production
-  tracesSampleRate: 0.5,
-
-  // Set sampling rate for profiling
-  // This is relative to tracesSampleRate
-  profilesSampleRate: 0.1,
-});
-
-type CallbackAsyncFn<T> = (span?: Sentry.Span) => Promise<T>;
+type CallbackAsyncFn<T> = () => Promise<T>;
 
 export async function instrumentAsync<T>(
   ctx: { name: string },
   callback: CallbackAsyncFn<T>
 ): Promise<T> {
-  if (env.SENTRY_DSN) {
-    return Sentry.startSpan(ctx, async (span) => {
-      return callback(span);
-    });
-  } else {
+  return newrelic.startSegment(ctx.name, true, async function () {
     return callback();
-  }
+  });
 }

worker/src/queues/batchExportQueue.ts

@@ -24,43 +24,36 @@ export const batchExportJobExecutor = redis
   ? new Worker<TQueueJobTypes[QueueName.BatchExport]>(
       QueueName.BatchExport,
       async (job: Job<TQueueJobTypes[QueueName.BatchExport]>) => {
-        return instrumentAsync(
-          { name: "batchExportJobExecutor" },
-          async (span) => {
-            try {
-              logger.info("Executing Batch Export Job", job.data.payload);
-              await handleBatchExportJob(job.data.payload);
+        return instrumentAsync({ name: "batchExportJobExecutor" }, async () => {
+          try {
+            logger.info("Executing Batch Export Job", job.data.payload);
+            await handleBatchExportJob(job.data.payload);
 
-              logger.info("Finished Batch Export Job", job.data.payload);
+            logger.info("Finished Batch Export Job", job.data.payload);
 
-              return true;
-            } catch (e) {
-              const displayError =
-                e instanceof BaseError
-                  ? e.message
-                  : "An internal error occurred";
+            return true;
+          } catch (e) {
+            const displayError =
+              e instanceof BaseError ? e.message : "An internal error occurred";
 
-              await kyselyPrisma.$kysely
-                .updateTable("batch_exports")
-                .set("status", BatchExportStatus.FAILED)
-                .set("finished_at", new Date())
-                .set("log", displayError)
-                .where("id", "=", job.data.payload.batchExportId)
-                .where("project_id", "=", job.data.payload.projectId)
-                .execute();
+            await kyselyPrisma.$kysely
+              .updateTable("batch_exports")
+              .set("status", BatchExportStatus.FAILED)
+              .set("finished_at", new Date())
+              .set("log", displayError)
+              .where("id", "=", job.data.payload.batchExportId)
+              .where("project_id", "=", job.data.payload.projectId)
+              .execute();
 
-              logger.error(
-                e,
-                `Failed Batch Export job for id ${job.data.payload.batchExportId} ${e}`
-              );
-              Sentry.captureException(e);
+            logger.error(
+              e,
+              `Failed Batch Export job for id ${job.data.payload.batchExportId} ${e}`
+            );
+            Sentry.captureException(e);
 
-              throw e;
-            } finally {
-              span?.end();
-            }
+            throw e;
           }
-        );
+        });
       },
       {
         connection: redis,

worker/src/queues/evalQueue.ts

@@ -26,7 +26,7 @@ export const evalJobCreator = redis
   ? new Worker<TQueueJobTypes[QueueName.TraceUpsert]>(
       QueueName.TraceUpsert,
       async (job: Job<TQueueJobTypes[QueueName.TraceUpsert]>) => {
-        return instrumentAsync({ name: "evalJobCreator" }, async (span) => {
+        return instrumentAsync({ name: "evalJobCreator" }, async () => {
           try {
             await createEvalJobs({ event: job.data.payload });
             return true;
@@ -37,8 +37,6 @@ export const evalJobCreator = redis
             );
             Sentry.captureException(e);
             throw e;
-          } finally {
-            span?.end();
           }
         });
       },
@@ -58,7 +56,7 @@ export const evalJobExecutor = redis
   ? new Worker<TQueueJobTypes[QueueName.EvaluationExecution]>(
       QueueName.EvaluationExecution,
       async (job: Job<TQueueJobTypes[QueueName.EvaluationExecution]>) => {
-        return instrumentAsync({ name: "evalJobExecutor" }, async (span) => {
+        return instrumentAsync({ name: "evalJobExecutor" }, async () => {
           try {
             logger.info("Executing Evaluation Execution Job", job.data);
             await evaluate({ event: job.data.payload });
@@ -92,8 +90,6 @@ export const evalJobExecutor = redis
             }
 
             throw e;
-          } finally {
-            span?.end();
           }
         });
       },