chore: release v2.65.0

* chore: enhance security measures in image verification
* style: add specific title for http image render error

.devcontainer/devcontainer.json

@@ -1,14 +1,14 @@
 {
-	"name": "langfuse dev",
-    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
-    "features": {
-        "ghcr.io/devcontainers/features/docker-in-docker:2": {
-        "version": "latest",
-        "moby": true
-        }
-    },
-    "forwardPorts": [3000],
+  "name": "langfuse dev",
+  "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "latest",
+      "moby": true
+    }
+  },
+  "forwardPorts": [3000],
 
-    "onCreateCommand": "npm install -g pnpm@8.15.5",
-    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
 }

.dockerignore

@@ -3,6 +3,6 @@ Dockerfile
 node_modules
 npm-debug.log
 README.md
-.next
+**/.next
 .git
 **/node_modules

.env.prod.example

@@ -134,13 +134,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Used to determine the Sentry sample rate
 # LANGFUSE_TRACING_SAMPLE_RATE=
 
+
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Betterstack
-# LANGFUSE_TEAM_BETTERSTACK_TOKEN=
-
 # Demo project that users can use to try the platform
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
@@ -153,4 +151,5 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Admin API
 # ADMIN_API_KEY=
 
+
 ### END Langfuse Cloud Config

.github/workflows/pipeline.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -78,7 +78,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -133,7 +133,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -173,7 +173,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -236,7 +236,7 @@ jobs:
     steps:
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -292,7 +292,7 @@ jobs:
           labels: ${{ steps.meta-web.outputs.labels }}
           platforms: |
             linux/amd64
-            linux/arm64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
@@ -319,4 +319,4 @@ jobs:
           labels: ${{ steps.meta-worker.outputs.labels }}
           platforms: |
             linux/amd64
-            linux/arm64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}

CONTRIBUTING.md

@@ -53,9 +53,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
-> [!NOTE]
-> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
+**Langfuse v2**
 
 ```mermaid
 flowchart TB
@@ -93,15 +91,66 @@ flowchart TB
         I --- Prisma
     end
 
-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker_API[Public HTTP API]
+    Prisma --- DB
+    JS --- API
+    Python --- API
+```
+
+**Langfuse v3 (work in progress, not released yet)**
+
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
+
+```mermaid
+flowchart TB
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
     end
 
-    API --> Worker_API
-    Worker_API --- DB
-    Worker_API --- Redis
+    DB[Postgres Database]
+    Redis[Redis Cache/Queue]
+    Clickhouse[Clickhouse Database]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        ORM
+        H --> G
+        H --> I
+        G --> I
+        G --- ORM
+        API --- ORM
+        I --- ORM
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker
+    end
+
+    Worker --- DB
+    Worker --- Redis
+    Worker --- Clickhouse
+
+    ORM --- DB
+    ORM --- Redis
+    ORM --- Clickhouse
 
-    Prisma --- DB
     JS --- API
     Python --- API
 ```
@@ -116,7 +165,7 @@ flowchart LR
    Browser ---|Web UI & TRPC API| App
    Integrations/SDKs ---|Public HTTP API| App
    subgraph i1["Application Network"]
-      App["Langfuse Application (Docker or Serverless)"]
+      App["Langfuse Application"]
    end
    subgraph i2["Database Network"]
       DB["Postgres Database"]
@@ -149,6 +198,7 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/
 Requirements
 
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
+- Pnpm v.9.5.0
 - Docker to run the database locally
 
 **Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".

README.md

@@ -159,7 +159,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

ee/package.json

@@ -15,23 +15,23 @@
     }
   },
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
     "next": "^14.2.3",
     "next-auth": "^4.24.7",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/node": "^20.11.29",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",

fern/apis/client/definition/score.yml

@@ -9,7 +9,6 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: Score
 
 types:
   CreateScoreRequest:
@@ -17,15 +16,119 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -42,11 +42,18 @@ types:
       htmlPath:
         type: string
         docs: Path of trace in Langfuse UI
+      latency:
+        type: double
+        docs: Latency of trace in seconds
       totalCost:
         type: double
         docs: Cost of trace in USD
-      observations: list<ObservationsView>
-      scores: list<Score>
+      observations:
+        type: list<ObservationsView>
+        docs: List of observations
+      scores:
+        type: list<Score>
+        docs: List of scores
   Session:
     properties:
       id: string
@@ -88,6 +95,7 @@ types:
       calculatedOutputCost: optional<double>
       calculatedTotalCost: optional<double>
       latency: optional<double>
+      timeToFirstToken: optional<double>
 
   Usage:
     docs: Standard interface for usage and cost
@@ -120,25 +128,81 @@ types:
       updatedAt: datetime
       projectId: string
       dataType: ScoreDataType
-      isArchived: boolean
-      minValue: optional<double>
-      maxValue: optional<double>
-      categories: optional<list<ConfigCategory>>
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
       description: optional<string>
   ConfigCategory:
     properties:
       value: double
       label: string
-  Score:
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
   Dataset:
     properties:
       id: string
@@ -185,6 +249,39 @@ types:
     extends: DatasetRun
     properties:
       datasetRunItems: list<DatasetRunItem>
+  Model:
+    docs: Model definition used for transforming usage into USD cost and/or tokenization.
+    properties:
+      id: string
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>
+      isLangfuseManaged:
+        type: boolean
 
   # Utilities
   ModelUsageUnit:
@@ -195,6 +292,7 @@ types:
       - MILLISECONDS
       - SECONDS
       - IMAGES
+      - REQUESTS
   ObservationLevel:
     enum:
       - DEBUG

fern/apis/server/definition/datasets.yml

@@ -22,7 +22,7 @@ service:
       response: PaginatedDatasets
     get:
       method: GET
-      docs: Get a dataset and its items
+      docs: Get a dataset
       path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
@@ -43,7 +43,7 @@ service:
       response: commons.DatasetRunWithItems
     getRuns:
       method: GET
-      docs: Get a dataset runs
+      docs: Get dataset runs
       path: /datasets/{datasetName}/runs
       path-parameters:
         datasetName: string

fern/apis/server/definition/ingestion.yml

@@ -185,9 +185,54 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/models.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      method: POST
+      docs: Create a model
+      path: /models
+      request: CreateModelRequest
+      response: commons.Model
+    list:
+      method: GET
+      docs: Get all models
+      path: /models
+      request:
+        name: GetModelsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedModels
+    get:
+      method: GET
+      docs: Get a model
+      path: /models/{id}
+      path-parameters:
+        id: string
+      response: commons.Model
+    delete:
+      method: DELETE
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      path: /models/{id}
+      path-parameters:
+        id: string
+
+types:
+  PaginatedModels:
+    properties:
+      data: list<commons.Model>
+      meta: pagination.MetaResponse
+  CreateModelRequest:
+    properties:
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: commons.ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total units. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>

fern/apis/server/definition/observations.yml

@@ -35,7 +35,10 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
+            docs: Retrieve only observations with a start_time or or after this datetime (ISO 8601).
+          toStartTime:
+            type: optional<datetime>
+            docs: Retrieve only observations with a start_time before this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/score-configs.yml

@@ -6,6 +6,12 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
     get:
       docs: Get all score configs
       method: GET
@@ -18,7 +24,7 @@ service:
             docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
       response: ScoreConfigs
     get-by-id:
       docs: Get a score config
@@ -34,3 +40,19 @@ types:
     properties:
       data: list<commons.ScoreConfig>
       meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -11,7 +11,7 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: commons.Score
+      response: CreateScoreResponse
     get:
       docs: Get a list of scores
       method: GET
@@ -25,11 +25,18 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this datetime (ISO 8601).
+            docs: Optional filter to only include scores created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include scores created before a certain datetime (ISO 8601)
           source:
             type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
@@ -42,6 +49,12 @@ service:
           scoreIds:
             type: optional<string>
             docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -66,9 +79,59 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  CreateScoreResponse:
+    properties:
+      id:
+        type: string
+        docs: The id of the created object in Langfuse
   Scores:
     properties:
       data: list<commons.Score>

fern/apis/server/definition/sessions.yml

@@ -6,6 +6,26 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    list:
+      docs: Get sessions.
+      method: GET
+      path: /sessions
+      request:
+        name: GetSessionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created before a certain datetime (ISO 8601)
+      response: PaginatedSessions
     get:
       docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
@@ -15,3 +35,8 @@ service:
           type: string
           docs: The unique id of a session
       response: commons.SessionWithTraces
+types:
+  PaginatedSessions:
+    properties:
+      data: list<commons.Session>
+      meta: pagination.MetaResponse

fern/apis/server/definition/trace.yml

@@ -33,7 +33,10 @@ service:
           sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this datetime (ISO 8601).
+            docs: Optional filter to only include traces with a trace.timestamp on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces with a trace.timestamp before a certain datetime (ISO 8601)
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

package.json

@@ -1,12 +1,11 @@
 {
   "name": "langfuse",
-  "version": "2.58.0",
+  "version": "2.65.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">=20",
-    "pnpm": "<9"
+    "node": "20"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -35,7 +34,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.2.5",
     "release-it": "^17.3.0",
-    "turbo": "^1.13.3"
+    "turbo": "^1.13.4"
   },
   "release-it": {
     "git": {
@@ -79,5 +78,6 @@
         "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
       }
     }
-  }
+  },
+  "packageManager": "pnpm@9.5.0"
 }

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.3",
+    "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -6,7 +6,7 @@
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "exports": {
     ".": {
@@ -61,12 +61,12 @@
     "next-auth": "^4.24.7",
     "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
-    "zod": "^3.22.4",
+    "zod": "^3.23.8",
     "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
     "@types/nodemailer": "^6.4.15",

packages/shared/prisma/generated/types.ts

@@ -286,7 +286,7 @@ export type ObservationView = {
     trace_id: string | null;
     project_id: string;
     type: ObservationType;
-    start_time: Generated<Timestamp>;
+    start_time: Timestamp;
     end_time: Timestamp | null;
     name: string | null;
     metadata: unknown | null;
@@ -294,7 +294,8 @@ export type ObservationView = {
     level: Generated<ObservationLevel>;
     status_message: string | null;
     version: string | null;
-    created_at: Generated<Timestamp>;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     model: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
@@ -357,7 +358,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
     author_user_id: string | null;
     comment: string | null;
@@ -438,6 +439,8 @@ export type TraceView = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     duration: number | null;
 };
 export type User = {

packages/shared/prisma/migrations/20240703214747_models_anthropic_aws_bedrock/migration.sql

@@ -0,0 +1,9 @@
+-- Add AWS Bedrock model names for Anthropic models
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku-20240307|anthropic\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet-20240229|anthropic\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus-20240229|anthropic\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-5-sonnet-20240620|anthropic\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$' WHERE "id" = 'clxt0n0m60000pumz1j5b7zsf';

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240704103901_scores_make_value_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "value" DROP NOT NULL;

packages/shared/prisma/migrations/20240705152639_traces_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,20 @@
+-- Drop and create to be able to change columns, otherwise new t.* cols cannot be added
+
+DROP VIEW IF EXISTS traces_view;
+CREATE VIEW traces_view AS
+WITH observations_metrics AS (
+    SELECT
+        trace_id,
+        project_id,
+        EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+    FROM
+        observations o
+    GROUP BY
+        project_id, trace_id
+)
+SELECT
+    t.*,
+    o.duration
+FROM
+    traces t
+    LEFT JOIN observations_metrics o ON t.id = o.trace_id and t.project_id = o.project_id

packages/shared/prisma/migrations/20240705154048_observation_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,89 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240704103900_observations_view_read_from_calculated and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240710114043_score_configs_drop_empty_categories_array_for_numeric_scores/migration.sql

@@ -0,0 +1,6 @@
+-- Migration script to update score_configs entries
+-- Set categories to NULL where data_type is 'NUMERIC' and categories is an empty array
+
+UPDATE score_configs
+SET categories = NULL
+WHERE data_type = 'NUMERIC' AND categories IS NOT NULL;

packages/shared/prisma/migrations/20240710114044_add_pricing_gpt4o_mini/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-mini
+  ('clyrjp56f0000t0mzapoocd7u', NULL, 'gpt-4o-mini', '(?i)^(gpt-4o-mini)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-mini-2024-07-18
+  ('clyrjpbe20000t0mzcbwc42rg', NULL, 'gpt-4o-mini-2024-07-18', '(?i)^(gpt-4o-mini-2024-07-18)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views", "relationJoins"]
+    previewFeatures = ["tracing", "views", "relationJoins", "metrics"]
 }
 
 datasource db {
@@ -272,6 +272,8 @@ view TraceView {
     input      Json?
     output     Json?
     sessionId  String?  @map("session_id")
+    createdAt  DateTime @map("created_at")
+    updatedAt  DateTime @map("updated_at")
 
     // calculated fields
     duration Float? @map("duration") // can be null if no observations in trace
@@ -299,9 +301,9 @@ model Observation {
     updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
-    model           String?
-    internalModel   String? @map("internal_model")
-    internalModelId String? @map("internal_model_id")
+    model           String? // user-provided model attribute
+    internalModel   String? @map("internal_model") // matched model.name that is matched at ingestion time, to be deprecated
+    internalModelId String? @map("internal_model_id") // matched model.id that is matched at ingestion time
 
     modelParameters  Json?
     input            Json?
@@ -352,7 +354,7 @@ view ObservationView {
     traceId             String?          @map("trace_id")
     projectId           String           @map("project_id")
     type                ObservationType
-    startTime           DateTime         @default(now()) @map("start_time")
+    startTime           DateTime         @map("start_time")
     endTime             DateTime?        @map("end_time")
     name                String?
     metadata            Json?
@@ -360,7 +362,8 @@ view ObservationView {
     level               ObservationLevel @default(DEFAULT)
     statusMessage       String?          @map("status_message")
     version             String?
-    createdAt           DateTime         @default(now()) @map("created_at")
+    createdAt           DateTime         @map("created_at")
+    updatedAt           DateTime         @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -410,14 +413,14 @@ model Score {
     projectId     String         @map("project_id")
     project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
-    value         Float
+    value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
     source        ScoreSource
     authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
-    stringValue   String?        @map("string_value")
+    stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
@@ -629,7 +632,7 @@ model Model {
     inputPrice      Decimal?  @map("input_price")
     outputPrice     Decimal?  @map("output_price")
     totalPrice      Decimal?  @map("total_price")
-    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, REQUESTS, or IMAGES
     tokenizerId     String?   @map("tokenizer_id")
     tokenizerConfig Json?     @map("tokenizer_config")
 

packages/shared/prisma/seed.ts

@@ -17,6 +17,11 @@ import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
 const options = {
   environment: { type: "string" },
 } as const;
@@ -459,7 +464,15 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
-  configIdsAndNames: Map<string, { name: string; id: string }[]>
+  configParams: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -518,13 +531,34 @@ function createObjects(
 
     traces.push(trace);
 
-    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const configArray = configParams.get(projectId) ?? [];
     const randomIndex = Math.floor(Math.random() * 3);
     const config =
       configArray.length >= randomIndex - 1 && configArray[randomIndex];
-    const { name: annotationScoreName, id: configId } = config || {
+    const {
+      name: annotationScoreName,
+      id: configId,
+      dataType,
+      categories,
+    } = config || {
       name: "manual-score",
       id: undefined,
+      dataType: ScoreDataType.NUMERIC,
+      categories: null,
+    };
+
+    const value = Math.floor(Math.random() * 2);
+    const scoreNumericAndStringValue = {
+      ...(dataType === ScoreDataType.NUMERIC && { value }),
+      ...(dataType === ScoreDataType.CATEGORICAL && {
+        value,
+        stringValue: categories?.find((category) => category.value === value)
+          ?.label,
+      }),
+      ...(dataType === ScoreDataType.BOOLEAN && {
+        value,
+        stringValue: value === 1 ? "True" : "False",
+      }),
     };
 
     const traceScores = [
@@ -533,12 +567,12 @@ function createObjects(
             {
               traceId: trace.id,
               name: annotationScoreName,
-              value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
-              dataType: ScoreDataType.NUMERIC,
+              dataType,
+              ...scoreNumericAndStringValue,
               ...(configId ? { configId } : {}),
             },
           ]
@@ -556,6 +590,20 @@ function createObjects(
             },
           ]
         : []),
+      ...(Math.random() < 0.8
+        ? [
+            {
+              traceId: trace.id,
+              name: "Completeness",
+              timestamp: traceTs,
+              source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.CATEGORICAL,
+              stringValue:
+                Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+            },
+          ]
+        : []),
     ];
 
     scores.push(...traceScores);
@@ -662,7 +710,7 @@ function createObjects(
                   },
                   {
                     role: "user",
-                    content: "How can i create a React component?",
+                    content: "How can i create a *React* component?",
                   },
                 ]
               : {
@@ -680,37 +728,8 @@ function createObjects(
                     },
                   ],
                 },
-          output: {
-            completion: `Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.
-
-              1.  **Functional Component**:
-              
-              A functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:
-              
-              
-              'import React from 'react';  function Greeting(props) {   return <h1>Hello, {props.name}</h1>; }  export default Greeting;'
-              
-              To use this component in another file, you can do:
-              
-              
-              'import Greeting from './Greeting';  function App() {   return (     <div>       <Greeting name="John" />     </div>   ); }  export default App;'
-              
-              2.  **Class Component**:
-              
-              You can also define components as classes in React. These have some additional features compared to functional components:
-              
-              
-              'import React, { Component } from 'react';  class Greeting extends Component {   render() {     return <h1>Hello, {this.props.name}</h1>;   } }  export default Greeting;'
-              
-              And here's how to use this component:
-              
-              
-              'import Greeting from './Greeting';  class App extends Component {   render() {     return (       <div>         <Greeting name="John" />       </div>     );   } }  export default App;'
-              
-              With the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.
-              
-              Remember to import React at the top of your file whenever you're creating a component, because JSX transpiles to 'React.createElement' calls under the hood.`,
-          },
+          output:
+            "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.\n\n**Image**\n\n![Languse Example Image](https://static.langfuse.com/langfuse-dev/langfuse-example-image.jpeg)\n\n1.  **Functional Component**:\n\nA functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:\n\n```javascript\nimport React from 'react';\nfunction Greeting(props) {\n  return <h1>Hello, {props.name}</h1>;\n}\nexport default Greeting;\n```\n\nTo use this component in another file, you can do:\n\n```javascript\nimport Greeting from './Greeting';\nfunction App() {\n  return (\n    <div>\n      <Greeting name=\"John\" />\n    </div>\n  );\n}\nexport default App;\n```\n\n2.  **Class Component**:\n\nYou can also define components as classes in React. These have some additional features compared to functional components:\n\n```javascript\nimport React, { Component } from 'react';\nclass Greeting extends Component {\n  render() {\n    return <h1>Hello, {this.props.name}</h1>;\n  }\n}\nexport default Greeting;\n```\n\nAnd here's how to use this component:\n\n```javascript\nimport Greeting from './Greeting';\nclass App extends Component {\n  render() {\n    return (\n      <div>\n        <Greeting name=\"John\" />\n      </div>\n    );\n  }\n}\nexport default App;\n```\n\nWith the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.\n\nRemember to import React at the top of your file whenever you're creating a component, because JSX transpiles to `React.createElement` calls under the hood.",
           model: model,
           internalModel: model,
           modelParameters: {
@@ -976,8 +995,15 @@ async function generatePrompts(project: Project) {
 }
 
 async function generateConfigsForProject(projects: Project[]) {
-  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
-    new Map();
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
 
   await Promise.all(
     projects.map(async (project) => {
@@ -989,7 +1015,12 @@ async function generateConfigsForProject(projects: Project[]) {
 }
 
 async function generateConfigs(project: Project) {
-  const configNameAndId: { name: string; id: string }[] = [];
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
 
   const configs = [
     {
@@ -1046,7 +1077,12 @@ async function generateConfigs(project: Project) {
         id: config.id,
       },
     });
-    configNameAndId.push({ name: config.name, id: config.id });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
   }
 
   return configNameAndId;

packages/shared/src/constants.ts

@@ -6,4 +6,5 @@ export enum ModelUsageUnit {
   Seconds = "SECONDS",
   Milliseconds = "MILLISECONDS",
   Images = "IMAGES",
+  Requests = "REQUESTS",
 }

packages/shared/src/errors/InternalServerError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InternalServerError extends BaseError {
+  constructor(description = "Internal Server Error") {
+    super("InternalServerError", 500, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -5,3 +5,4 @@ export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
+export { InternalServerError } from "./InternalServerError";

packages/shared/src/features/annotation/types.ts

@@ -1,15 +1,59 @@
 import z from "zod";
-import { ScoreConfig } from "../../db";
+import { type ScoreDataType } from "../../db";
 
-const configCategory = z.object({
-  label: z.string().min(1),
+const NUMERIC: ScoreDataType = "NUMERIC";
+const CATEGORICAL: ScoreDataType = "CATEGORICAL";
+const BOOLEAN: ScoreDataType = "BOOLEAN";
+
+export const availableDataTypes = [NUMERIC, CATEGORICAL, BOOLEAN] as const;
+
+const NumericData = z.object({
   value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
 });
 
-export const categoriesList = z.array(configCategory);
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
 
-export type ConfigCategory = z.infer<typeof configCategory>;
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
 
-export type CastedConfig = Omit<ScoreConfig, "categories"> & {
-  categories: ConfigCategory[] | null;
-};
+const CreateAnnotationScoreBase = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  traceId: z.string(),
+  configId: z.string().optional(),
+  observationId: z.string().optional(),
+  comment: z.string().optional().nullish(),
+});
+
+const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
+  id: z.string(),
+});
+
+/**
+ * CreateAnnotationScoreData is only used for annotation scores created via the UI.
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const CreateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  CreateAnnotationScoreBase.merge(NumericData),
+  CreateAnnotationScoreBase.merge(CategoricalData),
+  CreateAnnotationScoreBase.merge(BooleanData),
+]);
+
+/**
+ * UpdateAnnotationScoreData is only used for annotation scores updated via the UI
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  UpdateAnnotationScoreBase.merge(NumericData),
+  UpdateAnnotationScoreBase.merge(CategoricalData),
+  UpdateAnnotationScoreBase.merge(BooleanData),
+]);

packages/shared/src/features/evals/types.ts

@@ -80,6 +80,7 @@ export const availableEvalVariables = [
 export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
   [LLMAdapter.OpenAI]: [
     "gpt-4o",
+    "gpt-4o-mini",
     "gpt-4-turbo-preview",
     "gpt-3.5-turbo",
   ] as const,

packages/shared/src/features/ingestion/types.ts

@@ -28,7 +28,7 @@ const MixedUsage = z.object({
   totalCost: z.number().nullish(),
 });
 
-export const stringDate = z.string().datetime({ offset: true }).nullish();
+export const stringDateTime = z.string().datetime({ offset: true }).nullish();
 
 export const usage = MixedUsage.nullish()
   // transform mixed usage model input to new one
@@ -58,7 +58,7 @@ export const usage = MixedUsage.nullish()
 
 export const TraceBody = z.object({
   id: z.string().nullish(),
-  timestamp: stringDate,
+  timestamp: stringDateTime,
   name: z.string().nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),
@@ -75,7 +75,7 @@ export const TraceBody = z.object({
 export const OptionalObservationBody = z.object({
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
+  startTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -94,21 +94,27 @@ export const UpdateEventEvent = OptionalObservationBody.extend({
 });
 
 export const CreateSpanBody = CreateEventEvent.extend({
-  endTime: stringDate,
+  endTime: stringDateTime,
 });
 
 export const UpdateSpanBody = UpdateEventEvent.extend({
-  endTime: stringDate,
+  endTime: stringDateTime,
 });
 
 export const CreateGenerationBody = CreateSpanBody.extend({
-  completionStartTime: stringDate,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
       z.string(),
       z
-        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
+        .union([
+          z.string(),
+          z.number(),
+          z.boolean(),
+          z.array(z.string()),
+          z.record(z.string()),
+        ])
         .nullish()
     )
     .nullish(),
@@ -124,13 +130,19 @@ export const CreateGenerationBody = CreateSpanBody.extend({
 });
 
 export const UpdateGenerationBody = UpdateSpanBody.extend({
-  completionStartTime: stringDate,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
       z.string(),
       z
-        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
+        .union([
+          z.string(),
+          z.number(),
+          z.boolean(),
+          z.array(z.string()),
+          z.record(z.string()),
+        ])
         .nullish()
     )
     .nullish(),
@@ -145,22 +157,82 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
   return false;
 });
 
-export const ScoreBody = z.object({
+const BaseScoreBody = z.object({
   id: z.string().nullish(),
   name: NonEmptyString,
-  value: z.number(),
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
 });
 
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      })
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
   id: z.string().nullish(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -174,8 +246,8 @@ export const LegacySpanPatchSchema = z.object({
   spanId: z.string(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -188,9 +260,9 @@ export const LegacyGenerationsCreateSchema = z.object({
   id: z.string().nullish(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
@@ -212,9 +284,9 @@ export const LegacyGenerationPatchSchema = z.object({
   generationId: z.string(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
@@ -236,9 +308,9 @@ export const LegacyObservationBody = z.object({
   traceId: z.string().nullish(),
   type: z.enum(["GENERATION", "SPAN", "EVENT"]),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(

packages/shared/src/server/llm/types.ts

@@ -48,6 +48,8 @@ export const ZodModelConfig = z.object({
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-05-13",
+  "gpt-4o-mini",
+  "gpt-4o-mini-2024-07-18",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

turbo.json

@@ -17,7 +17,7 @@
     "db:push": {},
     "db:seed": {},
     "models:migrate": {
-      "dependsOn": ["db:generate"]
+      "dependsOn": ["db:generate", "@langfuse/shared#build"]
     },
     "db:seed:examples": {},
     "dev": {

web/Dockerfile

@@ -1,26 +1,24 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
-FROM --platform=${BUILDPLATFORM:-linux/amd64} alpine AS base
-RUN npm install turbo@^1.13.3 --global
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
+RUN npm install turbo@^1.13.4 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
-RUN corepack prepare pnpm@8.15.5 --activate
+RUN corepack prepare pnpm@9.5.0 --activate
 
-
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS pruner
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS pruner
 
 WORKDIR /app
 
 COPY . .
 RUN turbo prune --scope=web --docker
 
-
-FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS builder
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS builder
 
 WORKDIR /app
 
@@ -32,6 +30,7 @@ COPY --from=pruner /app/out/json/ .
 RUN pnpm install --frozen-lockfile
 
 ENV DOCKER_BUILD 1
+ENV NEXT_MANUAL_SIG_HANDLE true
 
 # pass public variables in build step
 ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
@@ -53,14 +52,13 @@ RUN rm -f ./web/src/middleware.ts
 # Learn more here: https://nextjs.org/telemetry
 # Uncomment the following line in case you want to disable telemetry during the build.
 ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_MANUAL_SIG_HANDLE true
 
 RUN turbo run build --filter=web
 
 # Production image, copy all the files and run next
 FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS runner
 
-RUN apk add --no-cache dumb-init
-
 WORKDIR /app
 
 ENV NODE_ENV production
@@ -68,13 +66,16 @@ ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1
 # Needed to re-enable validation of environment variables during runtime
 ENV DOCKER_BUILD 0
+# Set NEXT_MANUAL_SIG_HANDLE for runtime
+ENV NEXT_MANUAL_SIG_HANDLE true
+
+RUN apk add --no-cache dumb-init
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
 RUN npm install -g --no-package-lock --no-save prisma
 
-
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .
 
@@ -95,5 +96,8 @@ USER nextjs
 # Default port to 3000
 ENV PORT 3000
 
-# CMD ["node", "server.js"]
-CMD ["dumb-init", "--", "./web/entrypoint.sh"]
+
+ENTRYPOINT ["dumb-init", "--", "./web/entrypoint.sh"]
+
+# startup command
+CMD ["node", "./web/server.js"]

web/entrypoint.sh

@@ -35,5 +35,5 @@ if [ $status -ne 0 ]; then
     exit $status
 fi
 
-# Start server
-node web/server.js
+# Run the command passed to the docker image on start
+exec "$@"

web/next.config.mjs

@@ -97,6 +97,20 @@ const nextConfig = {
           },
         ]
         : []),
+      // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
+      {
+        source: "/generated/:path*",
+        headers: [
+          {
+            key: "Access-Control-Allow-Origin",
+            value: "*",
+          },
+          {
+            key: "Access-Control-Allow-Methods",
+            value: "GET",
+          },
+        ],
+      },
     ];
   },
 

web/package.json

@@ -1,19 +1,19 @@
 {
   "name": "web",
-  "version": "2.58.0",
+  "version": "2.65.0",
   "private": true,
   "license": "MIT",
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "scripts": {
     "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
-    "lint": "dotenv -e ../.env -- next lint",
+    "lint": "dotenv -e ../.env -- next lint --max-warnings 0",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
     "prettier": "prettier --write ./src *.{ts,js}",
     "clean": "rm -rf node_modules",
-    "start": "dotenv -e ../.env -- next start",
+    "start": "dotenv -e ../.env -- sh -c 'NEXT_MANUAL_SIG_HANDLE=true next start'",
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
@@ -87,6 +87,8 @@
     "decimal.js": "^10.4.3",
     "dompurify": "^3.1.5",
     "exponential-backoff": "^3.1.1",
+    "graphql": "^16.9.0",
+    "ip-address": "^9.0.5",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
     "langchain": "^0.2.6",
@@ -98,14 +100,19 @@
     "next-themes": "^0.3.0",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
+    "prexit": "^2.2.0",
     "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.51.5",
     "react-icons": "^5.2.1",
+    "react-markdown": "^9.0.1",
     "react-responsive": "^10.0.0",
+    "react-syntax-highlighter": "^15.5.0",
     "react18-json-view": "^0.2.8-canary.6",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
     "sonner": "^1.4.41",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.2.1",
@@ -113,7 +120,7 @@
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
     "vaul": "^0.9.1",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
@@ -131,6 +138,7 @@
     "@types/node": "20.10.5",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
+    "@types/react-syntax-highlighter": "^15.5.13",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",

web/public/generated/api-client/openapi.yml

@@ -11,12 +11,8 @@ paths:
         - Score
       parameters: []
       responses:
-        '200':
+        '204':
           description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
         '400':
           description: ''
           content:
@@ -59,21 +55,39 @@ components:
           type: string
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
         comment:
           type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
         - value
-    Score:
-      title: Score
+    BaseScore:
+      title: BaseScore
       type: object
       properties:
         id:
@@ -82,22 +96,147 @@ components:
           type: string
         name:
           type: string
-        value:
-          type: number
-          format: double
+        source:
+          $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
         timestamp:
           type: string
           format: date-time
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+        authorUserId:
+          type: string
         comment:
           type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
         - name
-        - value
+        - source
         - timestamp
+        - createdAt
+        - updatedAt
+    NumericScore:
+      title: NumericScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: The numeric value of the score
+      required:
+        - value
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+      required:
+        - value
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+      required:
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    ScoreSource:
+      title: ScoreSource
+      type: string
+      enum:
+        - ANNOTATION
+        - API
+        - EVAL
+    ScoreDataType:
+      title: ScoreDataType
+      type: string
+      enum:
+        - NUMERIC
+        - CATEGORICAL
+        - BOOLEAN
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: number
+          format: double
+        - type: string
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
   securitySchemes:
     BearerAuth:
       type: http

web/public/generated/api/openapi.yml

@@ -334,7 +334,7 @@ paths:
               $ref: '#/components/schemas/CreateDatasetRequest'
   /api/public/v2/datasets/{datasetName}:
     get:
-      description: Get a dataset and its items
+      description: Get a dataset
       operationId: datasets_get
       tags:
         - Datasets
@@ -429,7 +429,7 @@ paths:
       security: *ref_0
   /api/public/datasets/{datasetName}/runs:
     get:
-      description: Get a dataset runs
+      description: Get dataset runs
       operationId: datasets_getRuns
       tags:
         - Datasets
@@ -699,6 +699,193 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
+  /api/public/models:
+    post:
+      description: Create a model
+      operationId: models_create
+      tags:
+        - Models
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Model'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateModelRequest'
+    get:
+      description: Get all models
+      operationId: models_list
+      tags:
+        - Models
+      parameters:
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedModels'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/models/{id}:
+    get:
+      description: Get a model
+      operationId: models_get
+      tags:
+        - Models
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Model'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+    delete:
+      description: >-
+        Delete a model. Cannot delete models managed by Langfuse. You can create
+        your own definition with the same modelName to override the definition
+        though.
+      operationId: models_delete
+      tags:
+        - Models
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '204':
+          description: ''
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/observations/{observationId}:
     get:
       description: Get a observation
@@ -803,13 +990,23 @@ paths:
         - name: fromStartTime
           in: query
           description: >-
-            Retrieve only observations with a start_time greater than this
+            Retrieve only observations with a start_time or or after this
             datetime (ISO 8601).
           required: false
           schema:
             type: string
             format: date-time
             nullable: true
+        - name: toStartTime
+          in: query
+          description: >-
+            Retrieve only observations with a start_time before this datetime
+            (ISO 8601).
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1063,6 +1260,53 @@ paths:
             schema:
               $ref: '#/components/schemas/CreatePromptRequest'
   /api/public/score-configs:
+    post:
+      description: >-
+        Create a score configuration (config). Score configs are used to define
+        the structure of scores
+      operationId: scoreConfigs_create
+      tags:
+        - ScoreConfigs
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreConfig'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateScoreConfigRequest'
     get:
       description: Get all score configs
       operationId: scoreConfigs_get
@@ -1080,7 +1324,7 @@ paths:
           in: query
           description: >-
             Limit of items per page. If you encounter api issues due to too
-            large page sizes, try to reduce the limit.
+            large page sizes, try to reduce the limit
           required: false
           schema:
             type: integer
@@ -1177,7 +1421,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Score'
+                $ref: '#/components/schemas/CreateScoreResponse'
         '400':
           description: ''
           content:
@@ -1234,19 +1478,33 @@ paths:
             nullable: true
         - name: userId
           in: query
+          description: Retrieve only scores with this userId associated to the trace.
           required: false
           schema:
             type: string
             nullable: true
         - name: name
           in: query
+          description: Retrieve only scores with this name.
           required: false
           schema:
             type: string
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only scores newer than this datetime (ISO 8601).
+          description: >-
+            Optional filter to only include scores created on or after a certain
+            datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: >-
+            Optional filter to only include scores created before a certain
+            datetime (ISO 8601)
           required: false
           schema:
             type: string
@@ -1281,6 +1539,20 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: configId
+          in: query
+          description: Retrieve only scores with a specific configId.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: dataType
+          in: query
+          description: Retrieve only scores with a specific dataType.
+          required: false
+          schema:
+            $ref: '#/components/schemas/ScoreDataType'
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1401,6 +1673,82 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
+  /api/public/sessions:
+    get:
+      description: Get sessions.
+      operationId: sessions_list
+      tags:
+        - Sessions
+      parameters:
+        - name: page
+          in: query
+          description: Page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: fromTimestamp
+          in: query
+          description: >-
+            Optional filter to only include sessions created on or after a
+            certain datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: >-
+            Optional filter to only include sessions created before a certain
+            datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedSessions'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/sessions/{sessionId}:
     get:
       description: >-
@@ -1539,7 +1887,19 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only traces newer than this datetime (ISO 8601).
+          description: >-
+            Optional filter to only include traces with a trace.timestamp on or
+            after a certain datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: >-
+            Optional filter to only include traces with a trace.timestamp before
+            a certain datetime (ISO 8601)
           required: false
           schema:
             type: string
@@ -1682,6 +2042,10 @@ components:
         htmlPath:
           type: string
           description: Path of trace in Langfuse UI
+        latency:
+          type: number
+          format: double
+          description: Latency of trace in seconds
         totalCost:
           type: number
           format: double
@@ -1690,12 +2054,15 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/ObservationsView'
+          description: List of observations
         scores:
           type: array
           items:
             $ref: '#/components/schemas/Score'
+          description: List of scores
       required:
         - htmlPath
+        - latency
         - totalCost
         - observations
         - scores
@@ -1824,6 +2191,10 @@ components:
           type: number
           format: double
           nullable: true
+        timeToFirstToken:
+          type: number
+          format: double
+          nullable: true
       allOf:
         - $ref: '#/components/schemas/Observation'
     Usage:
@@ -1882,19 +2253,27 @@ components:
           $ref: '#/components/schemas/ScoreDataType'
         isArchived:
           type: boolean
+          description: Whether the score config is archived. Defaults to false
         minValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets minimum value for numerical scores. If not set, the minimum
+            value defaults to -∞
         maxValue:
           type: number
           format: double
           nullable: true
+          description: >-
+            Sets maximum value for numerical scores. If not set, the maximum
+            value defaults to +∞
         categories:
           type: array
           items:
             $ref: '#/components/schemas/ConfigCategory'
           nullable: true
+          description: Configures custom categories for categorical scores
         description:
           type: string
           nullable: true
@@ -1918,8 +2297,8 @@ components:
       required:
         - value
         - label
-    Score:
-      title: Score
+    BaseScore:
+      title: BaseScore
       type: object
       properties:
         id:
@@ -1928,9 +2307,6 @@ components:
           type: string
         name:
           type: string
-        value:
-          type: number
-          format: double
         source:
           $ref: '#/components/schemas/ScoreSource'
         observationId:
@@ -1939,16 +2315,130 @@ components:
         timestamp:
           type: string
           format: date-time
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+        authorUserId:
+          type: string
+          nullable: true
         comment:
           type: string
           nullable: true
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
         - name
-        - value
         - source
         - timestamp
+        - createdAt
+        - updatedAt
+    NumericScore:
+      title: NumericScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: The numeric value of the score
+      required:
+        - value
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+      required:
+        - value
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+      required:
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: number
+          format: double
+        - type: string
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
     Dataset:
       title: Dataset
       type: object
@@ -2086,6 +2576,71 @@ components:
         - datasetRunItems
       allOf:
         - $ref: '#/components/schemas/DatasetRun'
+    Model:
+      title: Model
+      type: object
+      description: >-
+        Model definition used for transforming usage into USD cost and/or
+        tokenization.
+      properties:
+        id:
+          type: string
+        modelName:
+          type: string
+          description: >-
+            Name of the model definition. If multiple with the same name exist,
+            they are applied in the following order: (1) custom over built-in,
+            (2) newest according to startTime where
+            model.startTime<observation.startTime
+        matchPattern:
+          type: string
+          description: >-
+            Regex pattern which matches this model definition to
+            generation.model. Useful in case of fine-tuned models. If you want
+            to exact match, use `(?i)^modelname$`
+        startDate:
+          type: string
+          nullable: true
+          description: Apply only to generations which are newer than this ISO date.
+        unit:
+          $ref: '#/components/schemas/ModelUsageUnit'
+          description: Unit used by this model.
+        inputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per input unit
+        outputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per output unit
+        totalPrice:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Price (USD) per total unit. Cannot be set if input or output price
+            is set.
+        tokenizerId:
+          type: string
+          nullable: true
+          description: >-
+            Optional. Tokenizer to be applied to observations which match to
+            this model. See docs for more details.
+        tokenizerConfig:
+          nullable: true
+          description: >-
+            Optional. Configuration for the selected tokenizer. Needs to be
+            JSON. See docs for more details.
+        isLangfuseManaged:
+          type: boolean
+      required:
+        - id
+        - modelName
+        - matchPattern
+        - unit
+        - isLangfuseManaged
     ModelUsageUnit:
       title: ModelUsageUnit
       type: string
@@ -2095,6 +2650,7 @@ components:
         - MILLISECONDS
         - SECONDS
         - IMAGES
+        - REQUESTS
       description: Unit of usage in Langfuse
     ObservationLevel:
       title: ObservationLevel
@@ -2639,17 +3195,37 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
@@ -2888,6 +3464,75 @@ components:
         - countTraces
         - countObservations
         - totalCost
+    PaginatedModels:
+      title: PaginatedModels
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/Model'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    CreateModelRequest:
+      title: CreateModelRequest
+      type: object
+      properties:
+        modelName:
+          type: string
+          description: >-
+            Name of the model definition. If multiple with the same name exist,
+            they are applied in the following order: (1) custom over built-in,
+            (2) newest according to startTime where
+            model.startTime<observation.startTime
+        matchPattern:
+          type: string
+          description: >-
+            Regex pattern which matches this model definition to
+            generation.model. Useful in case of fine-tuned models. If you want
+            to exact match, use `(?i)^modelname$`
+        startDate:
+          type: string
+          nullable: true
+          description: Apply only to generations which are newer than this ISO date.
+        unit:
+          $ref: '#/components/schemas/ModelUsageUnit'
+          description: Unit used by this model.
+        inputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per input unit
+        outputPrice:
+          type: number
+          format: double
+          nullable: true
+          description: Price (USD) per output unit
+        totalPrice:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Price (USD) per total units. Cannot be set if input or output price
+            is set.
+        tokenizerId:
+          type: string
+          nullable: true
+          description: >-
+            Optional. Tokenizer to be applied to observations which match to
+            this model. See docs for more details.
+        tokenizerConfig:
+          nullable: true
+          description: >-
+            Optional. Configuration for the selected tokenizer. Needs to be
+            JSON. See docs for more details.
+      required:
+        - modelName
+        - matchPattern
+        - unit
     Observations:
       title: Observations
       type: object
@@ -3146,6 +3791,47 @@ components:
       required:
         - data
         - meta
+    CreateScoreConfigRequest:
+      title: CreateScoreConfigRequest
+      type: object
+      properties:
+        name:
+          type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+        categories:
+          type: array
+          items:
+            $ref: '#/components/schemas/ConfigCategory'
+          nullable: true
+          description: >-
+            Configure custom categories for categorical scores. Pass a list of
+            objects with `label` and `value` properties. Categories are
+            autogenerated for boolean configs and cannot be passed
+        minValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a minimum value for numerical scores. If not set, the
+            minimum value defaults to -∞
+        maxValue:
+          type: number
+          format: double
+          nullable: true
+          description: >-
+            Configure a maximum value for numerical scores. If not set, the
+            maximum value defaults to +∞
+        description:
+          type: string
+          nullable: true
+          description: >-
+            Description is shown across the Langfuse UI and can be used to e.g.
+            explain the config categories in detail, why a numeric range was
+            set, or provide additional context on config name or usage
+      required:
+        - name
+        - dataType
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object
@@ -3155,21 +3841,49 @@ components:
           nullable: true
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
           nullable: true
         comment:
           type: string
           nullable: true
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
+          description: >-
+            The data type of the score. When passing a configId this field is
+            inferred. Otherwise, this field must be passed or will default to
+            numeric.
+        configId:
+          type: string
+          nullable: true
+          description: >-
+            Reference a score config on a score. The unique langfuse identifier
+            of a score config. When passing this field, the dataType and
+            stringValue fields are automatically populated.
       required:
         - traceId
         - name
         - value
+    CreateScoreResponse:
+      title: CreateScoreResponse
+      type: object
+      properties:
+        id:
+          type: string
+          description: The id of the created object in Langfuse
+      required:
+        - id
     Scores:
       title: Scores
       type: object
@@ -3183,6 +3897,19 @@ components:
       required:
         - data
         - meta
+    PaginatedSessions:
+      title: PaginatedSessions
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/Session'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     Traces:
       title: Traces
       type: object

web/public/generated/postman/collection.json

@@ -246,7 +246,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a dataset and its items",
+            "description": "Get a dataset",
             "url": {
               "raw": "{{baseUrl}}/api/public/v2/datasets/:datasetName",
               "host": [
@@ -352,7 +352,7 @@
           "_type": "endpoint",
           "name": "Get Runs",
           "request": {
-            "description": "Get a dataset runs",
+            "description": "Get dataset runs",
             "url": {
               "raw": "{{baseUrl}}/api/public/datasets/:datasetName/runs?page=&limit=",
               "host": [
@@ -535,6 +535,146 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Models",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"modelName\": \"example\",\n    \"matchPattern\": \"example\",\n    \"startDate\": \"1994-11-05\",\n    \"unit\": \"CHARACTERS\",\n    \"inputPrice\": 0,\n    \"outputPrice\": 0,\n    \"totalPrice\": 0,\n    \"tokenizerId\": \"example\",\n    \"tokenizerConfig\": \"UNKNOWN\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all models",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -578,7 +718,7 @@
           "request": {
             "description": "Get a list of observations",
             "url": {
-              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=",
+              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=&toStartTime=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -626,7 +766,12 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
+                  "description": "Retrieve only observations with a start_time or or after this datetime (ISO 8601)."
+                },
+                {
+                  "key": "toStartTime",
+                  "value": "",
+                  "description": "Retrieve only observations with a start_time before this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -814,6 +959,39 @@
       "description": null,
       "name": "Score Configs",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a score configuration (config). Score configs are used to define the structure of scores",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"dataType\": \"NUMERIC\",\n    \"categories\": [\n        {\n            \"value\": 0,\n            \"label\": \"example\"\n        }\n    ],\n    \"minValue\": 0,\n    \"maxValue\": 0,\n    \"description\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -838,7 +1016,7 @@
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit"
                 }
               ],
               "variable": []
@@ -912,7 +1090,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"value\": 0,\n    \"observationId\": \"example\",\n    \"comment\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"novelty\",\n    \"value\": 0.9,\n    \"traceId\": \"cdef-1234-5678-90ab\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -928,7 +1106,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&toTimestamp=&source=&operator=&value=&scoreIds=&configId=&dataType=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -951,17 +1129,22 @@
                 {
                   "key": "userId",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this userId associated to the trace."
                 },
                 {
                   "key": "name",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this name."
                 },
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
+                  "description": "Optional filter to only include scores created on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include scores created before a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "source",
@@ -982,6 +1165,16 @@
                   "key": "scoreIds",
                   "value": "",
                   "description": "Comma-separated list of score IDs to limit the results to."
+                },
+                {
+                  "key": "configId",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific configId."
+                },
+                {
+                  "key": "dataType",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific dataType."
                 }
               ],
               "variable": []
@@ -1064,6 +1257,52 @@
       "description": null,
       "name": "Sessions",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get sessions.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/sessions?page=&limit=&fromTimestamp=&toTimestamp=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "sessions"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "Page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include sessions created on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include sessions created before a certain datetime (ISO 8601)"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -1141,7 +1380,7 @@
           "request": {
             "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&toTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -1179,7 +1418,12 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
+                  "description": "Optional filter to only include traces with a trace.timestamp on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces with a trace.timestamp before a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "orderBy",

web/scripts/observations-backfill-calculated-cost.ts

@@ -77,15 +77,28 @@ const backfillCalculatedGenerationCost = async () => {
         ),
         updated_batch AS (
             UPDATE observations o
-            SET calculated_input_cost = COALESCE(batch.input_cost, batch.prompt_tokens::numeric * batch.input_price),
-                calculated_output_cost = COALESCE(batch.output_cost, batch.completion_tokens::numeric * batch.output_price),
-                calculated_total_cost = COALESCE(
-                    batch.total_cost,
+            SET calculated_input_cost = 
                     CASE
-                        WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
-                        ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
-                    END
-                ),
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.prompt_tokens::numeric * batch.input_price
+                        ELSE batch.input_cost
+                    END,
+                calculated_output_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.completion_tokens::numeric * batch.output_price
+                        ELSE batch.output_cost
+                    END,
+                calculated_total_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN
+                            CASE
+                                WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
+                                ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
+                            END
+                        ELSE batch.total_cost
+                    END,
                 internal_model_id = batch.model_id,
                 tmp_has_calculated_cost = TRUE
             FROM batch

web/src/__tests__/datasets.servertest.ts

@@ -95,7 +95,8 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dbDataset.length).toBeGreaterThan(0);
 
     // get dataset (v1) excluding items and runs
-    const getDatasetV1 = await makeAPICall(
+    const getDatasetV1 = await makeZodVerifiedAPICall(
+      GetDatasetV1Response,
       "GET",
       `/api/public/datasets/${encodeURIComponent("dataset + name")}`,
     );

web/src/__tests__/events.servertest.ts

@@ -1,8 +1,13 @@
 /** @jest-environment node */
 
 import { prisma } from "@langfuse/shared/src/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import { PostEventsV1Response } from "@/src/features/public-api/types/events";
 
 describe("/api/public/events API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -12,15 +17,20 @@ describe("/api/public/events API Endpoint", () => {
 
     const traceId = uuidv4();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -32,16 +42,21 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbTrace[0]?.id).toBe(traceId);
 
     const eventId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceId: traceId,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceId: traceId,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -66,16 +81,21 @@ describe("/api/public/events API Endpoint", () => {
     const traceId = uuidv4();
     const eventId = uuidv4();
 
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceId: traceId,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceId: traceId,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -93,15 +113,20 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbEvent?.metadata).toEqual({ meta: "value" });
     expect(dbEvent?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -118,15 +143,20 @@ describe("/api/public/events API Endpoint", () => {
 
     const traceId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     expect(response.status).toBe(200);
 
@@ -141,17 +171,22 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbTrace[0]?.id).not.toBe(traceId);
 
     const eventId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceIdType: "EXTERNAL",
-      traceId: dbTrace[0]?.id,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceIdType: "EXTERNAL",
+        traceId: dbTrace[0]?.id,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -174,16 +209,21 @@ describe("/api/public/events API Endpoint", () => {
     const eventName = uuidv4();
 
     const spanId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: spanId,
-      name: eventName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: spanId,
+        name: eventName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -215,14 +255,19 @@ describe("/api/public/events API Endpoint", () => {
     const generationName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/events", {
-      id: spanId,
-      name: generationName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { key: "value" },
-      metadata: { key: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: spanId,
+        name: generationName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { key: "value" },
+        metadata: { key: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbEvent = await prisma.observation.findFirstOrThrow({
       where: {

web/src/__tests__/generations.servertest.ts

@@ -2,9 +2,19 @@
 
 import { v4 as uuidv4 } from "uuid";
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import {
+  PatchGenerationsV1Response,
+  PostGenerationsV1Body,
+  PostGenerationsV1Response,
+} from "@/src/features/public-api/types/generations";
 
 describe("/api/public/generations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -90,15 +100,20 @@ describe("/api/public/generations API Endpoint", () => {
 
       const traceId = uuidv4();
 
-      await makeAPICall("POST", "/api/public/traces", {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      });
+      await makeZodVerifiedAPICall(
+        PostTracesV1Response,
+        "POST",
+        "/api/public/traces",
+        {
+          id: traceId,
+          name: "trace-name",
+          userId: "user-1",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          metadata: { key: "value" },
+          release: "1.0.0",
+          version: "2.0.0",
+        },
+      );
 
       const dbTrace = await prisma.trace.findMany({
         where: {
@@ -110,7 +125,8 @@ describe("/api/public/generations API Endpoint", () => {
       expect(dbTrace[0]?.id).toBe(traceId);
 
       const generationId = uuidv4();
-      const createGeneration = await makeAPICall(
+      const createGeneration = await makeZodVerifiedAPICall(
+        PostGenerationsV1Response,
         "POST",
         "/api/public/generations",
         {
@@ -164,7 +180,8 @@ describe("/api/public/generations API Endpoint", () => {
     const traceId = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -201,15 +218,20 @@ describe("/api/public/generations API Endpoint", () => {
     expect(dbGeneration?.metadata).toEqual({ key: "value" });
     expect(dbGeneration?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -224,15 +246,20 @@ describe("/api/public/generations API Endpoint", () => {
   it("should create generation after trace ignoring externalId", async () => {
     const traceId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     expect(response.status).toBe(200);
 
@@ -247,7 +274,8 @@ describe("/api/public/generations API Endpoint", () => {
     expect(dbTrace[0]?.id).not.toBe(traceId);
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -290,7 +318,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -340,7 +369,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -368,7 +398,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationId2 = uuidv4();
     const generationName2 = uuidv4();
 
-    const createGeneration2 = await makeAPICall(
+    const createGeneration2 = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -401,7 +432,8 @@ describe("/api/public/generations API Endpoint", () => {
 
     const generationId = uuidv4();
     const externalTraceId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -437,7 +469,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -488,7 +521,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -506,7 +540,8 @@ describe("/api/public/generations API Endpoint", () => {
 
     expect(createGeneration.status).toBe(200);
 
-    const updateGeneration = await makeAPICall(
+    const updateGeneration = await makeZodVerifiedAPICall(
+      PatchGenerationsV1Response,
       "PATCH",
       "/api/public/generations",
       {
@@ -540,7 +575,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {

web/src/__tests__/ingestion.servertest.ts

@@ -57,6 +57,17 @@ describe("/api/public/ingestion API Endpoint", () => {
       expectedCompletionTokens: 2,
       expectedTotalTokens: 3,
     },
+    {
+      usage: {
+        input: 1,
+        output: 2,
+        unit: ModelUsageUnit.Requests,
+      },
+      expectedUnit: ModelUsageUnit.Requests,
+      expectedPromptTokens: 1,
+      expectedCompletionTokens: 2,
+      expectedTotalTokens: 3,
+    },
     {
       usage: {
         input: 30,

web/src/__tests__/markdown.clienttest.ts

@@ -0,0 +1,68 @@
+import { containsAnyMarkdown } from "@/src/components/schemas/MarkdownSchema";
+
+describe("containsAnyMarkdown Function", () => {
+  it("Detects simple bold using asterisks", () => {
+    expect(containsAnyMarkdown("This is **bold** text")).toBe(true);
+  });
+
+  it("Detects simple italics using asterisks", () => {
+    expect(containsAnyMarkdown("This is *italic* text")).toBe(true);
+  });
+
+  it("Detects bold using underscores", () => {
+    expect(containsAnyMarkdown("This is __bold__ text")).toBe(true);
+  });
+
+  it("Detects italics using underscores", () => {
+    expect(containsAnyMarkdown("This is _italic_ text")).toBe(true);
+  });
+
+  it("Detects inline code with backticks", () => {
+    expect(containsAnyMarkdown("This is `code` inline")).toBe(true);
+  });
+
+  it("Detects fenced code blocks", () => {
+    const codeBlock = "```\nlet x = 10;\n```";
+    expect(containsAnyMarkdown(codeBlock)).toBe(true);
+  });
+
+  it("Detects headers", () => {
+    expect(containsAnyMarkdown("# Header 1")).toBe(true);
+    expect(containsAnyMarkdown("## Header 2")).toBe(true);
+  });
+
+  it("Detects unordered lists", () => {
+    expect(containsAnyMarkdown("- List item 1\n- List item 2")).toBe(true);
+  });
+
+  it("Detects ordered lists", () => {
+    expect(containsAnyMarkdown("1. First item\n2. Second item")).toBe(true);
+  });
+
+  it("Detects blockquotes", () => {
+    expect(containsAnyMarkdown("> This is a blockquote")).toBe(true);
+  });
+
+  it("Detects links", () => {
+    expect(containsAnyMarkdown("[Google](http://www.google.com)")).toBe(true);
+  });
+
+  it("Detects images", () => {
+    expect(containsAnyMarkdown("![Alt text](http://url/to/img.png)")).toBe(
+      true,
+    );
+  });
+
+  it("Returns false for non-markdown text", () => {
+    expect(
+      containsAnyMarkdown("This is plain text without any markdown syntax"),
+    ).toBe(false);
+    expect(containsAnyMarkdown("12345")).toBe(false);
+    expect(containsAnyMarkdown("Simple text.")).toBe(false);
+  });
+
+  it("Detects combination of markdown elements", () => {
+    const complexMarkdown = "**Bold** and _italic_ and `code`";
+    expect(containsAnyMarkdown(complexMarkdown)).toBe(true);
+  });
+});

web/src/__tests__/metrics-api.servertest.ts

@@ -1,4 +1,10 @@
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import { PostGenerationsV1Response } from "@/src/features/public-api/types/generations";
+import { GetMetricsDailyV1Response } from "@/src/features/public-api/types/metrics";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 import { v4 as uuidv4 } from "uuid";
 
 describe("/api/public/metrics/daily API Endpoint", () => {
@@ -6,53 +12,89 @@ describe("/api/public/metrics/daily API Endpoint", () => {
   afterEach(async () => await pruneDatabase());
 
   it("should handle daily metrics correctly", async () => {
-    await pruneDatabase();
-
     // Create traces with observations on different days
     const traceId1 = uuidv4();
     const traceId2 = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId1,
-      timestamp: "2021-01-01T00:00:00.000Z",
-      name: "trace-day-1",
-      userId: "user-daily-metrics",
-      projectId: "project-daily-metrics",
-    });
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId2,
-      timestamp: "2021-01-02T00:00:00.000Z",
-      name: "trace-day-2",
-      userId: "user-daily-metrics",
-      projectId: "project-daily-metrics",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId1,
+        timestamp: "2021-01-01T00:00:00.000Z",
+        name: "trace-day-1",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId2,
+        timestamp: "2021-01-02T00:00:00.000Z",
+        name: "trace-day-2",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
 
     // Simulate observations with usage metrics on different days
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId1,
-      model: "modelA",
-      usage: { input: 100, output: 200, total: 300 },
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:01:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId2,
-      model: "modelB",
-      usage: { input: 333 },
-      startTime: "2021-01-02T00:00:00.000Z",
-      endTime: "2021-01-02T00:02:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId2,
-      model: "modelC",
-      usage: { input: 666, output: 777, totalCost: 1024.22 },
-      startTime: "2021-01-02T00:00:00.000Z",
-      endTime: "2021-01-02T00:04:00.000Z",
-    });
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId1,
+        model: "modelA",
+        usage: { input: 100, output: 200, total: 300 },
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:01:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        model: "modelB",
+        usage: { input: 333 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:02:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        model: "modelC",
+        usage: { input: 666, output: 777, totalCost: 1024.22 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:04:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        usage: { output: 300 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:04:00.000Z",
+      },
+    );
 
     // Retrieve the daily metrics
-    const dailyMetricsResponse = await makeAPICall<{
-      data: Array<Record<string, unknown>>;
-    }>("GET", `/api/public/metrics/daily`);
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
     const dailyMetricsData = dailyMetricsResponse.body.data;
 
     // Check if the daily metrics are calculated correctly
@@ -81,6 +123,15 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         countTraces: 1,
         totalCost: 1024.22,
       },
+      {
+        model: null,
+        countObservations: 1,
+        countTraces: 1,
+        inputUsage: 0,
+        outputUsage: 300,
+        totalCost: 0,
+        totalUsage: 300,
+      },
     ]);
 
     if (!dailyMetricsData[1])
@@ -100,4 +151,48 @@ describe("/api/public/metrics/daily API Endpoint", () => {
       },
     ]);
   });
+
+  it("should handle daily metrics correctly when there is no data", async () => {
+    // Retrieve the daily metrics
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
+    const dailyMetricsData = dailyMetricsResponse.body.data;
+
+    // Check if the daily metrics are calculated correctly
+    expect(dailyMetricsData).toHaveLength(0); // No data
+  });
+
+  it("should handle daily metrics correctly when there is just a trace", async () => {
+    const traceId1 = uuidv4();
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId1,
+        timestamp: "2021-01-01T00:00:00.000Z",
+        name: "trace-day-1",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
+
+    // Retrieve the daily metrics
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
+    const dailyMetricsData = dailyMetricsResponse.body.data;
+
+    // Check if the daily metrics are calculated correctly
+    expect(dailyMetricsData).toHaveLength(1);
+    expect(dailyMetricsData[0].date).toBe("2021-01-01");
+    expect(dailyMetricsData[0].countTraces).toBe(1);
+    expect(dailyMetricsData[0].totalCost).toEqual(0);
+    expect(dailyMetricsData[0].usage).toEqual([]);
+  });
 });

web/src/__tests__/model-definitions.servertest.ts

@@ -0,0 +1,221 @@
+/** @jest-environment node */
+
+import { prisma } from "@langfuse/shared/src/db";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  DeleteModelV1Response,
+  GetModelV1Response,
+  GetModelsV1Response,
+  PostModelsV1Response,
+} from "@/src/features/public-api/types/models";
+
+describe("/models API Endpoints", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+    // create some default models that do not belong to a project
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0020",
+        outputPrice: "0.0040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("GET /models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(2);
+    expect(models.body.data[0]).toMatchObject({
+      isLangfuseManaged: true,
+      modelName: "gpt-3.5-turbo",
+    });
+  });
+
+  it("GET /models pagination", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models?page=2&limit=1",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(1);
+    expect(models.body.meta).toMatchObject({
+      page: 2,
+      totalPages: 2,
+      limit: 1,
+      totalItems: 2,
+    });
+  });
+
+  it("Create and get custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+    expect(customModel.body.isLangfuseManaged).toBe(false);
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    const getModel = await makeZodVerifiedAPICall(
+      GetModelV1Response,
+      "GET",
+      `/api/public/models/${customModel.body.id}`,
+    );
+    expect(getModel.body.id).toBe(customModel.body.id);
+    expect(getModel.body).toMatchObject({
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      startDate: new Date("2023-12-01").toISOString(),
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      isLangfuseManaged: false,
+    });
+  });
+
+  it("Post model with invalid matchPattern", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Post model without prices or tokenizer", async () => {
+    await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        unit: "TOKENS",
+      },
+    );
+  });
+
+  it("Post model with missing fields", async () => {
+    const { status } = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      // missing unit
+    });
+    expect(status).toBe(400);
+  });
+
+  it("Post model with invalid price (input and total cost)", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      totalPrice: 0.1,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Cannot delete built-in models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(2);
+
+    const deleteModel = await makeAPICall(
+      "DELETE",
+      `/api/public/models/${models.body.data[0].id}`,
+    );
+    expect(deleteModel.status).toBe(404);
+  });
+
+  it("Delete custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    await makeZodVerifiedAPICall(
+      DeleteModelV1Response,
+      "DELETE",
+      `/api/public/models/${customModel.body.id}`,
+    );
+
+    const modelsAfterDelete = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(modelsAfterDelete.body.data.length).toBe(2);
+  });
+});

web/src/__tests__/observations.servertest.ts

@@ -2,10 +2,16 @@
 
 import { v4 as uuidv4 } from "uuid";
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared";
+import {
+  GetObservationV1Response,
+  GetObservationsV1Response,
+} from "@/src/features/public-api/types/observations";
 
 describe("/api/public/observations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -79,7 +85,8 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
-    const fetchedObservations = await makeAPICall(
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
       "GET",
       "/api/public/observations",
       undefined,
@@ -87,13 +94,6 @@ describe("/api/public/observations API Endpoint", () => {
 
     expect(fetchedObservations.status).toBe(200);
 
-    if (!isObservationList(fetchedObservations.body)) {
-      throw new Error(
-        "Expected body to be an array of observations" +
-          JSON.stringify(fetchedObservations.body),
-      );
-    }
-
     expect(fetchedObservations.body.data.length).toBe(1);
     expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
     expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
@@ -183,7 +183,8 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
-    const fetchedObservations = await makeAPICall(
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
       "GET",
       "/api/public/observations?type=GENERATION",
       undefined,
@@ -193,10 +194,6 @@ describe("/api/public/observations API Endpoint", () => {
 
     expect(fetchedObservations.status).toBe(200);
 
-    if (!isObservationList(fetchedObservations.body)) {
-      throw new Error("Expected body to be an array of observations");
-    }
-
     expect(fetchedObservations.body.data.length).toBe(1);
     expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
     expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
@@ -205,37 +202,236 @@ describe("/api/public/observations API Endpoint", () => {
   });
 });
 
-const isObservationList = (val: unknown): val is ObservationResponse => {
-  return (
-    typeof val === "object" &&
-    val !== null &&
-    "data" in val &&
-    Array.isArray(val.data) &&
-    val.data.every(
-      (element) =>
-        typeof element === "object" &&
-        element !== null &&
-        "id" in element &&
-        "traceId" in element &&
-        "name" in element &&
-        "startTime" in element &&
-        "endTime" in element &&
-        "model" in element &&
-        "input" in element &&
-        "output" in element &&
-        "metadata" in element &&
-        "version" in element &&
-        "modelId" in element &&
-        "inputPrice" in element &&
-        "outputPrice" in element &&
-        "totalPrice" in element &&
-        "calculatedInputCost" in element &&
-        "calculatedOutputCost" in element &&
-        "calculatedTotalCost" in element,
-    )
-  );
-};
+it("GET /observations with timestamp filters and pagination", async () => {
+  await prisma.trace.create({
+    data: {
+      id: "trace-id",
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+  await prisma.observation.createMany({
+    data: [
+      {
+        id: "observation-2021-01-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-01-01T00:00:00.000Z"),
+        endTime: new Date("2021-01-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "GENERATION",
+      },
+      {
+        id: "observation-2021-02-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-02-01T00:00:00.000Z"),
+        endTime: new Date("2021-02-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "SPAN",
+      },
+      {
+        id: "observation-2021-03-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-03-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "EVENT",
+      },
+      {
+        id: "observation-2021-04-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-04-01T00:00:00.000Z"),
+        endTime: new Date("2021-04-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "GENERATION",
+      },
+    ],
+  });
 
-type ObservationResponse = {
-  data: ObservationView[];
-};
+  const fromTimestamp = "2021-02-01T00:00:00.000Z";
+  const toTimestamp = "2021-04-01T00:00:00.000Z";
+
+  // Test with both fromTimestamp and toTimestamp
+  let fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(2);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(2);
+
+  // Test with only fromTimestamp
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?fromStartTime=${fromTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(3);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+  // Test with only toTimestamp
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?toStartTime=${toTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(3);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+  // test pagination only
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?limit=1&page=2`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(1);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.meta).toMatchObject({
+    totalItems: 4,
+    totalPages: 4,
+    page: 2,
+    limit: 1,
+  });
+});
+
+it("Get a single EVENT from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  // EVENT
+  const eventId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: eventId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      type: "EVENT",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getEventRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + eventId,
+  );
+  expect(getEventRes.body).toMatchObject({
+    id: eventId,
+    traceId: traceId,
+    type: "EVENT",
+  });
+});
+it("Get a single GENERATION from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  const generationId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: generationId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      endTime: new Date("2021-01-01T00:00:00.000Z"),
+      model: "gpt-3.5-turbo",
+      internalModel: "gpt-3.5-turbo",
+      modelParameters: { key: "value" },
+      input: { key: "input" },
+      output: { key: "output" },
+      promptTokens: 10,
+      completionTokens: 20,
+      totalTokens: 30,
+      version: "2.0.0",
+      unit: ModelUsageUnit.Tokens,
+      type: "GENERATION",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getGenerationRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + generationId,
+  );
+  expect(getGenerationRes.body).toMatchObject({
+    id: generationId,
+    traceId: traceId,
+    input: { key: "input" },
+    output: { key: "output" },
+    type: "GENERATION",
+  });
+});
+it("Get a single SPAN from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  const spanId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: spanId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      endTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelParameters: { key: "value" },
+      input: { key: "input" },
+      output: { key: "output" },
+      version: "2.0.0",
+      type: "SPAN",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getSpanRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + spanId,
+  );
+  expect(getSpanRes.body).toMatchObject({
+    id: spanId,
+    traceId: traceId,
+    input: { key: "input" },
+    output: { key: "output" },
+    type: "SPAN",
+  });
+});

web/src/__tests__/prompts.v1.servertest.ts

@@ -219,7 +219,7 @@ describe("/api/public/prompts API Endpoint", () => {
     });
 
     // Third prompt is deactivated
-    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+    await makeAPICall("POST", "/api/public/prompts", {
       name: "prompt-name",
       projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
       prompt: "prompt3",

web/src/__tests__/score-configs.servertest.ts

@@ -1,24 +1,30 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { v4 as uuidv4 } from "uuid";
-import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
-import { type CastedConfig } from "@langfuse/shared";
-
-const CONFIG_ID_ONE = uuidv4();
-const CONFIG_ID_TWO = uuidv4();
-const CONFIG_ID_THREE = uuidv4();
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  type ScoreConfig,
+  prisma,
+  type ScoreDataType,
+} from "@langfuse/shared/src/db";
+import {
+  GetScoreConfigResponse,
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+} from "@/src/features/public-api/types/score-configs";
 
 const configOne = [
   {
-    id: CONFIG_ID_ONE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Boolean Config",
     description: "Test Description",
-    dataType: ScoreDataType.BOOLEAN,
+    dataType: "BOOLEAN" as ScoreDataType,
     categories: [
-      { label: "False", value: 0 },
       { label: "True", value: 1 },
+      { label: "False", value: 0 },
     ],
     createdAt: new Date("2024-05-10T00:00:00.000Z"),
     updatedAt: new Date("2024-05-10T00:00:00.000Z"),
@@ -26,11 +32,10 @@ const configOne = [
 ];
 const configTwo = [
   {
-    id: CONFIG_ID_TWO,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Numeric Config",
     description: "Test Description",
-    dataType: ScoreDataType.NUMERIC,
+    dataType: "NUMERIC" as ScoreDataType,
     minValue: 0,
     createdAt: new Date("2024-05-11T00:00:00.000Z"),
     updatedAt: new Date("2024-05-11T00:00:00.000Z"),
@@ -39,11 +44,10 @@ const configTwo = [
 
 const configThree = [
   {
-    id: CONFIG_ID_THREE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Categorical Config",
     description: "Test Description",
-    dataType: ScoreDataType.CATEGORICAL,
+    dataType: "CATEGORICAL" as ScoreDataType,
     categories: [
       { label: "A", value: 0 },
       { label: "B", value: 1 },
@@ -69,11 +73,18 @@ describe("/api/public/score-configs API Endpoint", () => {
   );
 
   it("should GET a score config", async () => {
-    const configId = CONFIG_ID_ONE;
+    const { id: configId } = (await prisma.scoreConfig.findFirst({
+      where: {
+        projectId: configOne[0].projectId,
+        name: configOne[0].name,
+      },
+    })) as ScoreConfig;
 
-    const getScoreConfig = await makeAPICall<{
-      id: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(200);
     expect(getScoreConfig.body).toMatchObject({
@@ -84,24 +95,12 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
-  it("test invalid config id input", async () => {
-    const configId = "invalid-config-id";
-
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
-
-    expect(getScoreConfig.status).toBe(404);
-    expect(getScoreConfig.body).toMatchObject({
-      message: "Score config not found within authorized project",
-    });
-  });
-
   it("should GET all score configs", async () => {
-    const fetchedConfigs = await makeAPICall<{
-      data: CastedConfig[];
-      meta: object;
-    }>("GET", `/api/public/score-configs?limit=50&page=1`);
+    const fetchedConfigs = await makeZodVerifiedAPICall(
+      GetScoreConfigsResponse,
+      "GET",
+      `/api/public/score-configs?limit=50&page=1`,
+    );
 
     expect(fetchedConfigs.status).toBe(200);
     expect(fetchedConfigs.body.meta).toMatchObject({
@@ -120,6 +119,20 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
+  it("test invalid config id input", async () => {
+    const configId = "invalid-config-id";
+
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
+
+    expect(getScoreConfig.status).toBe(404);
+    expect(getScoreConfig.body).toMatchObject({
+      message: "Score config not found within authorized project",
+    });
+  });
+
   it("should return 500 when hitting corrupted score config", async () => {
     const configId = "corrupted-config-id";
 
@@ -131,13 +144,234 @@ describe("/api/public/score-configs API Endpoint", () => {
       },
     });
 
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(500);
     expect(getScoreConfig.body).toMatchObject({
-      message: "Internal Server Error",
+      message: "Requested score config is corrupted",
+    });
+  });
+
+  it("should POST a numeric score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "numeric-config-name",
+        dataType: "NUMERIC",
+        maxValue: 0,
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("numeric-config-name");
+    expect(scoreConfig.body.dataType).toBe("NUMERIC");
+    expect(scoreConfig.body.maxValue).toBe(0);
+  });
+
+  it("should POST a boolean score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "boolean-config-name",
+        dataType: "BOOLEAN",
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("boolean-config-name");
+    expect(scoreConfig.body.dataType).toBe("BOOLEAN");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "True", value: 1 },
+      { label: "False", value: 0 },
+    ]);
+  });
+
+  it("should POST a categorical score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("categorical-config-name");
+    expect(scoreConfig.body.dataType).toBe("CATEGORICAL");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "Good", value: 1 },
+      { label: "Bad", value: 0 },
+    ]);
+  });
+
+  it("should fail POST of numeric score config with invalid range", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        PostScoreConfigResponse,
+        "POST",
+        "/api/public/score-configs",
+        {
+          name: "invalid-numeric-config-name",
+          dataType: "NUMERIC",
+          maxValue: 0,
+          minValue: 1,
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"custom\",\"message\":\"Maximum value must be greater than Minimum value\",\"path\":[]}]}`,
+      );
+    }
+  });
+
+  it("should fail POST of boolean score config with custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-boolean-config-name",
+        dataType: "BOOLEAN",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with NO custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with invalid custom categories format", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { key: "first", value: 1 },
+          { key: "second", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Category must be an array of objects with label value pairs, where labels and values are unique.",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category label", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "first", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category label: first, category labels must be unique",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category value", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "second", value: 1 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category value: 1, category values must be unique",
+          path: ["categories"],
+        },
+      ],
     });
   });
 });

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -19,11 +19,11 @@ describe("Traces TRPC Router", () => {
           id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           role: "ADMIN",
           name: "test",
+          cloudConfig: { defaultLookBackDays: null },
         },
       ],
       featureFlags: {
         templateFlag: true,
-        evals: true,
       },
       admin: true,
     },
@@ -36,7 +36,6 @@ describe("Traces TRPC Router", () => {
     const trace = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };
@@ -57,11 +56,42 @@ describe("Traces TRPC Router", () => {
     expect(traces).toMatchObject({ traces: [trace] });
   });
 
+  test("traces.all RPC must not return input, output, metadata", async () => {
+    const trace = {
+      name: "trace-name",
+      userId: "user-1",
+      input: { a: 1 },
+      output: { b: 2 },
+      metadata: { c: 3 },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    await prisma.trace.create({
+      data: { ...trace, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const traces = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: null,
+    });
+    expect(traces.traces).toBeDefined();
+    expect(traces.traces).toHaveLength(1);
+
+    const returnedTrace = traces.traces[0];
+    expect(returnedTrace).not.toHaveProperty("input");
+    expect(returnedTrace).not.toHaveProperty("output");
+    expect(returnedTrace).not.toHaveProperty("metadata");
+  });
+
   test("traces.all RPC orders traces by userId", async () => {
     const traceTmpl = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };

web/src/__tests__/sessions.servertest.ts

@@ -1,20 +1,33 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  GetSessionV1Response,
+  GetSessionsV1Response,
+} from "@/src/features/public-api/types/sessions";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 import { prisma } from "@langfuse/shared/src/db";
 
-describe("/api/public/traces API Endpoint", () => {
+describe("Create and get sessions", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
   it("should create a session via a trace", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      sessionId: "session-id",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        name: "trace-name",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        sessionId: "session-id",
+      },
+    );
 
     const dbSession = await prisma.traceSession.findFirst({
       where: {
@@ -32,16 +45,22 @@ describe("/api/public/traces API Endpoint", () => {
   it("should get session including traces", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      id: "trace-id",
-      input: { hello: "world" },
-      output: "hi",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      sessionId: "session-id",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        name: "trace-name",
+        id: "trace-id",
+        input: { hello: "world" },
+        output: "hi",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        sessionId: "session-id",
+      },
+    );
 
-    const response = await makeAPICall(
+    const response = await makeZodVerifiedAPICall(
+      GetSessionV1Response,
       "GET",
       "/api/public/sessions/session-id",
     );
@@ -60,3 +79,122 @@ describe("/api/public/traces API Endpoint", () => {
     });
   });
 });
+
+describe("GET /api/public/sessions API Endpoint", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+    await prisma.traceSession.createMany({
+      data: [
+        {
+          id: "session-2021-01-01",
+          createdAt: new Date("2021-01-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-02-01",
+          createdAt: new Date("2021-02-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-03-01",
+          createdAt: new Date("2021-03-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-04-01",
+          createdAt: new Date("2021-04-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-05-01",
+          createdAt: new Date("2021-05-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+      ],
+    });
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("should return all sessions", async () => {
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      "/api/public/sessions",
+    );
+    expect(sessions.body.data).toHaveLength(5);
+  });
+
+  it("should return paginated sessions with limit and page number", async () => {
+    const limit = 2;
+    const page = 2;
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?limit=${limit}&page=${page}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-03-01",
+      "session-2021-02-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(5);
+    expect(sessions.body.meta.totalPages).toBe(3);
+    expect(sessions.body.meta.page).toBe(page);
+    expect(sessions.body.meta.limit).toBe(limit);
+  });
+
+  it("should return sessions within a specific date range", async () => {
+    const fromTimestamp = "2021-02-01T00:00:00Z";
+    const toTimestamp = "2021-04-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?fromTimestamp=${fromTimestamp}&toTimestamp=${toTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-03-01",
+      "session-2021-02-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(2);
+  });
+
+  it("should return sessions from a specific date onwards (including the date)", async () => {
+    const fromTimestamp = "2021-03-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?fromTimestamp=${fromTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(3);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-05-01",
+      "session-2021-04-01",
+      "session-2021-03-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(3);
+  });
+
+  it("should return sessions up to a specific date (excluding it)", async () => {
+    const toTimestamp = "2021-03-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?toTimestamp=${toTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-02-01",
+      "session-2021-01-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(2);
+  });
+});

web/src/__tests__/spans.servertest.ts

@@ -1,27 +1,39 @@
 /** @jest-environment node */
 
 import { prisma } from "@langfuse/shared/src/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import {
+  PatchSpansV1Response,
+  PostSpansV1Response,
+} from "@/src/features/public-api/types/spans";
 
 describe("/api/public/spans API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
   it("should create span after trace", async () => {
-    await pruneDatabase();
-
     const traceId = uuidv4();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -33,16 +45,21 @@ describe("/api/public/spans API Endpoint", () => {
     expect(dbTrace[0]?.id).toBe(traceId);
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceId: traceId,
-      name: "span-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceId: traceId,
+        name: "span-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
     const dbSpan = await prisma.observation.findUnique({
@@ -62,21 +79,24 @@ describe("/api/public/spans API Endpoint", () => {
   });
 
   it("should create span before trace", async () => {
-    await pruneDatabase();
-
     const traceId = uuidv4();
     const spanId = uuidv4();
 
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceId: traceId,
-      name: "span-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceId: traceId,
+        name: "span-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
     const dbSpan = await prisma.observation.findUnique({
@@ -94,15 +114,20 @@ describe("/api/public/spans API Endpoint", () => {
     expect(dbSpan?.metadata).toEqual({ meta: "value" });
     expect(dbSpan?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -118,21 +143,31 @@ describe("/api/public/spans API Endpoint", () => {
     const traceId = uuidv4();
     const spanId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: uuidv4(),
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: uuidv4(),
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      },
+    );
     expect(response.status).toBe(200);
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: "spanName",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      traceId: traceId,
-      externalTraceIdType: "EXTERNAL",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: "spanName",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        traceId: traceId,
+        externalTraceIdType: "EXTERNAL",
+      },
+    );
     expect(createSpan.status).toBe(200);
 
     const dbSpan = await prisma.observation.findFirstOrThrow({
@@ -148,15 +183,20 @@ describe("/api/public/spans API Endpoint", () => {
     const spanName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -189,17 +229,22 @@ describe("/api/public/spans API Endpoint", () => {
 
     const spanId = uuidv4();
     const traceId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceIdType: "EXTERNAL",
-      traceId: traceId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceIdType: "EXTERNAL",
+        traceId: traceId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
 
@@ -221,15 +266,20 @@ describe("/api/public/spans API Endpoint", () => {
     const generationName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: generationName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { key: "value" },
-      metadata: { key: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: generationName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { key: "value" },
+        metadata: { key: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbSpan = await prisma.observation.findFirstOrThrow({
       where: {
@@ -262,22 +312,32 @@ describe("/api/public/spans API Endpoint", () => {
     const spanName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
 
-    const updatedSpan = await makeAPICall("PATCH", "/api/public/spans", {
-      spanId: spanId,
-      output: { key: "this is a great gpt output" },
-    });
+    const updatedSpan = await makeZodVerifiedAPICall(
+      PatchSpansV1Response,
+      "PATCH",
+      "/api/public/spans",
+      {
+        spanId: spanId,
+        output: { key: "this is a great gpt output" },
+      },
+    );
     expect(updatedSpan.status).toBe(200);
 
     const dbSpan = await prisma.observation.findUnique({

web/src/__tests__/test-utils.ts

@@ -3,7 +3,7 @@ import { hash } from "bcryptjs";
 import { env } from "@/src/env.mjs";
 import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type z, ZodObject } from "zod";
+import { type z } from "zod";
 
 export const pruneDatabase = async () => {
   if (!env.DATABASE_URL.includes("localhost:5432")) {
@@ -14,6 +14,7 @@ export const pruneDatabase = async () => {
   await prisma.scoreConfig.deleteMany();
   await prisma.observation.deleteMany();
   await prisma.trace.deleteMany();
+  await prisma.traceSession.deleteMany();
   await prisma.datasetItem.deleteMany();
   await prisma.dataset.deleteMany();
   await prisma.datasetRuns.deleteMany();
@@ -82,10 +83,17 @@ export async function makeZodVerifiedAPICall<T extends z.ZodTypeAny>(
   auth?: string,
 ): Promise<{ body: z.infer<T>; status: number }> {
   const { body: resBody, status } = await makeAPICall(method, url, body, auth);
-  if (responseZodSchema instanceof ZodObject) {
-    responseZodSchema.strict().parse(resBody);
-  } else {
-    responseZodSchema.parse(resBody);
+  if (status !== 200) {
+    throw new Error(
+      `API call did not return 200, returned status ${status}, body ${JSON.stringify(resBody)}`,
+    );
+  }
+  const typeCheckResult = responseZodSchema.safeParse(resBody);
+  if (!typeCheckResult.success) {
+    console.error(typeCheckResult.error);
+    throw new Error(
+      `API call (${method} ${url}) did not return valid response, returned status ${status}, body ${JSON.stringify(resBody)}, error ${typeCheckResult.error}`,
+    );
   }
   return { body: resBody, status };
 }

web/src/__tests__/tokenCost.servertest.ts

@@ -126,30 +126,119 @@ describe("Token Cost Calculation", () => {
 
   it("should correctly calculate token costs when only some user provided costs are given", async () => {
     const model = {
-      inputPrice: new Decimal(0.01),
-      outputPrice: new Decimal(0.02),
-      totalPrice: new Decimal(0.03),
+      inputPrice: new Decimal(1),
+      outputPrice: new Decimal(1),
     };
 
-    const tokenCounts = {
-      input: 100,
-      output: 200,
-      total: undefined,
-    };
+    const data = [
+      // missing total
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: 2,
+          total: undefined,
+        },
+        expectedCost: {
+          input: 1,
+          output: 2,
+          total: 3,
+        },
+      },
+      // only total
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: undefined,
+          total: 2,
+        },
+        expectedCost: {
+          input: undefined,
+          output: undefined,
+          total: 2,
+        },
+      },
+      // missing input
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+        expectedCost: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+      },
+      // only input
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: undefined,
+          total: undefined,
+        },
+        expectedCost: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+      },
 
-    const userProvidedCosts = {
-      outputCost: new Decimal(3.0),
-    };
+      // missing output
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+        expectedCost: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+      },
 
-    const costs = ObservationProcessor.calculateTokenCosts(
-      model as any,
-      userProvidedCosts,
-      tokenCounts,
-    );
+      // only output
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: 2,
+          total: undefined,
+        },
+        expectedCost: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+      },
+    ];
 
-    expect(costs.inputCost).toBe(undefined); // No user provided cost
-    expect(costs.outputCost?.toNumber()).toBe(3.0); // Overridden by user provided cost
-    expect(costs.totalCost).toBe(undefined); // No user provided cost
+    for (const { userProvidedCosts, expectedCost } of data) {
+      const tokenCounts = {
+        input: 0,
+        output: 0,
+        total: 0,
+      };
+
+      const userProvidedCostsDecimal = {
+        inputCost:
+          userProvidedCosts.input && new Decimal(userProvidedCosts.input),
+        outputCost:
+          userProvidedCosts.output && new Decimal(userProvidedCosts.output),
+        totalCost:
+          userProvidedCosts.total && new Decimal(userProvidedCosts.total),
+      };
+
+      const costs = ObservationProcessor.calculateTokenCosts(
+        model as any,
+        userProvidedCostsDecimal as any,
+        tokenCounts,
+      );
+
+      expect(costs.inputCost?.toNumber()).toBe(expectedCost.input);
+      expect(costs.outputCost?.toNumber()).toBe(expectedCost.output);
+      expect(costs.totalCost?.toNumber()).toBe(expectedCost.total);
+    }
   });
 
   it("should return empty costs if no model is provided", async () => {
@@ -911,7 +1000,7 @@ describe("Token Cost Calculation", () => {
     expect(generation?.calculatedOutputCost?.toNumber()).toBe(
       generationUsage2.usage.outputCost,
     );
-    expect(generation?.calculatedTotalCost?.toNumber()).toBe(undefined);
+    expect(generation?.calculatedTotalCost?.toNumber()).toBe(1);
     expect(generation?.promptTokens).toBe(generationUsage1.usage.input);
     expect(generation?.completionTokens).toBe(generationUsage1.usage.output);
     expect(generation?.totalTokens).toBe(generationUsage1.usage.total);

web/src/__tests__/traces.servertest.ts

@@ -1,58 +1,70 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import { PostGenerationsV1Response } from "@/src/features/public-api/types/generations";
+import { PostScoresResponse } from "@/src/features/public-api/types/scores";
+import {
+  GetTraceV1Response,
+  GetTracesV1Response,
+  PostTracesV1Response,
+} from "@/src/features/public-api/types/traces";
 import { prisma } from "@langfuse/shared/src/db";
 import { v4 as uuidv4 } from "uuid";
 
-interface GetTracesAPIResponse {
-  data: Array<{
-    id: string;
-    [key: string]: unknown;
-  }>;
-}
-
 describe("/api/public/traces API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
-  it("should create", async () => {
+  it("should create and get a trace via /traces", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
+    const traceCreate = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
         name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
       },
-    });
+       );
 
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.name).toBe("trace-name");
-    expect(dbTrace[0]?.release).toBe("1.0.0");
-    expect(dbTrace[0]?.externalId).toBeNull();
-    expect(dbTrace[0]?.version).toBe("2.0.0");
-    expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    const trace = await makeZodVerifiedAPICall(
+      GetTraceV1Response,
+      "GET",
+      "/api/public/traces/" + traceCreate.body.id,
+    );
+
+    expect(trace.body.name).toBe("trace-name");
+    expect(trace.body.release).toBe("1.0.0");
+    expect(trace.body.externalId).toBeNull();
+    expect(trace.body.version).toBe("2.0.0");
+    expect(trace.body.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
   });
 
   it("should upsert second trace", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-id",
-      name: "trace-name",
-      userId: "user-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-      public: true,
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-id",
+        name: "trace-name",
+        userId: "user-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+        public: true,
+      },
+       );
 
     const dbTrace1 = await prisma.trace.findFirst({
       where: {
@@ -70,14 +82,19 @@ describe("/api/public/traces API Endpoint", () => {
       userId: "user-1",
     });
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-id",
-      metadata: { key: "value" },
-      timestamp: "2021-01-01T00:00:00.000Z",
-      release: "1.0.0",
-      version: "5.0.0",
-      public: false,
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-id",
+        metadata: { key: "value" },
+        timestamp: "2021-01-01T00:00:00.000Z",
+        release: "1.0.0",
+        version: "5.0.0",
+        public: false,
+      },
+       );
 
     const dbTrace2 = await prisma.trace.findFirst({
       where: {
@@ -100,23 +117,39 @@ describe("/api/public/traces API Endpoint", () => {
   it("should use tags correctly on POST and GET", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-1",
-      tags: ["tag-1", "tag-2", "tag-3"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-1",
+        tags: ["tag-1", "tag-2", "tag-3"],
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-2",
-      tags: ["tag-1"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-2",
+        tags: ["tag-1"],
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-3",
-      tags: ["tag-2", "tag-3"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-3",
+        tags: ["tag-2", "tag-3"],
+      },
+       );
 
     // multiple tags
-    const traces = await makeAPICall<GetTracesAPIResponse>(
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-2&tags=tag-3",
     );
@@ -125,7 +158,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds).toEqual(["trace-3", "trace-1"]);
 
     // single tag
-    const traces2 = await makeAPICall<GetTracesAPIResponse>(
+    const traces2 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-1",
     );
@@ -134,7 +168,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds2).toEqual(["trace-2", "trace-1"]);
 
     // wrong tag
-    const traces3 = await makeAPICall<GetTracesAPIResponse>(
+    const traces3 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-10",
     );
@@ -143,7 +178,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds3).toEqual([]);
 
     // no tag
-    const traces4 = await makeAPICall<GetTracesAPIResponse>(
+    const traces4 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=",
     );
@@ -157,60 +193,94 @@ describe("/api/public/traces API Endpoint", () => {
 
     // Create a trace with some observations that have costs and latencies
     const traceId = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-with-costs",
-      userId: "user-costs",
-      projectId: "project-costs",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-with-costs",
+        userId: "user-costs",
+        projectId: "project-costs",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+       );
     console.log(traceId);
 
     // Simulate observations with costs and latencies
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId,
-      usage: { totalCost: 10.5 },
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:10:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId,
-      usage: { totalCost: 5.25 },
-      startTime: "2021-01-01T00:10:00.000Z",
-      endTime: "2021-01-01T00:20:00.000Z",
-    });
+    const generationId = uuidv4();
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId,
+        id: generationId,
+        name: "Generation1",
+        usage: { totalCost: 10.5 },
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:10:00.000Z",
+      },
+       );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId,
+        usage: { totalCost: 5.25 },
+        startTime: "2021-01-01T00:10:00.000Z",
+        endTime: "2021-01-01T00:20:00.000Z",
+      },
+       );
 
     // Simulate scores on the trace
     const scoreId1 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId1,
-      name: "score-1",
-      value: 75.0,
-      traceId: traceId,
-      comment: "First score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId1,
+        name: "score-1",
+        value: 75.0,
+        traceId: traceId,
+        comment: "First score",
+      },
+       );
     const scoreId2 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId2,
-      name: "score-2",
-      value: 85.5,
-      traceId: traceId,
-      comment: "Second score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId2,
+        name: "score-2",
+        value: 85.5,
+        traceId: traceId,
+        comment: "Second score",
+      },
+       );
     const scoreId3 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId3,
-      name: "score-3",
-      value: 95.0,
-      traceId: traceId,
-      comment: "Third score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId3,
+        name: "score-3",
+        value: 95.0,
+        traceId: traceId,
+        comment: "Third score",
+      },
+       );
 
     // GET traces
     // Retrieve the trace with totalCost and latency
-    const traces = await makeAPICall<GetTracesAPIResponse>(
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces`,
     );
@@ -226,17 +296,23 @@ describe("/api/public/traces API Endpoint", () => {
 
     // GET trace
     // Retrieve the trace with total
-    const trace = await makeAPICall<{
-      id: string;
-      totalCost: number;
-      htmlPath: string;
-    }>("GET", `/api/public/traces/${traceId}`);
+    const trace = await makeZodVerifiedAPICall(
+      GetTraceV1Response,
+      "GET",
+      `/api/public/traces/${traceId}`,
+    );
     console.log(trace.body);
     expect(trace.body.totalCost).toBeCloseTo(15.75);
     expect(trace.body.id).toBe(traceId);
     expect(trace.body.id).toBe(traceId);
     expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
     expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
+    expect(trace.body.scores).toHaveLength(3);
+    expect(trace.body.scores[0].id).toBe(scoreId3);
+    expect(trace.body.scores[0].name).toBe("score-3");
+    expect(trace.body.observations).toHaveLength(2);
+    expect(trace.body.observations[0].id).toBe(generationId);
+    expect(trace.body.observations[0].name).toBe("Generation1");
   });
 
   it("should filter traces by session ID", async () => {
@@ -244,30 +320,41 @@ describe("/api/public/traces API Endpoint", () => {
     const anotherSessionId = "another-session-id";
 
     // Create traces with different session IDs
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-1",
-      name: "test-trace-1",
-      sessionId,
-      userId: "user-1",
-      projectId: "project-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "1.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-1",
+        name: "test-trace-1",
+        sessionId,
+        userId: "user-1",
+        projectId: "project-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "1.0.0",
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-2",
-      name: "test-trace-2",
-      sessionId: anotherSessionId,
-      userId: "user-2",
-      projectId: "project-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "1.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-2",
+        name: "test-trace-2",
+        sessionId: anotherSessionId,
+        userId: "user-2",
+        projectId: "project-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "1.0.0",
+      },
+       );
 
     // Filter by session ID
-    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+    const tracesBySessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces?sessionId=${sessionId}`,
     );
@@ -277,7 +364,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
 
     // Filter by another session ID
-    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+    const tracesByAnotherSessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces?sessionId=${anotherSessionId}`,
     );
@@ -287,11 +375,11 @@ describe("/api/public/traces API Endpoint", () => {
     expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
 
     // Filter by non-existent session ID
-    const tracesByNonExistentSessionId =
-      await makeAPICall<GetTracesAPIResponse>(
-        "GET",
-        `/api/public/traces?sessionId=non-existent-session-id`,
-      );
+    const tracesByNonExistentSessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      `/api/public/traces?sessionId=non-existent-session-id`,
+    );
 
     expect(tracesByNonExistentSessionId.status).toBe(200);
     expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);

web/src/components/BatchExportTableButton.tsx

@@ -15,6 +15,7 @@ import {
 import React from "react";
 import { api } from "@/src/utils/api";
 import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 export type BatchExportTableButtonProps = {
@@ -31,6 +32,10 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
   const [isExporting, setIsExporting] = React.useState(false);
   const createExport = api.batchExport.create.useMutation();
   const isEeEnabled = useIsEeEnabled();
+  const hasAccess = useHasAccess({
+    projectId: props.projectId,
+    scope: "batchExport:create",
+  });
 
   const handleExport = async (format: BatchExportFileFormat) => {
     setIsExporting(true);
@@ -51,7 +56,7 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
     });
   };
 
-  if (!isEeEnabled) return null;
+  if (!isEeEnabled || !hasAccess) return null;
 
   return (
     <DropdownMenu>

web/src/components/date-picker.tsx

@@ -29,6 +29,7 @@ import { useMediaQuery } from "react-responsive";
 import { type DashboardDateRange } from "@/src/pages/project/[projectId]";
 import { isValidOption } from "@/src/utils/types";
 import { setBeginningOfDay, setEndOfDay } from "@/src/utils/dates";
+import { TimePicker } from "@/src/components/ui/time-picker";
 
 export const DEFAULT_DATE_RANGE_SELECTION = "Date range" as const;
 export type AvailableDateRangeSelections =
@@ -41,12 +42,14 @@ export function DatePicker({
   clearable = false,
   className,
   disabled,
+  includeTimePicker,
 }: {
   date?: Date | undefined;
   onChange: (date: Date | undefined) => void;
   clearable?: boolean;
   className?: string;
   disabled?: boolean;
+  includeTimePicker?: boolean;
 }) {
   return (
     <div className="flex flex-row gap-2 align-middle">
@@ -62,7 +65,11 @@ export function DatePicker({
             )}
           >
             <CalendarIcon className="mr-2 h-4 w-4" />
-            {date ? format(date, "PPP") : <span>Pick a date</span>}
+            {date ? (
+              format(date, includeTimePicker ? "PPP pp" : "PPP")
+            ) : (
+              <span>Pick a date</span>
+            )}
           </Button>
         </PopoverTrigger>
         <PopoverContent className="w-auto p-0">
@@ -72,6 +79,9 @@ export function DatePicker({
             onSelect={(d) => onChange(d)}
             initialFocus
           />
+          {includeTimePicker && (
+            <TimePicker date={date} setDate={(d) => onChange(d)} />
+          )}
         </PopoverContent>
       </Popover>
       {date && clearable && (

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
         <span key={i} className="group/score ml-1 first:ml-0">
           {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
             ? s.stringValue
-            : s.value.toFixed(2)}
+            : s.value?.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">

web/src/components/schemas/ChatMlSchema.ts

@@ -0,0 +1,25 @@
+import { z } from "zod";
+
+export const ChatMlMessageSchema = z
+  .object({
+    role: z.string().optional(),
+    name: z.string().optional(),
+    content: z
+      .union([z.record(z.any()), z.string(), z.array(z.any())])
+      .nullish(),
+    additional_kwargs: z.record(z.any()).optional(),
+  })
+  .passthrough()
+  .refine((value) => value.content !== null || value.role !== undefined)
+  .transform(({ additional_kwargs, ...other }) => ({
+    ...other,
+    ...additional_kwargs,
+  }))
+  .transform(({ role, name, content, ...other }) => ({
+    role,
+    name,
+    content,
+    json: Object.keys(other).length === 0 ? undefined : other,
+  }));
+
+export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);

web/src/components/schemas/MarkdownSchema.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+const MARKDOWN_PATTERNS = [
+  "(\\*\\*?|__?)(.*?)\\1", // Matches bold (** or __) and italic (* or _) with proper escaping
+  "`{3}[\\s\\S]*?`{3}", // Matches fenced code blocks with triple backticks
+  "`[\\s\\S]*?`", // Matches inline code with single backticks
+  "(^|\\s)[-+*]\\s", // Matches unordered lists that start with -, +, or *
+  "^\\s*#{1,6}\\s", // Matches headers that start with # to ######
+  "^>\\s+", // Matches blockquotes starting with >
+  "^\\d+\\.\\s", // Matches ordered lists starting with 1. or 2. etc
+  "!\\[.*?\\]\\(.*?\\)", // Matches images ![Alt text](URL)
+  "\\[.*?\\]\\(.*?\\)", // Matches links [Link text](URL)
+].join("|");
+
+const MARKDOWN_REGEX = new RegExp(MARKDOWN_PATTERNS, "gm");
+
+export function containsAnyMarkdown(...texts: string[]): boolean {
+  MARKDOWN_REGEX.lastIndex = 0;
+  return texts.some((text) => MARKDOWN_REGEX.test(text));
+}
+
+export const MarkdownSchema = z.string().refine(containsAnyMarkdown);

web/src/components/session/index.tsx

@@ -132,6 +132,7 @@ export const SessionPage: React.FC<{
                 variant="badge"
                 type="session"
                 source="SessionDetail"
+                key={"annotation-drawer" + trace.id}
               />
             </div>
           </Card>

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -38,7 +38,7 @@ export function TraceTableMultiSelectAction({
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
     onSuccess: () => {
       onDeleteSuccess();
-      void utils.traces.invalidate();
+      void utils.traces.all.invalidate();
     },
   });
 

web/src/components/table/use-cases/generations.tsx

@@ -21,18 +21,20 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import {
-  type Prisma,
   type ObservationLevel,
   type FilterState,
   type ObservationOptions,
 } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type BatchExportFileFormat,
@@ -44,7 +46,7 @@ import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAl
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 
 export type GenerationsTableRow = {
   id: string;
@@ -58,14 +60,14 @@ export type GenerationsTableRow = {
   timeToFirstToken?: number;
   name?: string;
   model?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
+  metadata?: unknown;
   inputCost?: Decimal;
   outputCost?: Decimal;
   totalCost?: Decimal;
   traceName?: string;
-  metadata?: Prisma.JsonValue;
   scores?: ScoreSimplified[];
   usage: {
     promptTokens: number;
@@ -113,7 +115,7 @@ export default function GenerationsTable({
         column: "Start Time",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "generations",
@@ -454,7 +456,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
     },
     {
@@ -470,7 +472,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
     },
     {
@@ -486,7 +488,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
     },
     {
@@ -519,10 +521,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -538,10 +540,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -553,12 +555,16 @@ export default function GenerationsTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values = row.getValue(
-          "metadata",
-        ) as GenerationsTableRow["metadata"];
-        return !!values ? (
-          <IOTableCell data={values} singleLine={rowHeight === "s"} />
-        ) : null;
+        const observationId: string = row.getValue("id");
+        const traceId: string = row.getValue("traceId");
+        return (
+          <GenerationsDynamicCell
+            observationId={observationId}
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
       defaultHidden: true,
@@ -617,7 +623,6 @@ export default function GenerationsTable({
           model: generation.model ?? "",
           scores: generation.scores,
           level: generation.level,
-          metadata: generation.metadata,
           statusMessage: generation.statusMessage ?? undefined,
           usage: {
             promptTokens: generation.promptTokens,
@@ -712,15 +717,15 @@ export default function GenerationsTable({
   );
 }
 
-const GenerationsIOCell = ({
+const GenerationsDynamicCell = ({
   traceId,
   observationId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
   observationId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine: boolean;
 }) => {
   const observation = api.observations.byId.useQuery(
@@ -742,9 +747,13 @@ const GenerationsIOCell = ({
     <IOTableCell
       isLoading={observation.isLoading}
       data={
-        io === "output" ? observation.data?.output : observation.data?.input
+        col === "output"
+          ? observation.data?.output
+          : col === "input"
+            ? observation.data?.input
+            : observation.data?.metadata
       }
-      className={cn(io === "output" && "bg-accent-light-green")}
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/table/use-cases/models.tsx

@@ -43,7 +43,7 @@ const modelConfigDescriptions = {
   outputPrice: "Price per 1000 units of output",
   totalPrice:
     "Price per 1000 units, for models that don't have input/output specific prices",
-  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, or IMAGES.",
+  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, REQUESTS or IMAGES.",
   tokenizerId:
     "Tokenizer used for this model to calculate token counts if none are ingested. Pick from list of supported tokenizers.",
   config:

web/src/components/table/use-cases/score-configs.tsx

@@ -6,11 +6,7 @@ import { api } from "@/src/utils/api";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import { DataTable } from "@/src/components/table/data-table";
-import {
-  type ScoreDataType,
-  type Prisma,
-  type ConfigCategory,
-} from "@langfuse/shared";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 import {
@@ -28,6 +24,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { type ConfigCategory } from "@/src/features/public-api/types/score-configs";
 
 type ScoreConfigTableRow = {
   id: string;
@@ -38,7 +35,7 @@ type ScoreConfigTableRow = {
   range: {
     maxValue?: number | null;
     minValue?: number | null;
-    categories?: Prisma.JsonValue | null;
+    categories?: ConfigCategory[] | null;
   };
   description?: string | null;
   isArchived: boolean;
@@ -46,7 +43,7 @@ type ScoreConfigTableRow = {
 
 function getConfigRange(
   originalRow: ScoreConfigTableRow,
-): Prisma.JsonValue | undefined {
+): undefined | Prisma.JsonValue {
   const { range, dataType } = originalRow;
 
   if (isNumericDataType(dataType)) {
@@ -57,7 +54,7 @@ function getConfigRange(
   }
 
   if (isCategoricalDataType(dataType) || isBooleanDataType(dataType)) {
-    const configCategories = (range.categories as ConfigCategory[]) ?? [];
+    const configCategories = range.categories ?? [];
 
     return configCategories.reduce(
       (acc, category) => {

web/src/components/table/use-cases/scores.tsx

@@ -9,13 +9,14 @@ import useColumnVisibility from "@/src/features/column-visibility/hooks/useColum
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 import {
   type ScoreOptions,
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { utcDateOffsetByDays } from "@/src/utils/dates";
+import { localtimeDateOffsetByDays } from "@/src/utils/dates";
+import { isPresent } from "@/src/utils/typeChecks";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
@@ -90,7 +91,7 @@ export default function ScoresTable({
         column: "Timestamp",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "scores",
@@ -116,9 +117,18 @@ export default function ScoresTable({
   });
   const totalCount = scores.data?.totalCount ?? 0;
 
-  const filterOptions = api.scores.filterOptions.useQuery({
-    projectId,
-  });
+  const filterOptions = api.scores.filterOptions.useQuery(
+    {
+      projectId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
 
   const rawColumns: LangfuseColumnDef<ScoresTableRow>[] = [
     {
@@ -310,11 +320,12 @@ export default function ScoresTable({
       source: score.source,
       name: score.name,
       dataType: score.dataType,
-      value: isNumericDataType(score.dataType)
-        ? score.value % 1 === 0
-          ? String(score.value)
-          : score.value.toFixed(4)
-        : score.stringValue ?? "",
+      value:
+        isNumericDataType(score.dataType) && isPresent(score.value)
+          ? score.value % 1 === 0
+            ? String(score.value)
+            : score.value.toFixed(4)
+          : score.stringValue ?? "",
       author: {
         image: score.authorUserImage ?? undefined,
         name: score.authorUserName ?? undefined,

web/src/components/table/use-cases/sessions.tsx

@@ -14,13 +14,16 @@ import {
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
-import { usdFormatter } from "@/src/utils/numbers";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
 import type Decimal from "decimal.js";
 import { useEffect } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 import { BatchExportTableButton } from "@/src/components/BatchExportTableButton";
 
 export type SessionTableRow = {
@@ -57,7 +60,7 @@ export default function SessionsTable({
         column: "Created At",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "sessions",
@@ -276,7 +279,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value: number | undefined = row.getValue("inputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -289,7 +294,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value = row.getValue("outputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -301,7 +308,9 @@ export default function SessionsTable({
       enableSorting: true,
       cell: ({ row }) => {
         const value = row.getValue("totalTokens");
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {

web/src/components/table/use-cases/traces.tsx

@@ -11,7 +11,10 @@ import { Checkbox } from "@/src/components/ui/checkbox";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
@@ -23,7 +26,7 @@ import {
   withDefault,
 } from "use-query-params";
 import type Decimal from "decimal.js";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { DeleteButton } from "@/src/components/deleteButton";
 import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
@@ -34,11 +37,11 @@ import {
   type TraceOptions,
   tracesTableColsWithOptions,
   type ObservationLevel,
-  type Score,
 } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -46,22 +49,22 @@ export type TracesTableRow = {
   timestamp: string;
   name: string;
   userId: string;
-  metadata?: string;
   level: ObservationLevel;
   observationCount: number;
   latency?: number;
   release?: string;
   version?: string;
   sessionId?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
-  scores: Score[];
+  metadata?: unknown;
+  scores: APIScore[];
   tags: string[];
   usage: {
-    promptTokens: number;
-    completionTokens: number;
-    totalTokens: number;
+    promptTokens: bigint;
+    completionTokens: bigint;
+    totalTokens: bigint;
   };
   inputCost?: Decimal;
   outputCost?: Decimal;
@@ -95,7 +98,7 @@ export default function TracesTable({
         column: "Timestamp",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "traces",
@@ -128,7 +131,6 @@ export default function TracesTable({
     filter: filterState,
     searchQuery,
     orderBy: orderByState,
-    returnIO: false,
   };
   const traces = api.traces.all.useQuery(tracesAllQueryFilter);
 
@@ -180,7 +182,6 @@ export default function TracesTable({
       name: trace.name ?? "",
       level: trace.level,
       observationCount: trace.observationCount,
-      metadata: JSON.stringify(trace.metadata),
       release: trace.release ?? undefined,
       version: trace.version ?? undefined,
       userId: trace.userId ?? "",
@@ -238,7 +239,8 @@ export default function TracesTable({
       header: undefined,
       id: "bookmarked",
       cell: ({ row }) => {
-        const bookmarked = row.getValue("bookmarked");
+        const bookmarked: TracesTableRow["bookmarked"] =
+          row.getValue("bookmarked");
         const traceId = row.getValue("id");
         return typeof traceId === "string" &&
           typeof bookmarked === "boolean" ? (
@@ -258,7 +260,7 @@ export default function TracesTable({
       header: "ID",
       id: "id",
       cell: ({ row }) => {
-        const value = row.getValue("id");
+        const value: TracesTableRow["id"] = row.getValue("id");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/traces/${value}`}
@@ -287,7 +289,7 @@ export default function TracesTable({
       header: "User ID",
       id: "userId",
       cell: ({ row }) => {
-        const value = row.getValue("userId");
+        const value: TracesTableRow["userId"] = row.getValue("userId");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/users/${encodeURIComponent(value)}`}
@@ -305,7 +307,7 @@ export default function TracesTable({
       id: "sessionId",
       header: "Session ID",
       cell: ({ row }) => {
-        const value = row.getValue("sessionId");
+        const value: TracesTableRow["sessionId"] = row.getValue("sessionId");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/sessions/${encodeURIComponent(value)}`}
@@ -323,7 +325,7 @@ export default function TracesTable({
       header: "Latency",
       // add seconds to the end of the latency
       cell: ({ row }) => {
-        const value: number | undefined = row.getValue("latency");
+        const value: TracesTableRow["latency"] = row.getValue("latency");
         return value !== undefined ? formatIntervalSeconds(value) : undefined;
       },
       enableHiding: true,
@@ -334,12 +336,8 @@ export default function TracesTable({
       id: "inputTokens",
       header: "Input Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -350,12 +348,8 @@ export default function TracesTable({
       id: "outputTokens",
       header: "Output Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -366,12 +360,8 @@ export default function TracesTable({
       id: "totalTokens",
       header: "Total Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -383,11 +373,7 @@ export default function TracesTable({
       header: "Usage",
       id: "usage",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
+        const value: TracesTableRow["usage"] = row.getValue("usage");
         return (
           <TokenUsageBadge
             promptTokens={value.promptTokens}
@@ -405,7 +391,7 @@ export default function TracesTable({
       id: "inputCost",
       header: "Input Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("inputCost");
+        const cost: TracesTableRow["inputCost"] = row.getValue("inputCost");
         return (
           <div>
             {cost ? (
@@ -425,7 +411,7 @@ export default function TracesTable({
       id: "outputCost",
       header: "Output Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("outputCost");
+        const cost: TracesTableRow["outputCost"] = row.getValue("outputCost");
         return (
           <div>
             {cost ? (
@@ -445,7 +431,7 @@ export default function TracesTable({
       id: "totalCost",
       header: "Total Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("totalCost");
+        const cost: TracesTableRow["totalCost"] = row.getValue("totalCost");
         return (
           <div>
             {cost ? (
@@ -465,7 +451,7 @@ export default function TracesTable({
       header: "Scores",
       enableColumnFilter: !omittedFilter.find((f) => f === "scores"),
       cell: ({ row }) => {
-        const values: Score[] = row.getValue("scores");
+        const values: TracesTableRow["scores"] = row.getValue("scores");
         return <GroupedScoreBadges scores={values} variant="headings" />;
       },
       enableHiding: true,
@@ -475,11 +461,11 @@ export default function TracesTable({
       header: "Input",
       id: "input",
       cell: ({ row }) => {
-        const traceId: string = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -492,11 +478,11 @@ export default function TracesTable({
       header: "Output",
       id: "output",
       cell: ({ row }) => {
-        const traceId: string = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -508,17 +494,24 @@ export default function TracesTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values: string = row.getValue("metadata");
-        return <IOTableCell data={values} singleLine={rowHeight === "s"} />;
+        const traceId: TracesTableRow["id"] = row.getValue("id");
+        return (
+          <TracesDynamicCell
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "level",
       id: "level",
       header: "Level",
       cell: ({ row }) => {
-        const value: ObservationLevel = row.getValue("level");
+        const value: TracesTableRow["level"] = row.getValue("level");
         return (
           <span
             className={cn(
@@ -563,8 +556,8 @@ export default function TracesTable({
       id: "tags",
       header: "Tags",
       cell: ({ row }) => {
-        const tags: string[] = row.getValue("tags");
-        const traceId: string = row.getValue("id");
+        const tags: TracesTableRow["tags"] = row.getValue("tags");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         const filterOptionTags = traceFilterOptions.data?.tags ?? [];
         const allTags = filterOptionTags.map((t) => t.value);
         return (
@@ -583,13 +576,13 @@ export default function TracesTable({
       accessorKey: "action",
       header: "Action",
       cell: ({ row }) => {
-        const traceId = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return traceId && typeof traceId === "string" ? (
           <DeleteButton
             itemId={traceId}
             projectId={projectId}
             scope="traces:delete"
-            invalidateFunc={() => void utils.traces.invalidate()}
+            invalidateFunc={() => void utils.traces.all.invalidate()}
             type="trace"
             isTableAction={true}
           />
@@ -668,13 +661,13 @@ export default function TracesTable({
   );
 }
 
-const TracesIOCell = ({
+const TracesDynamicCell = ({
   traceId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine?: boolean;
 }) => {
   const trace = api.traces.byId.useQuery(
@@ -692,8 +685,14 @@ const TracesIOCell = ({
   return (
     <IOTableCell
       isLoading={trace.isLoading}
-      data={io === "output" ? trace.data?.output : trace.data?.input}
-      className={cn(io === "output" && "bg-accent-light-green")}
+      data={
+        col === "output"
+          ? trace.data?.output
+          : col === "input"
+            ? trace.data?.input
+            : trace.data?.metadata
+      }
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/token-usage-badge.tsx

@@ -1,5 +1,6 @@
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+import { numberFormatter } from "@/src/utils/numbers";
 import { type Observation } from "@langfuse/shared";
 
 export const TraceAggUsageBadge = (props: {
@@ -25,9 +26,9 @@ export const TokenUsageBadge = (
         observation: Observation;
       }
     | {
-        promptTokens: number;
-        completionTokens: number;
-        totalTokens: number;
+        promptTokens: number | bigint;
+        completionTokens: number | bigint;
+        totalTokens: number | bigint;
       }
   ) & {
     inline?: boolean;
@@ -52,13 +53,13 @@ export const TokenUsageBadge = (
   if (props.inline)
     return (
       <span>
-        {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+        {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
       </span>
     );
 
   return (
     <Badge variant="outline">
-      {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+      {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
     </Badge>
   );
 };

web/src/components/trace/IOPreview.tsx

@@ -1,16 +1,60 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { z } from "zod";
-import { deepParseJson } from "@langfuse/shared";
+import { type Prisma, deepParseJson } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
-import { useState } from "react";
+import { useMemo, useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { Fragment } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { MarkdownView } from "@/src/components/ui/MarkdownViewer";
+import { MarkdownSchema } from "@/src/components/schemas/MarkdownSchema";
+import {
+  ChatMlArraySchema,
+  ChatMlMessageSchema,
+} from "@/src/components/schemas/ChatMlSchema";
+import { useMarkdownContext } from "@/src/features/theming/useMarkdownContext";
+
+// MarkdownOrJsonView will render markdown if `isMarkdownEnabled` (global context) is true and the content is valid markdown
+// otherwise, if content is valid markdown will render JSON with switch to enable markdown globally
+function MarkdownOrJsonView({
+  content,
+  title,
+  className,
+  customCodeHeaderClassName,
+}: {
+  content?: unknown;
+  title?: string;
+  className?: string;
+  customCodeHeaderClassName?: string;
+}) {
+  const validatedMarkdown = useMemo(
+    () => MarkdownSchema.safeParse(content),
+    [content],
+  );
+
+  const { isMarkdownEnabled } = useMarkdownContext();
+
+  return validatedMarkdown.success && isMarkdownEnabled ? (
+    <MarkdownView
+      markdown={validatedMarkdown.data}
+      title={title}
+      className={className}
+      customCodeHeaderClassName={customCodeHeaderClassName}
+    />
+  ) : (
+    <JSONView
+      json={content}
+      canEnableMarkdown={validatedMarkdown.success}
+      title={title}
+      className={className}
+    />
+  );
+}
 
 export const IOPreview: React.FC<{
-  input?: unknown;
-  output?: unknown;
+  input?: Prisma.JsonValue;
+  output?: Prisma.JsonValue;
   isLoading?: boolean;
   hideIfNull?: boolean;
 }> = ({ isLoading = false, hideIfNull = false, ...props }) => {
@@ -59,43 +103,67 @@ export const IOPreview: React.FC<{
     Array.isArray(output) ? output : [output],
   );
 
-  // Pretty view available
-  const isPrettyViewAvailable = inChatMlArray.success;
+  const inMarkdown = MarkdownSchema.safeParse(input);
+  const outMarkdown = MarkdownSchema.safeParse(output);
+
+  const isPrettyViewAvailable =
+    inChatMlArray.success || inMarkdown.success || outMarkdown.success;
 
   // default I/O
   return (
     <>
       {isPrettyViewAvailable ? (
-        <Tabs
-          value={currentView}
-          onValueChange={(v) => {
-            setCurrentView(v as "pretty" | "json"),
-              capture("trace_detail:io_mode_switch", { view: v });
-          }}
-        >
-          <TabsList>
-            <TabsTrigger value="pretty">Pretty ✨</TabsTrigger>
-            <TabsTrigger value="json">JSON</TabsTrigger>
-          </TabsList>
-        </Tabs>
+        <div className="flex flex-row justify-between">
+          <Tabs
+            value={currentView}
+            onValueChange={(v) => {
+              setCurrentView(v as "pretty" | "json"),
+                capture("trace_detail:io_mode_switch", { view: v });
+            }}
+          >
+            <TabsList>
+              <TabsTrigger value="pretty">Pretty ✨</TabsTrigger>
+              <TabsTrigger value="json">JSON</TabsTrigger>
+            </TabsList>
+          </Tabs>
+        </div>
       ) : null}
       {isPrettyViewAvailable && currentView === "pretty" ? (
-        <OpenAiMessageView
-          messages={[
-            ...inChatMlArray.data,
-            ...(outChatMlArray.success
-              ? outChatMlArray.data.map((m) => ({
-                  ...m,
-                  role: m.role ?? "assistant",
-                }))
-              : [
-                  ChatMlMessageSchema.parse({
-                    role: "assistant",
-                    content: outputClean ? JSON.stringify(outputClean) : null,
-                  }),
-                ]),
-          ]}
-        />
+        <>
+          {inChatMlArray.success ? (
+            <OpenAiMessageView
+              messages={[
+                ...inChatMlArray.data,
+                ...(outChatMlArray.success
+                  ? outChatMlArray.data.map((m) => ({
+                      ...m,
+                      role: m.role ?? "assistant",
+                    }))
+                  : [
+                      ChatMlMessageSchema.parse({
+                        role: "assistant",
+                        content: outputClean,
+                      }),
+                    ]),
+              ]}
+              shouldRenderMarkdown
+            />
+          ) : (
+            <>
+              {!(hideIfNull && !input) ? (
+                <MarkdownOrJsonView title="Input" content={input} />
+              ) : null}
+              {!(hideIfNull && !output) ? (
+                <MarkdownOrJsonView
+                  title="Output"
+                  content={output}
+                  className="bg-accent-light-green dark:border-accent-dark-green"
+                  customCodeHeaderClassName="bg-muted-green dark:bg-secondary"
+                />
+              ) : null}
+            </>
+          )}
+        </>
       ) : null}
       {currentView === "json" || !isPrettyViewAvailable ? (
         <>
@@ -121,57 +189,23 @@ export const IOPreview: React.FC<{
   );
 };
 
-const ChatMlMessageSchema = z
-  .object({
-    role: z
-      .enum(["system", "user", "assistant", "function", "tool"])
-      .optional(),
-    name: z.string().optional(),
-    content: z
-      .union([z.record(z.any()), z.string(), z.array(z.any())])
-      .nullish(),
-    additional_kwargs: z.record(z.any()).optional(),
-  })
-  .passthrough()
-  .refine((value) => value.content !== null || value.role !== undefined)
-  .transform(({ additional_kwargs, ...other }) => ({
-    ...other,
-    ...additional_kwargs,
-  }))
-  .transform(({ role, name, content, ...other }) => ({
-    role,
-    name,
-    content,
-    json: Object.keys(other).length === 0 ? undefined : other,
-  }));
-export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
-
 export const OpenAiMessageView: React.FC<{
-  title?: string;
   messages: z.infer<typeof ChatMlArraySchema>;
-}> = ({ title, messages }) => {
+  title?: string;
+  shouldRenderMarkdown?: boolean;
+}> = ({ title, messages, shouldRenderMarkdown = false }) => {
   const COLLAPSE_THRESHOLD = 3;
   const [isCollapsed, setCollapsed] = useState(
     messages.length > COLLAPSE_THRESHOLD ? true : null,
   );
 
-  const transformedMessages = messages;
-  // const transformedMessages = messages.map(
-  //   ({ role, name, content, ...rest }) => ({
-  //     role,
-  //     name,
-  //     content,
-  //     json: rest,
-  //   }),
-  // );
-
   return (
     <div className="rounded-md border">
       {title && (
         <div className="border-b px-3 py-1 text-xs font-medium">{title}</div>
       )}
       <div className="flex flex-col gap-2 p-3">
-        {transformedMessages
+        {messages
           .filter(
             (_, i) =>
               // show all if not collapsed or null; show first and last n if collapsed
@@ -182,20 +216,38 @@ export const OpenAiMessageView: React.FC<{
           .map((message, index) => (
             <Fragment key={index}>
               <div>
-                {!!message.content && (
-                  <JSONView
-                    title={message.name ?? message.role}
-                    json={message.content}
-                    className={cn(
-                      "bg-muted",
-                      message.role === "system" && "bg-primary-foreground",
-                      message.role === "assistant" &&
-                        "bg-accent-light-green dark:border-accent-dark-green",
-                      message.role === "user" && "bg-background",
-                      !!message.json && "rounded-b-none",
-                    )}
-                  />
-                )}
+                {!!message.content &&
+                  (shouldRenderMarkdown ? (
+                    <MarkdownOrJsonView
+                      title={message.name ?? message.role}
+                      content={message.content}
+                      className={cn(
+                        "bg-muted",
+                        message.role === "system" && "bg-primary-foreground",
+                        message.role === "assistant" &&
+                          "bg-accent-light-green dark:border-accent-dark-green",
+                        message.role === "user" && "bg-background",
+                        !!message.json && "rounded-b-none",
+                      )}
+                      customCodeHeaderClassName={cn(
+                        message.role === "assistant" &&
+                          "bg-muted-green dark:bg-secondary",
+                      )}
+                    />
+                  ) : (
+                    <JSONView
+                      title={message.name ?? message.role}
+                      json={message.content}
+                      className={cn(
+                        "bg-muted",
+                        message.role === "system" && "bg-primary-foreground",
+                        message.role === "assistant" &&
+                          "bg-accent-light-green dark:border-accent-dark-green",
+                        message.role === "user" && "bg-background",
+                        !!message.json && "rounded-b-none",
+                      )}
+                    />
+                  ))}
                 {!!message.json && (
                   <JSONView
                     title={

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type ScoreSource, type Score } from "@langfuse/shared";
+import { type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -22,11 +22,12 @@ import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string;
   traceId: string;
 }) => {
@@ -59,7 +60,7 @@ export const ObservationPreview = (props: {
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, APIScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
@@ -147,7 +148,11 @@ export const ObservationPreview = (props: {
                     .filter(Boolean)
                     .map(([key, value]) => (
                       <Badge variant="outline" key={key}>
-                        {key}: {value?.toString()}
+                        {key}:{" "}
+                        {Object.prototype.toString.call(value) ===
+                        "[object Object]"
+                          ? JSON.stringify(value)
+                          : value?.toString()}
                       </Badge>
                     ))
                 : null}
@@ -160,6 +165,7 @@ export const ObservationPreview = (props: {
               observationId={preloadedObservation.id}
               scores={props.scores}
               type="observation"
+              key={"annotation-drawer" + preloadedObservation.id}
             />
             {observationWithInputAndOutput.data?.type === "GENERATION" && (
               <JumpToPlaygroundButton

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,7 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, type $Enums } from "@langfuse/shared";
+import { type Trace, type $Enums } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -22,7 +23,7 @@ export const ObservationTree = (props: {
   collapseAll: () => void;
   expandAll: () => void;
   trace: Trace;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics: boolean;
@@ -61,7 +62,7 @@ const ObservationTreeTraceNode = (props: {
   trace: Trace & { latency?: number };
   expandAll: () => void;
   collapseAll: () => void;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics?: boolean;
@@ -119,7 +120,7 @@ const ObservationTreeNode = (props: {
   observations: NestedObservation[];
   collapsedObservations: string[];
   toggleCollapsedObservation: (id: string) => void;
-  scores: Score[];
+  scores: APIScore[];
   indentationLevel: number;
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;

web/src/components/trace/ScoresPreview.tsx

@@ -1,10 +1,10 @@
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { type Score } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const ScoresPreview = ({
   itemScoresBySource,
 }: {
-  itemScoresBySource: Map<string, Score[]>;
+  itemScoresBySource: Map<string, APIScore[]>;
 }) => {
   if (!Boolean(itemScoresBySource.size)) return null;
 

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Trace, type Score, type ScoreSource } from "@langfuse/shared";
+import { type Trace, type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -19,6 +19,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const TracePreview = ({
   trace,
@@ -27,7 +28,7 @@ export const TracePreview = ({
 }: {
   trace: Trace & { latency?: number };
   observations: ObservationReturnType[];
-  scores: Score[];
+  scores: APIScore[];
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -41,7 +42,7 @@ export const TracePreview = ({
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, APIScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
@@ -99,6 +100,7 @@ export const TracePreview = ({
               projectId={trace.projectId}
               traceId={trace.id}
               scores={scores}
+              key={"annotation-drawer" + trace.id}
             />
             <NewDatasetItemFromTrace
               traceId={trace.id}

web/src/components/trace/TraceTimelineView.tsx

@@ -1,12 +1,19 @@
 import { Card } from "@/src/components/ui/card";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Score, type Trace } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
-import React, { useEffect, useRef, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";
 import { SimpleTreeView } from "@mui/x-tree-view/SimpleTreeView";
 import { TreeItem } from "@mui/x-tree-view/TreeItem";
 
-import { MinusIcon, PlusIcon, PanelRightOpen } from "lucide-react";
+import {
+  MinusIcon,
+  PlusIcon,
+  PanelRightOpen,
+  PlusSquareIcon,
+  MinusSquare,
+} from "lucide-react";
 import { nestObservations } from "@/src/components/trace/lib/helpers";
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
@@ -36,6 +43,24 @@ const PREDEFINED_STEP_SIZES = [
   0.25, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10,
 ];
 
+const getNestedObservationKeys = (
+  observations: NestedObservation[],
+): string[] => {
+  const keys: string[] = [];
+
+  const collectKeys = (obs: NestedObservation[]) => {
+    obs.forEach((observation) => {
+      keys.push(`observation-${observation.id}`);
+      if (observation.children) {
+        collectKeys(observation.children);
+      }
+    });
+  };
+
+  collectKeys(observations);
+  return keys;
+};
+
 const calculateStepSize = (latency: number, scaleWidth: number) => {
   const calculatedStepSize = latency / (scaleWidth / STEP_SIZE);
   return (
@@ -163,7 +188,7 @@ function TraceTreeItem({
   traceStartTime: Date;
   totalScaleSpan: number;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
   observations: Array<ObservationReturnType>;
   cardWidth: number;
 }) {
@@ -242,7 +267,7 @@ export function TraceTimelineView({
   trace: Trace & { latency?: number };
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
 }) {
   const { latency, name, id } = trace;
   const [backgroundColor, setBackgroundColor] = useState("");
@@ -270,9 +295,17 @@ export function TraceTimelineView({
     };
   }, [parentRef]);
 
+  const nestedObservations = useMemo(
+    () => nestObservations(observations),
+    [observations],
+  );
+  const nestedObservationKeys = useMemo(
+    () => getNestedObservationKeys(nestedObservations),
+    [nestedObservations],
+  );
+
   if (!latency) return null;
 
-  const nestedObservations = nestObservations(observations);
   const stepSize = calculateStepSize(latency, SCALE_WIDTH);
   const totalScaleSpan = stepSize * (SCALE_WIDTH / STEP_SIZE);
 
@@ -283,14 +316,39 @@ export function TraceTimelineView({
         style={{ width: cardWidth }}
       >
         <div className="grid w-full grid-cols-[1fr,auto] items-center p-2">
-          <h3
-            className="p-2 text-2xl font-semibold tracking-tight"
+          <div
+            className="flex flex-row items-center gap-2"
             style={{
               minWidth: `${MIN_LABEL_WIDTH}px`,
             }}
           >
-            Trace Timeline
-          </h3>
+            <h3 className="text-2xl font-semibold tracking-tight">
+              Trace Timeline
+            </h3>
+            <div className="flex h-full items-center">
+              <Button
+                onClick={() =>
+                  setExpandedItems([
+                    `trace-${trace.id}`,
+                    ...nestedObservationKeys,
+                  ])
+                }
+                size="xs"
+                variant="ghost"
+                title="Expand all"
+              >
+                <PlusSquareIcon className="h-4 w-4" />
+              </Button>
+              <Button
+                onClick={() => setExpandedItems([])}
+                size="xs"
+                variant="ghost"
+                title="Collapse all"
+              >
+                <MinusSquare className="h-4 w-4" />
+              </Button>
+            </div>
+          </div>
           <div
             className="relative mr-2 h-4"
             style={{ width: `${SCALE_WIDTH}px` }}

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";
@@ -34,11 +34,12 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePos
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
 import { Alert, AlertDescription, AlertTitle } from "@/src/components/ui/alert";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
   trace: Trace;
-  scores: Score[];
+  scores: APIScore[];
   projectId: string;
 }) {
   const capture = usePostHogClientCapture();
@@ -101,6 +102,7 @@ export function Trace(props: {
   const expandAll = useCallback(() => {
     capture("trace_detail:observation_tree_expand", { type: "all" });
     setCollapsedObservations([]);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
   return (
@@ -260,7 +262,7 @@ export function TracePage({ traceId }: { traceId: string }) {
               itemId={traceId}
               projectId={trace.data.projectId}
               scope="traces:delete"
-              invalidateFunc={() => void utils.traces.invalidate()}
+              invalidateFunc={() => void utils.traces.all.invalidate()}
               type="trace"
               redirectUrl={`/project/${router.query.projectId as string}/traces`}
             />
@@ -293,7 +295,7 @@ export function TracePage({ traceId }: { traceId: string }) {
           </Badge>
         ) : undefined}
       </div>
-      <div className="mt-5 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
+      <div className="mt-4 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
         <div className="flex flex-row items-center gap-3 p-2.5">
           Tags
           <TagTraceDetailsPopover
@@ -311,19 +313,19 @@ export function TracePage({ traceId }: { traceId: string }) {
           setSelectedTab(tab);
           capture("trace_detail:display_mode_switch", { view: tab });
         }}
-        className="flex w-full justify-end border-b bg-background"
+        className="mt-2 flex w-full justify-end border-b bg-transparent"
       >
-        <TabsList className="bg-background py-0">
+        <TabsList className="bg-transparent py-0">
           <TabsTrigger
             value="details"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <Network className="mr-1 h-4 w-4"></Network>
             Tree
           </TabsTrigger>
           <TabsTrigger
             value="timeline"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <ListTree className="mr-1 h-4 w-4"></ListTree>
             Timeline
@@ -332,7 +334,7 @@ export function TracePage({ traceId }: { traceId: string }) {
         </TabsList>
       </Tabs>
       {selectedTab === "details" && (
-        <div className="mt-5 flex-1 overflow-hidden border-t pt-5">
+        <div className="mt-5 flex-1 overflow-hidden">
           <Trace
             key={trace.data.id}
             trace={trace.data}

web/src/components/trace/lib/helpers.ts

@@ -16,8 +16,19 @@ export function nestObservations(
 ): NestedObservation[] {
   if (list.length === 0) return [];
 
+  // Data prep: Remove parentObservationId attribute from observations if the id does not exist in the list of observations
+  const mutableList = list.map((o) => ({ ...o }));
+  mutableList.forEach((observation) => {
+    if (
+      observation.parentObservationId &&
+      !list.find((o) => o.id === observation.parentObservationId)
+    ) {
+      observation.parentObservationId = null;
+    }
+  });
+
   // Step 0: Sort the list by start time to ensure observations are in right order
-  const sortedObservations = list.sort(
+  const sortedObservations = mutableList.sort(
     (a, b) => a.startTime.getTime() - b.startTime.getTime(),
   );
 

web/src/components/ui/CodeJsonViewer.tsx

@@ -7,17 +7,37 @@ import "react18-json-view/src/dark.css";
 import { deepParseJson } from "@langfuse/shared";
 import { Skeleton } from "@/src/components/ui/skeleton";
 import { useTheme } from "next-themes";
+import { BsMarkdown } from "react-icons/bs";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { useMarkdownContext } from "@/src/features/theming/useMarkdownContext";
 
 export function JSONView(props: {
+  canEnableMarkdown?: boolean;
   json?: unknown;
   title?: string;
   className?: string;
   isLoading?: boolean;
   codeClassName?: string;
+  collapseStringsAfterLength?: number | null;
 }) {
   // some users ingest stringified json nested in json, parse it
+  const [isCopied, setIsCopied] = useState(false);
   const parsedJson = deepParseJson(props.json);
   const { resolvedTheme } = useTheme();
+  const { setIsMarkdownEnabled } = useMarkdownContext();
+  const capture = usePostHogClientCapture();
+
+  const collapseStringsAfterLength =
+    props.collapseStringsAfterLength === null
+      ? 100_000_000 // if null, show all (100M chars)
+      : props.collapseStringsAfterLength ?? 500;
+
+  const handleCopy = () => {
+    setIsCopied(true);
+    void navigator.clipboard.writeText(stringifyJsonNode(parsedJson));
+    setTimeout(() => setIsCopied(false), 1000);
+  };
+
   return (
     <div className={cn("rounded-md border", props.className)}>
       {props.title ? (
@@ -26,10 +46,43 @@ export function JSONView(props: {
             props.title === "assistant" || props.title === "Output"
               ? "dark:border-accent-dark-green"
               : "",
-            "border-b px-3 py-1 text-xs font-medium",
+            "flex flex-row items-center justify-between border-b px-3 py-1 text-xs font-medium",
           )}
         >
           {props.title}
+          <div className="flex items-center gap-1">
+            {props.canEnableMarkdown && (
+              <Button
+                title="Enable Markdown"
+                variant="ghost"
+                type="button"
+                size="xs"
+                onClick={() => {
+                  setIsMarkdownEnabled(true);
+                  capture("trace_detail:io_pretty_format_toggle_group", {
+                    renderMarkdown: true,
+                  });
+                }}
+                className="opacity-50 hover:bg-border"
+              >
+                <BsMarkdown className="h-4 w-4 text-foreground" />
+              </Button>
+            )}
+            <Button
+              title="Copy to clipboard"
+              variant="ghost"
+              size="xs"
+              type="button"
+              onClick={handleCopy}
+              className="hover:bg-border"
+            >
+              {isCopied ? (
+                <Check className="h-3 w-3" />
+              ) : (
+                <Copy className="h-3 w-3" />
+              )}
+            </Button>
+          </div>
         </div>
       ) : undefined}
       <div
@@ -46,7 +99,15 @@ export function JSONView(props: {
             theme="github"
             dark={resolvedTheme === "dark"}
             collapseObjectsAfterLength={20}
-            collapseStringsAfterLength={500}
+            collapseStringsAfterLength={collapseStringsAfterLength}
+            collapseStringMode="word"
+            customizeCollapseStringUI={(fullSTring, truncated) =>
+              truncated ? (
+                <div className="opacity-50">{`\n...expand (${Math.max(fullSTring.length - collapseStringsAfterLength, 0)} more characters)`}</div>
+              ) : (
+                ""
+              )
+            }
             displaySize={"collapsed"}
             matchesURL={true}
             customizeCopy={(node) => stringifyJsonNode(node)}
@@ -148,6 +209,7 @@ export const IOTableCell = ({
             className,
           )}
           codeClassName="py-1 px-2"
+          collapseStringsAfterLength={null} // in table, show full strings as row height is fixed
         />
       )}
     </>

web/src/components/ui/Codeblock.tsx

@@ -0,0 +1,107 @@
+import { Button } from "@/src/components/ui/button";
+import { cn } from "@/src/utils/tailwind";
+import { Check, Copy } from "lucide-react";
+import { type FC, memo, useState } from "react";
+import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
+import {
+  coldarkDark,
+  solarizedlight,
+} from "react-syntax-highlighter/dist/cjs/styles/prism";
+
+interface Props {
+  language: string;
+  value: string;
+  theme?: string;
+  className?: string;
+}
+
+interface languageMap {
+  [key: string]: string;
+}
+
+export const programmingLanguages: languageMap = {
+  javascript: ".js",
+  python: ".py",
+  java: ".java",
+  c: ".c",
+  cpp: ".cpp",
+  "c++": ".cpp",
+  "c#": ".cs",
+  ruby: ".rb",
+  php: ".php",
+  swift: ".swift",
+  "objective-c": ".m",
+  kotlin: ".kt",
+  typescript: ".ts",
+  go: ".go",
+  perl: ".pl",
+  rust: ".rs",
+  scala: ".scala",
+  haskell: ".hs",
+  lua: ".lua",
+  shell: ".sh",
+  sql: ".sql",
+  html: ".html",
+  css: ".css",
+  // add more file extensions here, make sure the key is same as language prop in CodeBlock.tsx component
+};
+
+const CodeBlock: FC<Props> = memo(({ language, value, theme, className }) => {
+  const [isCopied, setIsCopied] = useState(false);
+  const handleCopy = () => {
+    setIsCopied(true);
+    void navigator.clipboard.writeText(value ?? "");
+    setTimeout(() => setIsCopied(false), 1000);
+  };
+
+  return (
+    <div className="codeblock relative w-full overflow-hidden rounded border font-sans dark:bg-zinc-950">
+      <div
+        className={cn(
+          "flex w-full items-center justify-between bg-secondary px-2",
+          className,
+        )}
+      >
+        <span className="text-xs lowercase">{language}</span>
+        <div className="flex items-center py-1">
+          <Button
+            variant="ghost"
+            size="xs"
+            className="text-xs hover:bg-border focus-visible:ring-1 focus-visible:ring-offset-0"
+            onClick={handleCopy}
+          >
+            {isCopied ? (
+              <Check className="h-3 w-3" />
+            ) : (
+              <Copy className="h-3 w-3" />
+            )}
+            <span className="sr-only">Copy code</span>
+          </Button>
+        </div>
+      </div>
+      <SyntaxHighlighter
+        language={language}
+        style={theme === "dark" ? coldarkDark : solarizedlight}
+        PreTag="div"
+        customStyle={{
+          margin: 0,
+          width: "100%",
+          background: "transparent",
+          padding: "0.5rem",
+        }}
+        codeTagProps={{
+          style: {
+            fontSize: "0.75rem",
+            fontFamily: "var(--font-mono)",
+            display: "block",
+          },
+        }}
+      >
+        {value}
+      </SyntaxHighlighter>
+    </div>
+  );
+});
+CodeBlock.displayName = "CodeBlock";
+
+export { CodeBlock };

web/src/components/ui/MarkdownViewer.tsx

@@ -0,0 +1,343 @@
+import { cn } from "@/src/utils/tailwind";
+import {
+  type FC,
+  type ReactNode,
+  type ReactElement,
+  memo,
+  useState,
+  isValidElement,
+  Children,
+  createElement,
+} from "react";
+import ReactMarkdown, { type Components, type Options } from "react-markdown";
+import Link from "next/link";
+import Image from "next/image";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import { CodeBlock } from "@/src/components/ui/Codeblock";
+import { useTheme } from "next-themes";
+import { Button } from "@/src/components/ui/button";
+import { Check, Copy, ImageOff, Maximize2, Minimize2 } from "lucide-react";
+import { api } from "@/src/utils/api";
+import { isPresent } from "@/src/utils/typeChecks";
+import { BsMarkdown } from "react-icons/bs";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { useMarkdownContext } from "@/src/features/theming/useMarkdownContext";
+import { captureException } from "@sentry/nextjs";
+
+// ReactMarkdown does not render raw HTML by default for security reasons, to prevent XSS (Cross-Site Scripting) attacks.
+// html is rendered as plain text by default.
+const MemoizedReactMarkdown: FC<Options> = memo(
+  ReactMarkdown,
+  (prevProps, nextProps) =>
+    prevProps.children === nextProps.children &&
+    prevProps.className === nextProps.className,
+);
+
+const isChecklist = (children: ReactNode) =>
+  Array.isArray(children) &&
+  children.some((child: any) => child?.props?.className === "task-list-item");
+
+/**
+ * Implemented customLoader as we cannot whitelist user provided image domains
+ * Security risks are taken care of by a validation in api.utilities.validateImgUrl
+ * Fetching image will fail if SSL/TLS certificate is invalid or expired, will be handled by onError
+ * Do not use this customLoader in production if you are not using the above mentioned security measures */
+const customLoader = ({ src }: { src: string }) => {
+  return src;
+};
+
+const ImageErrorDisplay = ({
+  src,
+  errorDescription,
+}: {
+  src: string;
+  errorDescription: string;
+}) => (
+  <div className="flex flex-row items-center gap-2">
+    <span title={errorDescription} className="h-4 w-4">
+      <ImageOff className="h-4 w-4" />
+    </span>
+    <Link href={src} className="underline" target="_blank">
+      {src}
+    </Link>
+  </div>
+);
+
+const MarkdownImage: Components["img"] = ({ src, alt }) => {
+  const [isZoomedIn, setIsZoomedIn] = useState(true);
+  const [hasFetchError, setHasFetchError] = useState(false);
+
+  if (!isPresent(src)) return null;
+
+  const isValidImage = api.utilities.validateImgUrl.useQuery(src);
+  if (isValidImage.isLoading) {
+    return (
+      <Skeleton className="h-8 w-1/2 items-center p-2 text-xs">
+        <span className="opacity-80">Loading image...</span>
+      </Skeleton>
+    );
+  }
+
+  const isHttp = new URL(src).protocol === "http:";
+  const errorDescription = `Cannot load image. ${isHttp ? "Http images are not rendered in Langfuse for security reasons." : ""} Click to view image in new tab`;
+
+  if (isValidImage.data?.isValid) {
+    return (
+      <div>
+        {hasFetchError ? (
+          <ImageErrorDisplay src={src} errorDescription={errorDescription} />
+        ) : (
+          <div
+            className={cn(
+              "group relative w-full overflow-hidden rounded border",
+              isZoomedIn ? "h-1/2 w-1/2" : "h-full w-full",
+            )}
+          >
+            <Image
+              loader={customLoader}
+              src={src}
+              alt={alt ?? `Markdown Image-${Math.random()}`}
+              loading="lazy"
+              width={0}
+              height={0}
+              className="h-full w-full object-contain"
+              onError={(error) => {
+                setHasFetchError(true);
+                captureException(error);
+              }}
+            />
+            <Button
+              type="button"
+              className="absolute right-0 top-0 mr-1 mt-1 h-8 w-8 opacity-0 group-hover:!bg-accent/30 group-hover:opacity-100"
+              variant="ghost"
+              size="icon"
+              onClick={() => setIsZoomedIn(!isZoomedIn)}
+            >
+              {isZoomedIn ? (
+                <Maximize2 className="h-4 w-4"></Maximize2>
+              ) : (
+                <Minimize2 className="h-4 w-4"></Minimize2>
+              )}
+            </Button>
+          </div>
+        )}
+      </div>
+    );
+  }
+
+  return <ImageErrorDisplay src={src} errorDescription={errorDescription} />;
+};
+
+const isTextElement = (child: ReactNode): child is ReactElement =>
+  isValidElement(child) &&
+  typeof child.type !== "string" &&
+  ["p", "h1", "h2", "h3", "h4", "h5", "h6"].includes(child.type.name);
+
+const transformListItemChildren = (children: ReactNode) =>
+  Children.map(children, (child) =>
+    isTextElement(child) ? (
+      <div className="mb-1 inline-flex">
+        {createElement(child.type, { ...child.props })}
+      </div>
+    ) : (
+      child
+    ),
+  );
+
+export function MarkdownView({
+  markdown,
+  title,
+  className,
+  customCodeHeaderClassName,
+}: {
+  markdown: string;
+  title?: string;
+  className?: string;
+  customCodeHeaderClassName?: string;
+}) {
+  const [isCopied, setIsCopied] = useState(false);
+  const { resolvedTheme: theme } = useTheme();
+  const capture = usePostHogClientCapture();
+  const { setIsMarkdownEnabled } = useMarkdownContext();
+
+  const handleCopy = () => {
+    setIsCopied(true);
+    void navigator.clipboard.writeText(markdown);
+    setTimeout(() => setIsCopied(false), 1000);
+  };
+
+  return (
+    <div
+      className={cn("overflow-hidden rounded-md border", className)}
+      key={theme}
+    >
+      {title ? (
+        <div
+          className={cn(
+            title === "assistant" || title === "Output"
+              ? "dark:border-accent-dark-green"
+              : "",
+            "flex flex-row items-center justify-between border-b px-3 py-1 text-xs font-medium",
+          )}
+        >
+          {title}
+          <div className="flex items-center gap-1">
+            <Button
+              title="Disable Markdown"
+              variant="ghost"
+              size="xs"
+              type="button"
+              onClick={() => {
+                setIsMarkdownEnabled(false);
+                capture("trace_detail:io_pretty_format_toggle_group", {
+                  renderMarkdown: false,
+                });
+              }}
+              className="hover:bg-border"
+            >
+              <BsMarkdown className="h-4 w-4" />
+            </Button>
+            <Button
+              title="Copy to clipboard"
+              variant="ghost"
+              size="xs"
+              type="button"
+              onClick={handleCopy}
+              className="hover:bg-border"
+            >
+              {isCopied ? (
+                <Check className="h-3 w-3" />
+              ) : (
+                <Copy className="h-3 w-3" />
+              )}
+            </Button>
+          </div>
+        </div>
+      ) : undefined}
+      <MemoizedReactMarkdown
+        className={cn(
+          "space-y-4 overflow-x-auto break-words p-3 text-sm",
+          className,
+        )}
+        remarkPlugins={[remarkGfm, remarkMath]}
+        components={{
+          p({ children }) {
+            return (
+              <p className="mb-2 whitespace-pre-wrap last:mb-0">{children}</p>
+            );
+          },
+          a({ children, href }) {
+            if (href)
+              return (
+                <Link href={href} className="underline" target="_blank">
+                  {children}
+                </Link>
+              );
+          },
+          ul({ children }) {
+            if (isChecklist(children))
+              return <ul className="list-none">{children}</ul>;
+
+            return <ul className="list-inside list-disc pl-2">{children}</ul>;
+          },
+          ol({ children }) {
+            return (
+              <ol className="list-inside list-decimal pl-2">{children}</ol>
+            );
+          },
+          li({ children }) {
+            return (
+              <li className="mb-1 list-item">
+                {transformListItemChildren(children)}
+              </li>
+            );
+          },
+          pre({ children }) {
+            return <pre className="rounded p-2">{children}</pre>;
+          },
+          h1({ children }) {
+            return <h1 className="text-2xl font-bold">{children}</h1>;
+          },
+          h2({ children }) {
+            return <h2 className="text-xl font-bold">{children}</h2>;
+          },
+          h3({ children }) {
+            return <h3 className="text-lg font-bold">{children}</h3>;
+          },
+          h4({ children }) {
+            return <h4 className="text-base font-bold">{children}</h4>;
+          },
+          h5({ children }) {
+            return <h5 className="text-sm font-bold">{children}</h5>;
+          },
+          h6({ children }) {
+            return <h6 className="text-xs font-bold">{children}</h6>;
+          },
+          code({ children, className }) {
+            const languageMatch = /language-(\w+)/.exec(className || "");
+            const language = languageMatch ? languageMatch[1] : "";
+            const codeContent = String(children).replace(/\n$/, "");
+            const isMultiLine = codeContent.includes("\n");
+
+            return language || isMultiLine ? (
+              // code block
+              <CodeBlock
+                key={Math.random()}
+                language={language}
+                value={codeContent}
+                theme={theme}
+                className={customCodeHeaderClassName}
+              />
+            ) : (
+              // inline code
+              <code className="rounded border bg-secondary px-0.5">
+                {codeContent}
+              </code>
+            );
+          },
+          blockquote({ children }) {
+            return (
+              <blockquote className="border-l-4 pl-4 italic">
+                {children}
+              </blockquote>
+            );
+          },
+          img: MarkdownImage,
+          hr() {
+            return <hr className="my-4" />;
+          },
+          table({ children }) {
+            return (
+              <div className="overflow-hidden rounded border">
+                <table className="min-w-full divide-y">{children}</table>
+              </div>
+            );
+          },
+          thead({ children }) {
+            return <thead>{children}</thead>;
+          },
+          tbody({ children }) {
+            return <tbody className="divide-y divide-border">{children}</tbody>;
+          },
+          tr({ children }) {
+            return <tr>{children}</tr>;
+          },
+          th({ children }) {
+            return (
+              <th className="px-4 py-2 text-left text-xs font-medium uppercase tracking-wider">
+                {children}
+              </th>
+            );
+          },
+          td({ children }) {
+            return <td className="whitespace-nowrap px-4 py-2">{children}</td>;
+          },
+        }}
+      >
+        {markdown}
+      </MemoizedReactMarkdown>
+    </div>
+  );
+}

web/src/components/ui/time-icon.tsx

@@ -0,0 +1,28 @@
+import * as React from "react";
+import { Clock, Moon, Sun } from "lucide-react";
+import { type Period } from "@/src/components/ui/time-picker-utils";
+
+const isNoon = (date: Date) =>
+  date.getHours() === 12 && date.getMinutes() === 0 && date.getSeconds() === 0;
+
+const isMidnight = (date: Date) =>
+  date.getHours() === 0 && date.getMinutes() === 0 && date.getSeconds() === 0;
+
+const getIconForPeriod = (period: Period) => {
+  const icons = {
+    PM: <Sun className={"size-5"} />,
+    AM: <Moon className={"size-5"} />,
+  };
+
+  return icons[period] || <Clock className={"size-5"} />;
+};
+
+export const TimeIcon: React.FC<{ time: Date | Period }> = ({ time }) => {
+  if (time instanceof Date) {
+    if (isNoon(time)) return <Sun className="size-5" />;
+    if (isMidnight(time)) return <Moon className="size-5" />;
+    return <Clock className="size-5" />;
+  }
+
+  return getIconForPeriod(time);
+};

web/src/components/ui/time-period-select.tsx

@@ -0,0 +1,77 @@
+"use client";
+
+import * as React from "react";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import {
+  type Period,
+  display12HourValue,
+  setDateByType,
+} from "./time-picker-utils";
+
+export interface PeriodSelectorProps {
+  period: Period;
+  setPeriod: (m: Period) => void;
+  date: Date | undefined;
+  setDate: (date: Date | undefined) => void;
+  onRightFocus?: () => void;
+  onLeftFocus?: () => void;
+}
+
+export const TimePeriodSelect = React.forwardRef<
+  HTMLButtonElement,
+  PeriodSelectorProps
+>(({ period, setPeriod, date, setDate, onLeftFocus, onRightFocus }, ref) => {
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLButtonElement>) => {
+    if (e.key === "ArrowRight") onRightFocus?.();
+    if (e.key === "ArrowLeft") onLeftFocus?.();
+  };
+
+  const handleValueChange = (value: Period) => {
+    setPeriod(value);
+
+    /**
+     * trigger an update whenever the user switches between AM and PM;
+     * otherwise user must manually change the hour each time
+     */
+    if (date) {
+      const hours = display12HourValue(date.getHours());
+      setDate(
+        setDateByType(
+          new Date(date),
+          hours.toString(),
+          "12hours",
+          period === "AM" ? "PM" : "AM",
+        ),
+      );
+    }
+  };
+
+  return (
+    <div className="flex h-7 items-center">
+      <Select
+        defaultValue={period}
+        onValueChange={(value: Period) => handleValueChange(value)}
+      >
+        <SelectTrigger
+          ref={ref}
+          className="w-13 h-7 p-1 pr-0.5 focus:bg-accent focus:text-accent-foreground focus:ring-0"
+          onKeyDown={handleKeyDown}
+        >
+          <SelectValue />
+        </SelectTrigger>
+        <SelectContent className="min-w-20">
+          <SelectItem value="AM">AM</SelectItem>
+          <SelectItem value="PM">PM</SelectItem>
+        </SelectContent>
+      </Select>
+    </div>
+  );
+});
+
+TimePeriodSelect.displayName = "TimePeriodSelect";

web/src/components/ui/time-picker-input.tsx

@@ -0,0 +1,129 @@
+import React from "react";
+
+import {
+  type Period,
+  type TimePickerType,
+  getArrowByType,
+  getDateByType,
+  setDateByType,
+} from "./time-picker-utils";
+import { Input } from "@/src/components/ui/input";
+import { cn } from "@/src/utils/tailwind";
+
+export interface TimePickerInputProps
+  extends React.InputHTMLAttributes<HTMLInputElement> {
+  picker: TimePickerType;
+  date: Date | undefined;
+  setDate: (date: Date | undefined) => void;
+  period?: Period;
+  onRightFocus?: () => void;
+  onLeftFocus?: () => void;
+}
+
+const TimePickerInput = React.forwardRef<
+  HTMLInputElement,
+  TimePickerInputProps
+>(
+  (
+    {
+      className,
+      type = "tel",
+      value,
+      id,
+      name,
+      date = new Date(new Date().setHours(0, 0, 0, 0)),
+      setDate,
+      onChange,
+      onKeyDown,
+      picker,
+      period,
+      onLeftFocus,
+      onRightFocus,
+      ...props
+    },
+    ref,
+  ) => {
+    const [flag, setFlag] = React.useState<boolean>(false);
+    const [prevIntKey, setPrevIntKey] = React.useState<string>("0");
+
+    /**
+     * allow the user to enter the second digit within 2 seconds
+     * otherwise start again with entering first digit
+     */
+    React.useEffect(() => {
+      if (flag) {
+        const timer = setTimeout(() => {
+          setFlag(false);
+        }, 2000);
+
+        return () => clearTimeout(timer);
+      }
+    }, [flag]);
+
+    const calculatedValue = React.useMemo(() => {
+      return getDateByType(date, picker);
+    }, [date, picker]);
+
+    const calculateNewValue = (key: string) => {
+      /*
+       * If picker is '12hours' and the first digit is 0, then the second digit is automatically set to 1.
+       * The second entered digit will break the condition and the value will be set to 10-12.
+       */
+      if (picker === "12hours") {
+        if (flag && calculatedValue.slice(1, 2) === "1" && prevIntKey === "0")
+          return "0" + key;
+      }
+
+      return !flag ? "0" + key : calculatedValue.slice(1, 2) + key;
+    };
+
+    const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
+      if (e.key === "Tab") return;
+      e.preventDefault();
+      if (e.key === "ArrowRight") onRightFocus?.();
+      if (e.key === "ArrowLeft") onLeftFocus?.();
+      if (["ArrowUp", "ArrowDown"].includes(e.key)) {
+        const step = e.key === "ArrowUp" ? 1 : -1;
+        const newValue = getArrowByType(calculatedValue, step, picker);
+        if (flag) setFlag(false);
+        setDate(setDateByType(new Date(date), newValue, picker, period));
+      }
+      if (e.key >= "0" && e.key <= "9") {
+        if (picker === "12hours") setPrevIntKey(e.key);
+
+        const newValue = calculateNewValue(e.key);
+        if (flag) onRightFocus?.();
+        setFlag((prev) => !prev);
+        setDate(setDateByType(new Date(date), newValue, picker, period));
+      }
+    };
+
+    return (
+      <Input
+        ref={ref}
+        id={id || picker}
+        name={name || picker}
+        className={cn(
+          "size-7 p-0 text-center tabular-nums caret-transparent focus:bg-accent focus:text-accent-foreground [&::-webkit-inner-spin-button]:appearance-none",
+          className,
+        )}
+        value={value || calculatedValue}
+        onChange={(e) => {
+          e.preventDefault();
+          onChange?.(e);
+        }}
+        type={type}
+        inputMode="decimal"
+        onKeyDown={(e) => {
+          onKeyDown?.(e);
+          handleKeyDown(e);
+        }}
+        {...props}
+      />
+    );
+  },
+);
+
+TimePickerInput.displayName = "TimePickerInput";
+
+export { TimePickerInput };

web/src/components/ui/time-picker-utils.tsx

@@ -0,0 +1,203 @@
+/**
+ * regular expression to check for valid hour format (01-23)
+ */
+export function isValidHour(value: string) {
+  return /^(0[0-9]|1[0-9]|2[0-3])$/.test(value);
+}
+
+/**
+ * regular expression to check for valid 12 hour format (01-12)
+ */
+export function isValid12Hour(value: string) {
+  return /^(0[1-9]|1[0-2])$/.test(value);
+}
+
+/**
+ * regular expression to check for valid minute format (00-59)
+ */
+export function isValidMinuteOrSecond(value: string) {
+  return /^[0-5][0-9]$/.test(value);
+}
+
+type GetValidNumberConfig = { max: number; min?: number; loop?: boolean };
+
+export function getValidNumber(
+  value: string,
+  { max, min = 0, loop = false }: GetValidNumberConfig,
+) {
+  let numericValue = parseInt(value, 10);
+
+  if (!isNaN(numericValue)) {
+    if (!loop) {
+      if (numericValue > max) numericValue = max;
+      if (numericValue < min) numericValue = min;
+    } else {
+      if (numericValue > max) numericValue = min;
+      if (numericValue < min) numericValue = max;
+    }
+    return numericValue.toString().padStart(2, "0");
+  }
+
+  return "00";
+}
+
+export function getValidHour(value: string) {
+  if (isValidHour(value)) return value;
+  return getValidNumber(value, { max: 23 });
+}
+
+export function getValid12Hour(value: string) {
+  if (isValid12Hour(value)) return value;
+  return getValidNumber(value, { min: 1, max: 12 });
+}
+
+export function getValidMinuteOrSecond(value: string) {
+  if (isValidMinuteOrSecond(value)) return value;
+  return getValidNumber(value, { max: 59 });
+}
+
+type GetValidArrowNumberConfig = {
+  min: number;
+  max: number;
+  step: number;
+};
+
+export function getValidArrowNumber(
+  value: string,
+  { min, max, step }: GetValidArrowNumberConfig,
+) {
+  let numericValue = parseInt(value, 10);
+  if (!isNaN(numericValue)) {
+    numericValue += step;
+    return getValidNumber(String(numericValue), { min, max, loop: true });
+  }
+  return "00";
+}
+
+export function getValidArrowHour(value: string, step: number) {
+  return getValidArrowNumber(value, { min: 0, max: 23, step });
+}
+
+export function getValidArrow12Hour(value: string, step: number) {
+  return getValidArrowNumber(value, { min: 1, max: 12, step });
+}
+
+export function getValidArrowMinuteOrSecond(value: string, step: number) {
+  return getValidArrowNumber(value, { min: 0, max: 59, step });
+}
+
+export function setMinutes(date: Date, value: string) {
+  const minutes = getValidMinuteOrSecond(value);
+  date.setMinutes(parseInt(minutes, 10));
+  return date;
+}
+
+export function setSeconds(date: Date, value: string) {
+  const seconds = getValidMinuteOrSecond(value);
+  date.setSeconds(parseInt(seconds, 10));
+  return date;
+}
+
+export function setHours(date: Date, value: string) {
+  const hours = getValidHour(value);
+  date.setHours(parseInt(hours, 10));
+  return date;
+}
+
+export function set12Hours(date: Date, value: string, period: Period) {
+  const hours = parseInt(getValid12Hour(value), 10);
+  const convertedHours = convert12HourTo24Hour(hours, period);
+  date.setHours(convertedHours);
+  return date;
+}
+
+export type TimePickerType = "minutes" | "seconds" | "hours" | "12hours";
+export type Period = "AM" | "PM";
+
+export function setDateByType(
+  date: Date,
+  value: string,
+  type: TimePickerType,
+  period?: Period,
+) {
+  switch (type) {
+    case "minutes":
+      return setMinutes(date, value);
+    case "seconds":
+      return setSeconds(date, value);
+    case "hours":
+      return setHours(date, value);
+    case "12hours": {
+      if (!period) return date;
+      return set12Hours(date, value, period);
+    }
+    default:
+      return date;
+  }
+}
+
+export function getDateByType(date: Date, type: TimePickerType) {
+  switch (type) {
+    case "minutes":
+      return getValidMinuteOrSecond(String(date.getMinutes()));
+    case "seconds":
+      return getValidMinuteOrSecond(String(date.getSeconds()));
+    case "hours":
+      return getValidHour(String(date.getHours()));
+    case "12hours":
+      const hours = display12HourValue(date.getHours());
+      return getValid12Hour(String(hours));
+    default:
+      return "00";
+  }
+}
+
+export function getArrowByType(
+  value: string,
+  step: number,
+  type: TimePickerType,
+) {
+  switch (type) {
+    case "minutes":
+      return getValidArrowMinuteOrSecond(value, step);
+    case "seconds":
+      return getValidArrowMinuteOrSecond(value, step);
+    case "hours":
+      return getValidArrowHour(value, step);
+    case "12hours":
+      return getValidArrow12Hour(value, step);
+    default:
+      return "00";
+  }
+}
+
+/**
+ * handles value change of 12-hour input
+ * 12:00 PM is 12:00
+ * 12:00 AM is 00:00
+ */
+export function convert12HourTo24Hour(hour: number, period: Period) {
+  if (period === "PM") {
+    if (hour <= 11) {
+      return hour + 12;
+    } else {
+      return hour;
+    }
+  } else if (period === "AM") {
+    if (hour === 12) return 0;
+    return hour;
+  }
+  return hour;
+}
+
+/**
+ * time is stored in the 24-hour form,
+ * but needs to be displayed to the user
+ * in its 12-hour representation
+ */
+export function display12HourValue(hours: number) {
+  if (hours === 0 || hours === 12) return "12";
+  if (hours >= 22) return `${hours - 12}`;
+  if (hours % 12 > 9) return `${hours}`;
+  return `0${hours % 12}`;
+}

web/src/components/ui/time-picker.tsx

@@ -0,0 +1,83 @@
+"use client";
+
+import * as React from "react";
+import { TimePickerInput } from "./time-picker-input";
+import { TimePeriodSelect } from "./time-period-select";
+import { type Period } from "./time-picker-utils";
+import { getTimezoneDetails, getShortLocalTimezone } from "@/src/utils/dates";
+import { TimeIcon } from "@/src/components/ui/time-icon";
+
+interface TimePickerProps {
+  date: Date | undefined;
+  setDate: (date: Date | undefined) => void;
+}
+
+export function TimePicker({ date, setDate }: TimePickerProps) {
+  const [period, setPeriod] = React.useState<Period>("AM");
+
+  const minuteRef = React.useRef<HTMLInputElement>(null);
+  const hourRef = React.useRef<HTMLInputElement>(null);
+  const secondRef = React.useRef<HTMLInputElement>(null);
+  const periodRef = React.useRef<HTMLButtonElement>(null);
+
+  const shortTimezone = React.useMemo(() => getShortLocalTimezone(), []);
+  const timezoneDetails = React.useMemo(() => getTimezoneDetails(), []);
+
+  return (
+    <div className="flex w-full flex-1 items-center gap-1 rounded-b-md border-t-2 bg-transparent px-3 py-2 text-sm ring-offset-background">
+      <div className="mx-1 grid gap-1 text-center">
+        <TimeIcon time={date ?? period} />
+      </div>
+      <div className="grid gap-1 text-center">
+        <TimePickerInput
+          picker="12hours"
+          period={period}
+          date={date}
+          setDate={setDate}
+          ref={hourRef}
+          onRightFocus={() => minuteRef.current?.focus()}
+        />
+      </div>
+      {":"}
+      <div className="grid gap-1 text-center">
+        <TimePickerInput
+          picker="minutes"
+          id="minutes"
+          date={date}
+          setDate={setDate}
+          ref={minuteRef}
+          onLeftFocus={() => hourRef.current?.focus()}
+          onRightFocus={() => secondRef.current?.focus()}
+        />
+      </div>
+      {":"}
+      <div className="grid gap-1">
+        <TimePickerInput
+          picker="seconds"
+          id="seconds"
+          date={date}
+          setDate={setDate}
+          ref={secondRef}
+          onLeftFocus={() => minuteRef.current?.focus()}
+          onRightFocus={() => periodRef.current?.focus()}
+        />
+      </div>
+      <div className="ml-0.5 grid gap-1 text-center">
+        <TimePeriodSelect
+          period={period}
+          setPeriod={setPeriod}
+          date={date}
+          setDate={setDate}
+          ref={periodRef}
+          onLeftFocus={() => secondRef.current?.focus()}
+        />
+      </div>
+      <div className="group relative ml-1">
+        <span>{shortTimezone}</span>
+        <div className="text-s absolute left-1/2 top-full mt-2 hidden -translate-x-1/2 transform whitespace-nowrap rounded bg-card px-2 py-1 text-card-foreground shadow-md ring-1 ring-border group-hover:block">
+          {timezoneDetails}
+        </div>
+      </div>
+    </div>
+  );
+}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.58.0";
+export const VERSION = "v2.65.0";

web/src/dd-instrumentation.ts

@@ -0,0 +1,13 @@
+// import { registerOTel } from "@vercel/otel";
+
+// const { TracerProvider } = (await import("dd-trace")).default.init({
+//   runtimeMetrics: true,
+//   profiling: true,
+// });
+
+// const provider = new TracerProvider();
+
+// registerOTel({
+//   // instrumentations: [new PrismaInstrumentation()],
+// });
+// provider.register();

web/src/ee/features/evals/components/eval-config-detail.tsx

@@ -75,6 +75,13 @@ export const EvalConfigDetail = () => {
             isLoading={config.isLoading}
           />
         }
+        breadcrumb={[
+          {
+            name: "Eval Configs",
+            href: `/project/${router.query.projectId as string}/evals/configs`,
+          },
+          { name: config.data?.id },
+        ]}
       />
       {existingEvalConfig && (
         <>
@@ -82,7 +89,10 @@ export const EvalConfigDetail = () => {
             <Label>Eval Template</Label>
             <TableLink
               path={`/project/${projectId}/evals/templates/${existingEvalConfig.evalTemplateId}`}
-              value={existingEvalConfig.evalTemplateId ?? ""}
+              value={
+                `${existingEvalConfig.evalTemplate.name} (v${existingEvalConfig.evalTemplate.version})` ??
+                ""
+              }
               truncateAt={40}
             />
           </div>

web/src/ee/features/evals/components/eval-config-form.tsx

@@ -82,11 +82,15 @@ export const EvalConfigForm = (props: {
             <SelectValue placeholder="Select a template to run this eval config" />
           </SelectTrigger>
           <SelectContent>
-            {props.evalTemplates.map((template) => (
-              <SelectItem value={template.id} key={template.id}>
-                {`${template.name}-v${template.version}`}
-              </SelectItem>
-            ))}
+            {props.evalTemplates
+              .sort(
+                (a, b) => a.name.localeCompare(b.name) || a.version - b.version,
+              )
+              .map((template) => (
+                <SelectItem value={template.id} key={template.id}>
+                  {`${template.name}-v${template.version}`}
+                </SelectItem>
+              ))}
           </SelectContent>
         </Select>
       ) : undefined}
@@ -153,7 +157,7 @@ export const InnerEvalConfigForm = (props: {
   });
 
   useEffect(() => {
-    if (props.evalTemplate) {
+    if (props.evalTemplate && form.getValues("mapping").length === 0) {
       form.setValue(
         "mapping",
         props.evalTemplate.vars.map((v) => ({

web/src/ee/features/evals/components/eval-config-table.tsx

@@ -1,3 +1,4 @@
+import { StatusBadge } from "@/src/components/layouts/status-badge";
 import { DataTable } from "@/src/components/table/data-table";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
@@ -48,8 +49,12 @@ export default function EvalConfigTable({ projectId }: { projectId: string }) {
       },
     }),
     columnHelper.accessor("status", {
-      id: "state",
-      header: "State",
+      header: "Status",
+      id: "status",
+      cell: (row) => {
+        const status = row.getValue();
+        return <StatusBadge type={status.toLowerCase()} />;
+      },
     }),
     columnHelper.accessor("createdAt", {
       id: "createdAt",

web/src/ee/features/evals/components/eval-template-detail.tsx

@@ -72,6 +72,13 @@ export const EvalTemplateDetail = () => {
             </>
           )
         }
+        breadcrumb={[
+          {
+            name: "Eval Templates",
+            href: `/project/${router.query.projectId as string}/evals/templates`,
+          },
+          { name: template.data?.name ?? "Loading..." },
+        ]}
       />
       {allTemplates.isLoading || !allTemplates.data ? (
         <div>Loading...</div>
@@ -117,7 +124,7 @@ export function EvalVersionDropdown(props: {
         <SelectGroup>
           {props.options?.map((template) => (
             <SelectItem key={template.id} value={template.id}>
-              {template.version} - {template.createdAt.toLocaleDateString()}
+              v{template.version} - {template.createdAt.toLocaleDateString()}
             </SelectItem>
           ))}
         </SelectGroup>