chore: release v2.65.0

* chore: enhance security measures in image verification
* style: add specific title for http image render error

.devcontainer/devcontainer.json

@@ -1,14 +1,14 @@
 {
-	"name": "langfuse dev",
-    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
-    "features": {
-        "ghcr.io/devcontainers/features/docker-in-docker:2": {
-        "version": "latest",
-        "moby": true
-        }
-    },
-    "forwardPorts": [3000],
+  "name": "langfuse dev",
+  "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "latest",
+      "moby": true
+    }
+  },
+  "forwardPorts": [3000],
 
-    "onCreateCommand": "npm install -g pnpm@8.15.5",
-    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
 }

.dockerignore

@@ -3,6 +3,6 @@ Dockerfile
 node_modules
 npm-debug.log
 README.md
-.next
+**/.next
 .git
 **/node_modules

.env.prod.example

@@ -9,6 +9,8 @@
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
+# optional, set to true to disable automated database migrations on Docker start
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED= 
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
@@ -23,6 +25,7 @@ SALT="salt"
 # Use CSP headers to enforce HTTPS, optional
 LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
+
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
@@ -34,6 +37,9 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Enable experimental features, optional
 # LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
+# Disable automated database migrations on Docker start, optional
+# LANFUSE_WEB_MIGRATION_DISABLED=true
+
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
@@ -67,6 +73,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
@@ -124,14 +131,14 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_SENTRY_ORG=
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
+# Used to determine the Sentry sample rate
+# LANGFUSE_TRACING_SAMPLE_RATE=
+
 
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Betterstack
-# LANGFUSE_TEAM_BETTERSTACK_TOKEN=
-
 # Demo project that users can use to try the platform
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
@@ -144,4 +151,5 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Admin API
 # ADMIN_API_KEY=
 
+
 ### END Langfuse Cloud Config

.github/workflows/pipeline.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -64,6 +64,7 @@ jobs:
 
   tests-web:
     runs-on: ubuntu-latest
+    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
@@ -77,7 +78,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -118,6 +119,7 @@ jobs:
 
   tests-worker:
     runs-on: ubuntu-latest
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
@@ -131,7 +133,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -171,7 +173,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -234,7 +236,7 @@ jobs:
     steps:
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -257,6 +259,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
@@ -280,6 +290,9 @@ jobs:
           push: true
           tags: ${{ steps.meta-web.outputs.tags }}
           labels: ${{ steps.meta-web.outputs.labels }}
+          platforms: |
+            linux/amd64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
@@ -304,3 +317,6 @@ jobs:
           push: true
           tags: ${{ steps.meta-worker.outputs.tags }}
           labels: ${{ steps.meta-worker.outputs.labels }}
+          platforms: |
+            linux/amd64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}

.github/workflows/porter_stack_web_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - production
+name: Deploy to web
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: web
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - production
+name: Deploy to worker (EU temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

CONTRIBUTING.md

@@ -53,9 +53,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
-> [!NOTE]
-> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
+**Langfuse v2**
 
 ```mermaid
 flowchart TB
@@ -93,15 +91,66 @@ flowchart TB
         I --- Prisma
     end
 
-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker_API[Public HTTP API]
+    Prisma --- DB
+    JS --- API
+    Python --- API
+```
+
+**Langfuse v3 (work in progress, not released yet)**
+
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
+
+```mermaid
+flowchart TB
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
     end
 
-    API --> Worker_API
-    Worker_API --- DB
-    Worker_API --- Redis
+    DB[Postgres Database]
+    Redis[Redis Cache/Queue]
+    Clickhouse[Clickhouse Database]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        ORM
+        H --> G
+        H --> I
+        G --> I
+        G --- ORM
+        API --- ORM
+        I --- ORM
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker
+    end
+
+    Worker --- DB
+    Worker --- Redis
+    Worker --- Clickhouse
+
+    ORM --- DB
+    ORM --- Redis
+    ORM --- Clickhouse
 
-    Prisma --- DB
     JS --- API
     Python --- API
 ```
@@ -116,7 +165,7 @@ flowchart LR
    Browser ---|Web UI & TRPC API| App
    Integrations/SDKs ---|Public HTTP API| App
    subgraph i1["Application Network"]
-      App["Langfuse Application (Docker or Serverless)"]
+      App["Langfuse Application"]
    end
    subgraph i2["Database Network"]
       DB["Postgres Database"]
@@ -149,6 +198,7 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/
 Requirements
 
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
+- Pnpm v.9.5.0
 - Docker to run the database locally
 
 **Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".

README.md

@@ -159,7 +159,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

ee/package.json

@@ -15,23 +15,23 @@
     }
   },
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
     "next": "^14.2.3",
     "next-auth": "^4.24.7",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/node": "^20.11.29",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",

ee/src/sso/index.ts

@@ -179,6 +179,9 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
+      authorization: {
+        params: { scope: provider.authConfig.scope ?? "openid email profile" },
+      },
     });
   else {
     // Type check to ensure we handle all providers

ee/src/sso/types.ts

@@ -84,6 +84,7 @@ export const CustomProviderSchema = base.extend({
       clientId: z.string(),
       clientSecret: z.string(),
       issuer: z.string(),
+      scope: z.string().nullish(),
       allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
     })
     .nullish(),

fern/apis/client/definition/score.yml

@@ -9,7 +9,6 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: Score
 
 types:
   CreateScoreRequest:
@@ -17,15 +16,119 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -42,11 +42,18 @@ types:
       htmlPath:
         type: string
         docs: Path of trace in Langfuse UI
+      latency:
+        type: double
+        docs: Latency of trace in seconds
       totalCost:
         type: double
         docs: Cost of trace in USD
-      observations: list<ObservationsView>
-      scores: list<Score>
+      observations:
+        type: list<ObservationsView>
+        docs: List of observations
+      scores:
+        type: list<Score>
+        docs: List of scores
   Session:
     properties:
       id: string
@@ -88,6 +95,7 @@ types:
       calculatedOutputCost: optional<double>
       calculatedTotalCost: optional<double>
       latency: optional<double>
+      timeToFirstToken: optional<double>
 
   Usage:
     docs: Standard interface for usage and cost
@@ -120,26 +128,82 @@ types:
       updatedAt: datetime
       projectId: string
       dataType: ScoreDataType
-      isArchived: boolean
-      minValue: optional<double>
-      maxValue: optional<double>
-      categories: optional<list<ConfigCategory>>
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
       description: optional<string>
   ConfigCategory:
     properties:
       value: double
       label: string
-  Score:
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
-  DatasetCore:
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
+  Dataset:
     properties:
       id: string
       name: string
@@ -148,21 +212,6 @@ types:
       projectId: string
       createdAt: datetime
       updatedAt: datetime
-  DatasetWithReferences: # GET datasets
-    extends: DatasetCore
-    properties:
-      items:
-        type: list<string>
-        docs: list of dataset item ids
-      runs:
-        type: list<string>
-        docs: list of dataset run names
-  Dataset: # GET datasets/[name]
-    docs: Dataset including all items
-    extends: DatasetCore
-    properties:
-      items: list<DatasetItem>
-      runs: list<string>
   DatasetItem:
     properties:
       id: string
@@ -196,7 +245,43 @@ types:
       datasetName: string
       createdAt: datetime
       updatedAt: datetime
+  DatasetRunWithItems:
+    extends: DatasetRun
+    properties:
       datasetRunItems: list<DatasetRunItem>
+  Model:
+    docs: Model definition used for transforming usage into USD cost and/or tokenization.
+    properties:
+      id: string
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>
+      isLangfuseManaged:
+        type: boolean
 
   # Utilities
   ModelUsageUnit:
@@ -207,6 +292,7 @@ types:
       - MILLISECONDS
       - SECONDS
       - IMAGES
+      - REQUESTS
   ObservationLevel:
     enum:
       - DEBUG

fern/apis/server/definition/dataset-items.yml

@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
@@ -19,6 +20,23 @@ service:
         id:
           type: string
       response: commons.DatasetItem
+    list:
+      docs: Get dataset items
+      method: GET
+      path: /dataset-items
+      request:
+        name: GetDatasetItemsRequest
+        query-parameters:
+          datasetName: optional<string>
+          sourceTraceId: optional<string>
+          sourceObservationId: optional<string>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetItems
 
 types:
   CreateDatasetItemRequest:
@@ -35,3 +53,7 @@ types:
       status:
         type: optional<commons.DatasetStatus>
         docs: Defaults to ACTIVE for newly created items
+  PaginatedDatasetItems:
+    properties:
+      data: list<commons.DatasetItem>
+      meta: pagination.MetaResponse

fern/apis/server/definition/datasets.yml

@@ -9,42 +9,66 @@ service:
     list:
       method: GET
       docs: Get all datasets
-      path: /datasets
+      path: /v2/datasets
       request:
         name: GetDatasetsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
       response: PaginatedDatasets
     get:
       method: GET
-      docs: Get a dataset and its items
-      path: /datasets/{datasetName}
+      docs: Get a dataset
+      path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
       response: commons.Dataset
     create:
       method: POST
       docs: Create a dataset
-      path: /datasets
+      path: /v2/datasets
       request: CreateDatasetRequest
       response: commons.Dataset
-    getRuns:
+    getRun:
       method: GET
       docs: Get a dataset run and its items
       path: /datasets/{datasetName}/runs/{runName}
       path-parameters:
         datasetName: string
         runName: string
-      response: commons.DatasetRun
+      response: commons.DatasetRunWithItems
+    getRuns:
+      method: GET
+      docs: Get dataset runs
+      path: /datasets/{datasetName}/runs
+      path-parameters:
+        datasetName: string
+      request:
+        name: GetDatasetRunsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetRuns
 
 types:
   PaginatedDatasets:
     properties:
-      data: list<commons.DatasetWithReferences>
+      data: list<commons.Dataset>
       meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
       description: optional<string>
       metadata: optional<unknown>
+  PaginatedDatasetRuns:
+    properties:
+      data: list<commons.DatasetRun>
+      meta: pagination.MetaResponse

fern/apis/server/definition/ingestion.yml

@@ -185,9 +185,54 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/models.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      method: POST
+      docs: Create a model
+      path: /models
+      request: CreateModelRequest
+      response: commons.Model
+    list:
+      method: GET
+      docs: Get all models
+      path: /models
+      request:
+        name: GetModelsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedModels
+    get:
+      method: GET
+      docs: Get a model
+      path: /models/{id}
+      path-parameters:
+        id: string
+      response: commons.Model
+    delete:
+      method: DELETE
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      path: /models/{id}
+      path-parameters:
+        id: string
+
+types:
+  PaginatedModels:
+    properties:
+      data: list<commons.Model>
+      meta: pagination.MetaResponse
+  CreateModelRequest:
+    properties:
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: commons.ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total units. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>

fern/apis/server/definition/observations.yml

@@ -35,7 +35,10 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
+            docs: Retrieve only observations with a start_time or or after this datetime (ISO 8601).
+          toStartTime:
+            type: optional<datetime>
+            docs: Retrieve only observations with a start_time before this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -35,8 +35,12 @@ service:
           name: optional<string>
           label: optional<string>
           tag: optional<string>
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
 
       response: PromptMetaListResponse
 

fern/apis/server/definition/score-configs.yml

@@ -6,6 +6,12 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
     get:
       docs: Get all score configs
       method: GET
@@ -18,7 +24,7 @@ service:
             docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
       response: ScoreConfigs
     get-by-id:
       docs: Get a score config
@@ -34,3 +40,19 @@ types:
     properties:
       data: list<commons.ScoreConfig>
       meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -11,7 +11,7 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: commons.Score
+      response: CreateScoreResponse
     get:
       docs: Get a list of scores
       method: GET
@@ -25,11 +25,18 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this datetime (ISO 8601).
+            docs: Optional filter to only include scores created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include scores created before a certain datetime (ISO 8601)
           source:
             type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
@@ -42,6 +49,12 @@ service:
           scoreIds:
             type: optional<string>
             docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -66,9 +79,59 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  CreateScoreResponse:
+    properties:
+      id:
+        type: string
+        docs: The id of the created object in Langfuse
   Scores:
     properties:
       data: list<commons.Score>

fern/apis/server/definition/sessions.yml

@@ -6,6 +6,26 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    list:
+      docs: Get sessions.
+      method: GET
+      path: /sessions
+      request:
+        name: GetSessionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created before a certain datetime (ISO 8601)
+      response: PaginatedSessions
     get:
       docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
@@ -15,3 +35,8 @@ service:
           type: string
           docs: The unique id of a session
       response: commons.SessionWithTraces
+types:
+  PaginatedSessions:
+    properties:
+      data: list<commons.Session>
+      meta: pagination.MetaResponse

fern/apis/server/definition/trace.yml

@@ -33,7 +33,10 @@ service:
           sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this datetime (ISO 8601).
+            docs: Optional filter to only include traces with a trace.timestamp on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces with a trace.timestamp before a certain datetime (ISO 8601)
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/definition/utils/pagination.yml

@@ -1,4 +1,13 @@
 types:
+  # to be used as copy/paste template for query strings
+  # PaginationRequest:
+  #   properties:
+  #     page:
+  #       type: optional<integer>
+  #       docs: page number, starts at 1
+  #     limit:
+  #       type: optional<integer>
+  #       docs: limit of items per page
   MetaResponse:
     properties:
       page:

package.json

@@ -1,12 +1,11 @@
 {
   "name": "langfuse",
-  "version": "2.57.0",
+  "version": "2.65.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">=20",
-    "pnpm": "<9"
+    "node": "20"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -35,7 +34,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.2.5",
     "release-it": "^17.3.0",
-    "turbo": "^1.13.3"
+    "turbo": "^1.13.4"
   },
   "release-it": {
     "git": {
@@ -79,5 +78,6 @@
         "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
       }
     }
-  }
+  },
+  "packageManager": "pnpm@9.5.0"
 }

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.3",
+    "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -6,7 +6,7 @@
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "exports": {
     ".": {
@@ -61,12 +61,12 @@
     "next-auth": "^4.24.7",
     "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
-    "zod": "^3.22.4",
+    "zod": "^3.23.8",
     "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
     "@types/nodemailer": "^6.4.15",

packages/shared/prisma/generated/types.ts

@@ -286,7 +286,7 @@ export type ObservationView = {
     trace_id: string | null;
     project_id: string;
     type: ObservationType;
-    start_time: Generated<Timestamp>;
+    start_time: Timestamp;
     end_time: Timestamp | null;
     name: string | null;
     metadata: unknown | null;
@@ -294,7 +294,8 @@ export type ObservationView = {
     level: Generated<ObservationLevel>;
     status_message: string | null;
     version: string | null;
-    created_at: Generated<Timestamp>;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     model: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
@@ -357,7 +358,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
     author_user_id: string | null;
     comment: string | null;
@@ -438,6 +439,8 @@ export type TraceView = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     duration: number | null;
 };
 export type User = {

packages/shared/prisma/migrations/20240703214747_models_anthropic_aws_bedrock/migration.sql

@@ -0,0 +1,9 @@
+-- Add AWS Bedrock model names for Anthropic models
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku-20240307|anthropic\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet-20240229|anthropic\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus-20240229|anthropic\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-5-sonnet-20240620|anthropic\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$' WHERE "id" = 'clxt0n0m60000pumz1j5b7zsf';

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240704103901_scores_make_value_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "value" DROP NOT NULL;

packages/shared/prisma/migrations/20240705152639_traces_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,20 @@
+-- Drop and create to be able to change columns, otherwise new t.* cols cannot be added
+
+DROP VIEW IF EXISTS traces_view;
+CREATE VIEW traces_view AS
+WITH observations_metrics AS (
+    SELECT
+        trace_id,
+        project_id,
+        EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+    FROM
+        observations o
+    GROUP BY
+        project_id, trace_id
+)
+SELECT
+    t.*,
+    o.duration
+FROM
+    traces t
+    LEFT JOIN observations_metrics o ON t.id = o.trace_id and t.project_id = o.project_id

packages/shared/prisma/migrations/20240705154048_observation_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,89 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240704103900_observations_view_read_from_calculated and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240710114043_score_configs_drop_empty_categories_array_for_numeric_scores/migration.sql

@@ -0,0 +1,6 @@
+-- Migration script to update score_configs entries
+-- Set categories to NULL where data_type is 'NUMERIC' and categories is an empty array
+
+UPDATE score_configs
+SET categories = NULL
+WHERE data_type = 'NUMERIC' AND categories IS NOT NULL;

packages/shared/prisma/migrations/20240710114044_add_pricing_gpt4o_mini/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-mini
+  ('clyrjp56f0000t0mzapoocd7u', NULL, 'gpt-4o-mini', '(?i)^(gpt-4o-mini)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-mini-2024-07-18
+  ('clyrjpbe20000t0mzcbwc42rg', NULL, 'gpt-4o-mini-2024-07-18', '(?i)^(gpt-4o-mini-2024-07-18)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views", "relationJoins"]
+    previewFeatures = ["tracing", "views", "relationJoins", "metrics"]
 }
 
 datasource db {
@@ -272,6 +272,8 @@ view TraceView {
     input      Json?
     output     Json?
     sessionId  String?  @map("session_id")
+    createdAt  DateTime @map("created_at")
+    updatedAt  DateTime @map("updated_at")
 
     // calculated fields
     duration Float? @map("duration") // can be null if no observations in trace
@@ -299,9 +301,9 @@ model Observation {
     updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
-    model           String?
-    internalModel   String? @map("internal_model")
-    internalModelId String? @map("internal_model_id")
+    model           String? // user-provided model attribute
+    internalModel   String? @map("internal_model") // matched model.name that is matched at ingestion time, to be deprecated
+    internalModelId String? @map("internal_model_id") // matched model.id that is matched at ingestion time
 
     modelParameters  Json?
     input            Json?
@@ -352,7 +354,7 @@ view ObservationView {
     traceId             String?          @map("trace_id")
     projectId           String           @map("project_id")
     type                ObservationType
-    startTime           DateTime         @default(now()) @map("start_time")
+    startTime           DateTime         @map("start_time")
     endTime             DateTime?        @map("end_time")
     name                String?
     metadata            Json?
@@ -360,7 +362,8 @@ view ObservationView {
     level               ObservationLevel @default(DEFAULT)
     statusMessage       String?          @map("status_message")
     version             String?
-    createdAt           DateTime         @default(now()) @map("created_at")
+    createdAt           DateTime         @map("created_at")
+    updatedAt           DateTime         @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -410,14 +413,14 @@ model Score {
     projectId     String         @map("project_id")
     project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
-    value         Float
+    value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
     source        ScoreSource
     authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
-    stringValue   String?        @map("string_value")
+    stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
@@ -629,7 +632,7 @@ model Model {
     inputPrice      Decimal?  @map("input_price")
     outputPrice     Decimal?  @map("output_price")
     totalPrice      Decimal?  @map("total_price")
-    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, REQUESTS, or IMAGES
     tokenizerId     String?   @map("tokenizer_id")
     tokenizerConfig Json?     @map("tokenizer_config")
 

packages/shared/prisma/seed.ts

@@ -17,6 +17,11 @@ import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
 const options = {
   environment: { type: "string" },
 } as const;
@@ -459,7 +464,15 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
-  configIdsAndNames: Map<string, { name: string; id: string }[]>
+  configParams: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -518,13 +531,34 @@ function createObjects(
 
     traces.push(trace);
 
-    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const configArray = configParams.get(projectId) ?? [];
     const randomIndex = Math.floor(Math.random() * 3);
     const config =
       configArray.length >= randomIndex - 1 && configArray[randomIndex];
-    const { name: annotationScoreName, id: configId } = config || {
+    const {
+      name: annotationScoreName,
+      id: configId,
+      dataType,
+      categories,
+    } = config || {
       name: "manual-score",
       id: undefined,
+      dataType: ScoreDataType.NUMERIC,
+      categories: null,
+    };
+
+    const value = Math.floor(Math.random() * 2);
+    const scoreNumericAndStringValue = {
+      ...(dataType === ScoreDataType.NUMERIC && { value }),
+      ...(dataType === ScoreDataType.CATEGORICAL && {
+        value,
+        stringValue: categories?.find((category) => category.value === value)
+          ?.label,
+      }),
+      ...(dataType === ScoreDataType.BOOLEAN && {
+        value,
+        stringValue: value === 1 ? "True" : "False",
+      }),
     };
 
     const traceScores = [
@@ -533,12 +567,12 @@ function createObjects(
             {
               traceId: trace.id,
               name: annotationScoreName,
-              value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
-              dataType: ScoreDataType.NUMERIC,
+              dataType,
+              ...scoreNumericAndStringValue,
               ...(configId ? { configId } : {}),
             },
           ]
@@ -556,6 +590,20 @@ function createObjects(
             },
           ]
         : []),
+      ...(Math.random() < 0.8
+        ? [
+            {
+              traceId: trace.id,
+              name: "Completeness",
+              timestamp: traceTs,
+              source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.CATEGORICAL,
+              stringValue:
+                Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+            },
+          ]
+        : []),
     ];
 
     scores.push(...traceScores);
@@ -662,7 +710,7 @@ function createObjects(
                   },
                   {
                     role: "user",
-                    content: "How can i create a React component?",
+                    content: "How can i create a *React* component?",
                   },
                 ]
               : {
@@ -680,37 +728,8 @@ function createObjects(
                     },
                   ],
                 },
-          output: {
-            completion: `Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.
-
-              1.  **Functional Component**:
-              
-              A functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:
-              
-              
-              'import React from 'react';  function Greeting(props) {   return <h1>Hello, {props.name}</h1>; }  export default Greeting;'
-              
-              To use this component in another file, you can do:
-              
-              
-              'import Greeting from './Greeting';  function App() {   return (     <div>       <Greeting name="John" />     </div>   ); }  export default App;'
-              
-              2.  **Class Component**:
-              
-              You can also define components as classes in React. These have some additional features compared to functional components:
-              
-              
-              'import React, { Component } from 'react';  class Greeting extends Component {   render() {     return <h1>Hello, {this.props.name}</h1>;   } }  export default Greeting;'
-              
-              And here's how to use this component:
-              
-              
-              'import Greeting from './Greeting';  class App extends Component {   render() {     return (       <div>         <Greeting name="John" />       </div>     );   } }  export default App;'
-              
-              With the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.
-              
-              Remember to import React at the top of your file whenever you're creating a component, because JSX transpiles to 'React.createElement' calls under the hood.`,
-          },
+          output:
+            "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.\n\n**Image**\n\n![Languse Example Image](https://static.langfuse.com/langfuse-dev/langfuse-example-image.jpeg)\n\n1.  **Functional Component**:\n\nA functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:\n\n```javascript\nimport React from 'react';\nfunction Greeting(props) {\n  return <h1>Hello, {props.name}</h1>;\n}\nexport default Greeting;\n```\n\nTo use this component in another file, you can do:\n\n```javascript\nimport Greeting from './Greeting';\nfunction App() {\n  return (\n    <div>\n      <Greeting name=\"John\" />\n    </div>\n  );\n}\nexport default App;\n```\n\n2.  **Class Component**:\n\nYou can also define components as classes in React. These have some additional features compared to functional components:\n\n```javascript\nimport React, { Component } from 'react';\nclass Greeting extends Component {\n  render() {\n    return <h1>Hello, {this.props.name}</h1>;\n  }\n}\nexport default Greeting;\n```\n\nAnd here's how to use this component:\n\n```javascript\nimport Greeting from './Greeting';\nclass App extends Component {\n  render() {\n    return (\n      <div>\n        <Greeting name=\"John\" />\n      </div>\n    );\n  }\n}\nexport default App;\n```\n\nWith the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.\n\nRemember to import React at the top of your file whenever you're creating a component, because JSX transpiles to `React.createElement` calls under the hood.",
           model: model,
           internalModel: model,
           modelParameters: {
@@ -976,8 +995,15 @@ async function generatePrompts(project: Project) {
 }
 
 async function generateConfigsForProject(projects: Project[]) {
-  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
-    new Map();
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
 
   await Promise.all(
     projects.map(async (project) => {
@@ -989,7 +1015,12 @@ async function generateConfigsForProject(projects: Project[]) {
 }
 
 async function generateConfigs(project: Project) {
-  const configNameAndId: { name: string; id: string }[] = [];
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
 
   const configs = [
     {
@@ -1046,7 +1077,12 @@ async function generateConfigs(project: Project) {
         id: config.id,
       },
     });
-    configNameAndId.push({ name: config.name, id: config.id });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
   }
 
   return configNameAndId;

packages/shared/src/constants.ts

@@ -6,4 +6,5 @@ export enum ModelUsageUnit {
   Seconds = "SECONDS",
   Milliseconds = "MILLISECONDS",
   Images = "IMAGES",
+  Requests = "REQUESTS",
 }

packages/shared/src/errors/InternalServerError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InternalServerError extends BaseError {
+  constructor(description = "Internal Server Error") {
+    super("InternalServerError", 500, description, true);
+  }
+}

packages/shared/src/errors/InvalidRequestError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InvalidRequestError extends BaseError {
+  constructor(description = "Invalid Request Error") {
+    super("InvalidRequestError", 400, description, true);
+  }
+}

packages/shared/src/errors/ValidationError.ts

@@ -1,7 +0,0 @@
-import { BaseError } from "./BaseError";
-
-export class ValidationError extends BaseError {
-  constructor(description = "Validation Error") {
-    super("ValidationError", 400, description, true);
-  }
-}

packages/shared/src/errors/index.ts

@@ -1,7 +1,8 @@
 export { BaseError } from "./BaseError";
 export { LangfuseNotFoundError } from "./NotFoundError";
-export { ValidationError } from "./ValidationError";
+export { InvalidRequestError } from "./InvalidRequestError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
+export { InternalServerError } from "./InternalServerError";

packages/shared/src/features/annotation/types.ts

@@ -1,15 +1,59 @@
 import z from "zod";
-import { ScoreConfig } from "../../db";
+import { type ScoreDataType } from "../../db";
 
-const configCategory = z.object({
-  label: z.string().min(1),
+const NUMERIC: ScoreDataType = "NUMERIC";
+const CATEGORICAL: ScoreDataType = "CATEGORICAL";
+const BOOLEAN: ScoreDataType = "BOOLEAN";
+
+export const availableDataTypes = [NUMERIC, CATEGORICAL, BOOLEAN] as const;
+
+const NumericData = z.object({
   value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
 });
 
-export const categoriesList = z.array(configCategory);
+const CategoricalData = z.object({
+  value: z.number().optional().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
 
-export type ConfigCategory = z.infer<typeof configCategory>;
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
 
-export type CastedConfig = Omit<ScoreConfig, "categories"> & {
-  categories: ConfigCategory[] | null;
-};
+const CreateAnnotationScoreBase = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  traceId: z.string(),
+  configId: z.string().optional(),
+  observationId: z.string().optional(),
+  comment: z.string().optional().nullish(),
+});
+
+const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
+  id: z.string(),
+});
+
+/**
+ * CreateAnnotationScoreData is only used for annotation scores created via the UI.
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const CreateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  CreateAnnotationScoreBase.merge(NumericData),
+  CreateAnnotationScoreBase.merge(CategoricalData),
+  CreateAnnotationScoreBase.merge(BooleanData),
+]);
+
+/**
+ * UpdateAnnotationScoreData is only used for annotation scores updated via the UI
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  UpdateAnnotationScoreBase.merge(NumericData),
+  UpdateAnnotationScoreBase.merge(CategoricalData),
+  UpdateAnnotationScoreBase.merge(BooleanData),
+]);

packages/shared/src/features/evals/types.ts

@@ -80,6 +80,7 @@ export const availableEvalVariables = [
 export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
   [LLMAdapter.OpenAI]: [
     "gpt-4o",
+    "gpt-4o-mini",
     "gpt-4-turbo-preview",
     "gpt-3.5-turbo",
   ] as const,

packages/shared/src/features/ingestion/types.ts

@@ -28,7 +28,7 @@ const MixedUsage = z.object({
   totalCost: z.number().nullish(),
 });
 
-export const stringDate = z.string().datetime({ offset: true }).nullish();
+export const stringDateTime = z.string().datetime({ offset: true }).nullish();
 
 export const usage = MixedUsage.nullish()
   // transform mixed usage model input to new one
@@ -58,7 +58,7 @@ export const usage = MixedUsage.nullish()
 
 export const TraceBody = z.object({
   id: z.string().nullish(),
-  timestamp: stringDate,
+  timestamp: stringDateTime,
   name: z.string().nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),
@@ -75,7 +75,7 @@ export const TraceBody = z.object({
 export const OptionalObservationBody = z.object({
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
+  startTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -94,21 +94,27 @@ export const UpdateEventEvent = OptionalObservationBody.extend({
 });
 
 export const CreateSpanBody = CreateEventEvent.extend({
-  endTime: stringDate,
+  endTime: stringDateTime,
 });
 
 export const UpdateSpanBody = UpdateEventEvent.extend({
-  endTime: stringDate,
+  endTime: stringDateTime,
 });
 
 export const CreateGenerationBody = CreateSpanBody.extend({
-  completionStartTime: stringDate,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
       z.string(),
       z
-        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
+        .union([
+          z.string(),
+          z.number(),
+          z.boolean(),
+          z.array(z.string()),
+          z.record(z.string()),
+        ])
         .nullish()
     )
     .nullish(),
@@ -124,13 +130,19 @@ export const CreateGenerationBody = CreateSpanBody.extend({
 });
 
 export const UpdateGenerationBody = UpdateSpanBody.extend({
-  completionStartTime: stringDate,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
       z.string(),
       z
-        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
+        .union([
+          z.string(),
+          z.number(),
+          z.boolean(),
+          z.array(z.string()),
+          z.record(z.string()),
+        ])
         .nullish()
     )
     .nullish(),
@@ -145,22 +157,82 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
   return false;
 });
 
-export const ScoreBody = z.object({
+const BaseScoreBody = z.object({
   id: z.string().nullish(),
   name: NonEmptyString,
-  value: z.number(),
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
 });
 
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = z
+  .discriminatedUnion("dataType", [
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      })
+    ),
+    BaseScoreBody.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      })
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType) {
+      if (typeof data.value === "number") {
+        if (data.dataType === "CATEGORICAL") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a string for data type ${data.dataType}`,
+          });
+        }
+      } else if (typeof data.value === "string") {
+        if (data.dataType === "NUMERIC") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must be a number for data type ${data.dataType}`,
+          });
+        } else if (data.dataType === "BOOLEAN") {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
+          });
+        }
+      }
+    }
+  });
+
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
   id: z.string().nullish(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -174,8 +246,8 @@ export const LegacySpanPatchSchema = z.object({
   spanId: z.string(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
   metadata: jsonSchema.nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -188,9 +260,9 @@ export const LegacyGenerationsCreateSchema = z.object({
   id: z.string().nullish(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
@@ -212,9 +284,9 @@ export const LegacyGenerationPatchSchema = z.object({
   generationId: z.string(),
   traceId: z.string().nullish(),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(
@@ -236,9 +308,9 @@ export const LegacyObservationBody = z.object({
   traceId: z.string().nullish(),
   type: z.enum(["GENERATION", "SPAN", "EVENT"]),
   name: z.string().nullish(),
-  startTime: stringDate,
-  endTime: stringDate,
-  completionStartTime: stringDate,
+  startTime: stringDateTime,
+  endTime: stringDateTime,
+  completionStartTime: stringDateTime,
   model: z.string().nullish(),
   modelParameters: z
     .record(

packages/shared/src/server/auth.ts

@@ -65,7 +65,7 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     name: "CustomSSOProvider",
     type: "oauth",
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
-    authorization: { params: { scope: "openid email profile" } },
+    authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
     idToken: true,
     profile(profile) {

packages/shared/src/server/llm/types.ts

@@ -48,6 +48,8 @@ export const ZodModelConfig = z.object({
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-05-13",
+  "gpt-4o-mini",
+  "gpt-4o-mini-2024-07-18",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

packages/shared/src/server/services/email/projectInvitation/ProjectInvitationEmailTemplate.tsx

@@ -19,7 +19,7 @@ interface ProjectInvitationTemplateProps {
   invitedByUsername: string;
   invitedByUserEmail: string;
   projectName: string;
-  recieverEmail: string;
+  receiverEmail: string;
   inviteLink: string;
   emailFromAddress: string;
   langfuseCloudRegion?: string;
@@ -29,7 +29,7 @@ export const ProjectInvitationTemplate = ({
   invitedByUsername,
   invitedByUserEmail,
   projectName,
-  recieverEmail,
+  receiverEmail,
   inviteLink,
   emailFromAddress,
   langfuseCloudRegion,
@@ -90,7 +90,7 @@ export const ProjectInvitationTemplate = ({
             <Hr className="mx-0 my-[26px] w-full border border-solid border-[#eaeaea]" />
             <Text className="text-xs leading-6 text-[#666666]">
               This invitation was intended for{" "}
-              <span className="text-black">{recieverEmail}</span>. This invite
+              <span className="text-black">{receiverEmail}</span>. This invite
               was sent from{" "}
               <span className="text-black">{emailFromAddress}</span>. If you
               were not expecting this invitation, you can ignore this email.

packages/shared/src/server/services/email/projectInvitation/sendProjectInvitationEmail.ts

@@ -63,7 +63,7 @@ export const sendProjectInvitationEmail = async ({
         invitedByUsername: inviterName,
         invitedByUserEmail: inviterEmail,
         projectName: projectName,
-        recieverEmail: to,
+        receiverEmail: to,
         inviteLink: authUrl,
         emailFromAddress: env.EMAIL_FROM_ADDRESS,
         langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,

packages/shared/src/utils/zod.ts

@@ -58,7 +58,32 @@ export const optionalPaginationZod = {
     .optional(),
 };
 
+export const queryStringZod = z
+  .string()
+  .transform((val) => decodeURIComponent(val));
+
+export const paginationMetaResponseZod = z.object({
+  page: z.number().int().positive(),
+  limit: z.number().int().positive(),
+  totalItems: z.number().int().nonnegative(),
+  totalPages: z.number().int().nonnegative(),
+});
+
 export const noHtmlRegex = /<[^>]*>/;
 export const noHtmlCheck = (value: string) => !noHtmlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);
+
+/**
+ * Validates an object against a Zod schema and helps with IDE type warnings.
+ *
+ * @param schema - The Zod schema to validate against.
+ * @param object - The object to be validated.
+ * @returns The parsed object if validation is successful.
+ */
+export const validateZodSchema = <T extends z.ZodTypeAny>(
+  schema: T,
+  object: z.infer<T>
+): z.infer<T> => {
+  return schema.parse(object);
+};

turbo.json

@@ -17,7 +17,7 @@
     "db:push": {},
     "db:seed": {},
     "models:migrate": {
-      "dependsOn": ["db:generate"]
+      "dependsOn": ["db:generate", "@langfuse/shared#build"]
     },
     "db:seed:examples": {},
     "dev": {

web/Dockerfile

@@ -1,26 +1,24 @@
-FROM node:20-alpine3.20 AS alpine
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
-FROM alpine AS base
-RUN npm install turbo@^1.13.3 --global
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
+RUN npm install turbo@^1.13.4 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
-RUN corepack prepare pnpm@8.15.5 --activate
+RUN corepack prepare pnpm@9.5.0 --activate
 
-
-FROM base AS pruner
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS pruner
 
 WORKDIR /app
 
 COPY . .
 RUN turbo prune --scope=web --docker
 
-
-FROM base AS builder
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS builder
 
 WORKDIR /app
 
@@ -32,6 +30,7 @@ COPY --from=pruner /app/out/json/ .
 RUN pnpm install --frozen-lockfile
 
 ENV DOCKER_BUILD 1
+ENV NEXT_MANUAL_SIG_HANDLE true
 
 # pass public variables in build step
 ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
@@ -41,6 +40,7 @@ ARG NEXT_PUBLIC_TURNSTILE_SITE_KEY
 ARG NEXT_PUBLIC_POSTHOG_KEY
 ARG NEXT_PUBLIC_POSTHOG_HOST
 ARG NEXT_PUBLIC_CRISP_WEBSITE_ID
+ARG LANFUSE_WEB_MIGRATION_DISABLED
 
 # Copy source code of isolated subworkspace
 COPY --from=pruner /app/out/full/ .
@@ -52,13 +52,12 @@ RUN rm -f ./web/src/middleware.ts
 # Learn more here: https://nextjs.org/telemetry
 # Uncomment the following line in case you want to disable telemetry during the build.
 ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_MANUAL_SIG_HANDLE true
 
 RUN turbo run build --filter=web
 
 # Production image, copy all the files and run next
-FROM base AS runner
-
-RUN apk add --no-cache dumb-init
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS runner
 
 WORKDIR /app
 
@@ -67,13 +66,16 @@ ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1
 # Needed to re-enable validation of environment variables during runtime
 ENV DOCKER_BUILD 0
+# Set NEXT_MANUAL_SIG_HANDLE for runtime
+ENV NEXT_MANUAL_SIG_HANDLE true
+
+RUN apk add --no-cache dumb-init
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
 RUN npm install -g --no-package-lock --no-save prisma
 
-
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .
 
@@ -94,5 +96,8 @@ USER nextjs
 # Default port to 3000
 ENV PORT 3000
 
-# CMD ["node", "server.js"]
-CMD ["dumb-init", "--", "./web/entrypoint.sh"]
+
+ENTRYPOINT ["dumb-init", "--", "./web/entrypoint.sh"]
+
+# startup command
+CMD ["node", "./web/server.js"]

web/entrypoint.sh

@@ -19,10 +19,13 @@ if [ -z "$DIRECT_URL" ]; then
     export DIRECT_URL=$DATABASE_URL
 fi
 
-prisma db execute --url "$DIRECT_URL" --file "./packages/shared/scripts/cleanup.sql"
+# Always execute the scripts, except when disabled.
+if [ "$LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED" != "true" ]; then
+    prisma db execute --url "$DIRECT_URL" --file "./packages/shared/scripts/cleanup.sql"
 
-# Apply migrations
-prisma migrate deploy --schema=./packages/shared/prisma/schema.prisma
+    # Apply migrations
+    prisma migrate deploy --schema=./packages/shared/prisma/schema.prisma
+fi
 status=$?
 
 # If migration fails (returns non-zero exit status), exit script with that status
@@ -32,5 +35,5 @@ if [ $status -ne 0 ]; then
     exit $status
 fi
 
-# Start server
-node web/server.js
+# Run the command passed to the docker image on start
+exec "$@"

web/next.config.mjs

@@ -11,8 +11,8 @@ import { env } from "./src/env.mjs";
  * img-src https to allow loading images from SSO providers
  */
 const cspHeader = `
-  default-src 'self' https://ph.langfuse.com https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://*.sentry.io https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  default-src 'self' https://*.langfuse.com https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
+  script-src 'self' 'unsafe-eval' https://*.langfuse.com https://*.crisp.chat https://challenges.cloudflare.com https://*.sentry.io https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
   style-src 'self' 'unsafe-inline' https://*.crisp.chat;
   img-src 'self' https: blob: data:;
   font-src 'self' https://*.crisp.chat;
@@ -80,23 +80,37 @@ const nextConfig = {
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
+      // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
+      {
+        source: "/generated/:path*",
+        headers: [
+          {
+            key: "Access-Control-Allow-Origin",
+            value: "*",
+          },
+          {
+            key: "Access-Control-Allow-Methods",
+            value: "GET",
+          },
+        ],
+      },
     ];
   },
 

web/package.json

@@ -1,23 +1,24 @@
 {
   "name": "web",
-  "version": "2.57.0",
+  "version": "2.65.0",
   "private": true,
   "license": "MIT",
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "scripts": {
     "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
-    "lint": "dotenv -e ../.env -- next lint",
+    "lint": "dotenv -e ../.env -- next lint --max-warnings 0",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
     "prettier": "prettier --write ./src *.{ts,js}",
     "clean": "rm -rf node_modules",
-    "start": "dotenv -e ../.env -- next start",
+    "start": "dotenv -e ../.env -- sh -c 'NEXT_MANUAL_SIG_HANDLE=true next start'",
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
+    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts",
+    "generationCost:backfill": "dotenv -e ../.env -- tsx scripts/observations-backfill-calculated-cost.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
@@ -59,10 +60,10 @@
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.9.2",
-    "@sentry/node": "^8.9.2",
-    "@sentry/profiling-node": "^8.9.2",
-    "@sentry/types": "^8.9.2",
+    "@sentry/nextjs": "^8.13.0",
+    "@sentry/node": "^8.13.0",
+    "@sentry/profiling-node": "^8.13.0",
+    "@sentry/types": "^8.13.0",
     "@t3-oss/env-nextjs": "^0.10.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -86,6 +87,8 @@
     "decimal.js": "^10.4.3",
     "dompurify": "^3.1.5",
     "exponential-backoff": "^3.1.1",
+    "graphql": "^16.9.0",
+    "ip-address": "^9.0.5",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
     "langchain": "^0.2.6",
@@ -97,14 +100,19 @@
     "next-themes": "^0.3.0",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
+    "prexit": "^2.2.0",
     "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.51.5",
     "react-icons": "^5.2.1",
+    "react-markdown": "^9.0.1",
     "react-responsive": "^10.0.0",
+    "react-syntax-highlighter": "^15.5.0",
     "react18-json-view": "^0.2.8-canary.6",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
     "sonner": "^1.4.41",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.2.1",
@@ -112,7 +120,7 @@
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
     "vaul": "^0.9.1",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
@@ -130,6 +138,7 @@
     "@types/node": "20.10.5",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
+    "@types/react-syntax-highlighter": "^15.5.13",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",

web/public/generated/api-client/openapi.yml

@@ -11,12 +11,8 @@ paths:
         - Score
       parameters: []
       responses:
-        '200':
+        '204':
           description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
         '400':
           description: ''
           content:
@@ -59,21 +55,39 @@ components:
           type: string
         traceId:
           type: string
+          example: cdef-1234-5678-90ab
         name:
           type: string
+          example: novelty
         value:
-          type: number
-          format: double
+          $ref: '#/components/schemas/CreateScoreValue'
+          description: >-
+            The value of the score. Must be passed as string for categorical
+            scores, and numeric for boolean and numeric scores. Boolean score
+            values must equal either 1 or 0 (true or false)
         observationId:
           type: string
         comment:
           type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+          description: >-
+            When set, must match the score value's type. If not set, will be
+            inferred from the score value or config
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, the score name must
+            equal the config name and scores must comply with the config's range
+            and data type. For categorical scores, the value must map to a
+            config category. Numeric scores might be constrained by the score
+            config's max and min values
       required:
         - traceId
         - name
         - value
-    Score:
-      title: Score
+    BaseScore:
+      title: BaseScore
       type: object
       properties:
         id:
@@ -82,22 +96,147 @@ components:
           type: string
         name:
           type: string
-        value:
-          type: number
-          format: double
+        source:
+          $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
         timestamp:
           type: string
           format: date-time
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+        authorUserId:
+          type: string
         comment:
           type: string
+        configId:
+          type: string
+          description: >-
+            Reference a score config on a score. When set, config and score name
+            must be equal and value must comply to optionally defined numerical
+            range
       required:
         - id
         - traceId
         - name
-        - value
+        - source
         - timestamp
+        - createdAt
+        - updatedAt
+    NumericScore:
+      title: NumericScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: The numeric value of the score
+      required:
+        - value
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    BooleanScore:
+      title: BooleanScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: >-
+            The numeric value of the score. Equals 1 for "True" and 0 for
+            "False"
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. Is inferred from the
+            numeric value and equals "True" or "False"
+      required:
+        - value
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    CategoricalScore:
+      title: CategoricalScore
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+          description: >-
+            Only defined if a config is linked. Represents the numeric category
+            mapping of the stringValue
+        stringValue:
+          type: string
+          description: >-
+            The string representation of the score value. If no config is
+            linked, can be any string. Otherwise, must map to a config category
+      required:
+        - stringValue
+      allOf:
+        - $ref: '#/components/schemas/BaseScore'
+    Score:
+      title: Score
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - NUMERIC
+            - $ref: '#/components/schemas/NumericScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - CATEGORICAL
+            - $ref: '#/components/schemas/CategoricalScore'
+          required:
+            - dataType
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                dataType:
+                  type: string
+                  enum:
+                    - BOOLEAN
+            - $ref: '#/components/schemas/BooleanScore'
+          required:
+            - dataType
+    ScoreSource:
+      title: ScoreSource
+      type: string
+      enum:
+        - ANNOTATION
+        - API
+        - EVAL
+    ScoreDataType:
+      title: ScoreDataType
+      type: string
+      enum:
+        - NUMERIC
+        - CATEGORICAL
+        - BOOLEAN
+    CreateScoreValue:
+      title: CreateScoreValue
+      oneOf:
+        - type: number
+          format: double
+        - type: string
+      description: >-
+        The value of the score. Must be passed as string for categorical scores,
+        and numeric for boolean and numeric scores
   securitySchemes:
     BearerAuth:
       type: http

web/public/generated/postman/collection.json

@@ -106,6 +106,57 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get dataset items",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/dataset-items?datasetName=&sourceTraceId=&sourceObservationId=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "dataset-items"
+              ],
+              "query": [
+                {
+                  "key": "datasetName",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "sourceTraceId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "sourceObservationId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -160,25 +211,26 @@
           "request": {
             "description": "Get all datasets",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets?page=&limit=",
+              "raw": "{{baseUrl}}/api/public/v2/datasets?page=&limit=",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets"
               ],
               "query": [
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 }
               ],
               "variable": []
@@ -194,15 +246,16 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a dataset and its items",
+            "description": "Get a dataset",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets/:datasetName",
+              "raw": "{{baseUrl}}/api/public/v2/datasets/:datasetName",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets",
                 ":datasetName"
               ],
@@ -228,13 +281,14 @@
           "request": {
             "description": "Create a dataset",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets",
+              "raw": "{{baseUrl}}/api/public/v2/datasets",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets"
               ],
               "query": [],
@@ -257,7 +311,7 @@
         },
         {
           "_type": "endpoint",
-          "name": "Get Runs",
+          "name": "Get Run",
           "request": {
             "description": "Get a dataset run and its items",
             "url": {
@@ -293,6 +347,50 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Runs",
+          "request": {
+            "description": "Get dataset runs",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/datasets/:datasetName/runs?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "datasets",
+                ":datasetName",
+                "runs"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": [
+                {
+                  "key": "datasetName",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -437,6 +535,146 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Models",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"modelName\": \"example\",\n    \"matchPattern\": \"example\",\n    \"startDate\": \"1994-11-05\",\n    \"unit\": \"CHARACTERS\",\n    \"inputPrice\": 0,\n    \"outputPrice\": 0,\n    \"totalPrice\": 0,\n    \"tokenizerId\": \"example\",\n    \"tokenizerConfig\": \"UNKNOWN\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all models",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get a model",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/models/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "models",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -480,7 +718,7 @@
           "request": {
             "description": "Get a list of observations",
             "url": {
-              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=",
+              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=&toStartTime=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -528,7 +766,12 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
+                  "description": "Retrieve only observations with a start_time or or after this datetime (ISO 8601)."
+                },
+                {
+                  "key": "toStartTime",
+                  "value": "",
+                  "description": "Retrieve only observations with a start_time before this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -658,12 +901,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 }
               ],
               "variable": []
@@ -716,6 +959,39 @@
       "description": null,
       "name": "Score Configs",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a score configuration (config). Score configs are used to define the structure of scores",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"dataType\": \"NUMERIC\",\n    \"categories\": [\n        {\n            \"value\": 0,\n            \"label\": \"example\"\n        }\n    ],\n    \"minValue\": 0,\n    \"maxValue\": 0,\n    \"description\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -740,7 +1016,7 @@
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit"
                 }
               ],
               "variable": []
@@ -814,7 +1090,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"value\": 0,\n    \"observationId\": \"example\",\n    \"comment\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"novelty\",\n    \"value\": 0.9,\n    \"traceId\": \"cdef-1234-5678-90ab\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -830,7 +1106,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&toTimestamp=&source=&operator=&value=&scoreIds=&configId=&dataType=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -853,17 +1129,22 @@
                 {
                   "key": "userId",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this userId associated to the trace."
                 },
                 {
                   "key": "name",
                   "value": "",
-                  "description": null
+                  "description": "Retrieve only scores with this name."
                 },
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
+                  "description": "Optional filter to only include scores created on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include scores created before a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "source",
@@ -884,6 +1165,16 @@
                   "key": "scoreIds",
                   "value": "",
                   "description": "Comma-separated list of score IDs to limit the results to."
+                },
+                {
+                  "key": "configId",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific configId."
+                },
+                {
+                  "key": "dataType",
+                  "value": "",
+                  "description": "Retrieve only scores with a specific dataType."
                 }
               ],
               "variable": []
@@ -966,6 +1257,52 @@
       "description": null,
       "name": "Sessions",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get sessions.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/sessions?page=&limit=&fromTimestamp=&toTimestamp=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "sessions"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "Page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include sessions created on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include sessions created before a certain datetime (ISO 8601)"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -1043,7 +1380,7 @@
           "request": {
             "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&toTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -1081,7 +1418,12 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
+                  "description": "Optional filter to only include traces with a trace.timestamp on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces with a trace.timestamp before a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "orderBy",

web/scripts/observations-backfill-calculated-cost.ts

@@ -0,0 +1,249 @@
+import "dotenv/config";
+
+import { z } from "zod";
+import { prisma, Prisma } from "@langfuse/shared/src/db";
+
+const BackfillCalculatedGenerationArgsSchema = z
+  .object({
+    batchSize: z.coerce.number().optional().default(5_000),
+    maxRowsToProcess: z.coerce.number().optional().default(Infinity), // Default to process all rows
+    maxDate: z.coerce.date().optional().default(new Date()), // Default to today
+  })
+  .strict();
+
+const backfillCalculatedGenerationCost = async () => {
+  let previousTimeout;
+  try {
+    const args = parseArgs(process.argv.slice(2));
+    const { batchSize, maxRowsToProcess, maxDate } = args;
+
+    log("Starting backfillCalculatedGenerationCost with params", args);
+
+    // Set the statement timeout
+    const newTimeout = "19min";
+    previousTimeout = await updateStatementTimeout(newTimeout, previousTimeout);
+
+    // Drop column if it exists and add temporary column
+    await addTemporaryColumnIfNotExists();
+
+    let currentDateCutoff = maxDate.toISOString();
+    let totalRowsProcessed = 0;
+
+    log("Starting batch update loop...");
+
+    // Step 3: Batch update in a loop
+    while (true) {
+      log(`Starting batch update for generations before: ${currentDateCutoff}`);
+      const startDate = Date.now();
+      const batchUpdate = await prisma.$queryRaw<
+        { start_time: Date }[]
+      >(Prisma.sql`
+        WITH batch AS (
+            SELECT o.id,
+                o.start_time,
+                o.prompt_tokens,
+                o.completion_tokens,
+                o.total_tokens,
+                o.input_cost,
+                o.output_cost,
+                o.total_cost,
+                m.id AS model_id,
+                m.input_price,
+                m.output_price,
+                m.total_price
+            FROM observations o
+            LEFT JOIN LATERAL (
+                SELECT models.id,
+                    models.input_price,
+                    models.output_price,
+                    models.total_price
+                FROM models
+                WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+                  AND models.model_name = o.internal_model
+                  AND (models.start_date < o.start_time OR models.start_date IS NULL)
+                  AND o.unit = models.unit
+                ORDER BY models.project_id, models.start_date DESC NULLS LAST
+                LIMIT 1
+            ) m ON true
+            WHERE
+                start_time <= ${currentDateCutoff}::TIMESTAMP WITH TIME ZONE AT TIME ZONE 'UTC'
+              	AND (internal_model IS NOT NULL
+                      OR input_cost IS NOT NULL
+                      OR output_cost IS NOT NULL
+                      OR total_cost IS NOT NULL) 
+            ORDER BY
+              start_time DESC
+            LIMIT ${batchSize}
+        ),
+        updated_batch AS (
+            UPDATE observations o
+            SET calculated_input_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.prompt_tokens::numeric * batch.input_price
+                        ELSE batch.input_cost
+                    END,
+                calculated_output_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN batch.completion_tokens::numeric * batch.output_price
+                        ELSE batch.output_cost
+                    END,
+                calculated_total_cost = 
+                    CASE
+                        WHEN batch.input_cost IS NULL AND batch.output_cost IS NULL AND batch.total_cost IS NULL 
+                        THEN
+                            CASE
+                                WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
+                                ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
+                            END
+                        ELSE batch.total_cost
+                    END,
+                internal_model_id = batch.model_id,
+                tmp_has_calculated_cost = TRUE
+            FROM batch
+            WHERE o.id = batch.id
+            RETURNING o.id
+        )
+        -- Get the last id of the updated batch
+        SELECT start_time FROM batch LIMIT 1 OFFSET ${batchSize - 1};
+      `);
+
+      log(`Batch update completed in ${Date.now() - startDate} ms`);
+
+      if (!batchUpdate[0]?.start_time) {
+        log(
+          `No more rows to process, breaking loop after ${totalRowsProcessed.toLocaleString()} rows processed.`,
+        );
+
+        break;
+      }
+
+      currentDateCutoff = batchUpdate[0]?.start_time.toISOString();
+      totalRowsProcessed += batchSize;
+
+      log(
+        `Total rows processed after increment: ${totalRowsProcessed.toLocaleString()} rows`,
+      );
+      if (maxRowsToProcess && totalRowsProcessed >= maxRowsToProcess) {
+        log(
+          `Max rows to process reached: ${maxRowsToProcess.toLocaleString()}, breaking loop.`,
+        );
+
+        break;
+      }
+    }
+
+    log("✅ Finished batch update loop.");
+
+    // Drop the temporary column
+    log("Dropping temporary column...");
+    await prisma.$executeRaw`ALTER TABLE observations DROP COLUMN IF EXISTS tmp_has_calculated_cost;`;
+    log("✅ Dropped temporary column");
+
+    log("✅ Finished backfillCalculatedGenerationCost");
+  } catch (err) {
+    console.error("Error executing script", err);
+  } finally {
+    // Reset the statement timeout to two minutes
+    await prisma.$executeRawUnsafe(
+      `SET statement_timeout = '${previousTimeout}';`,
+    );
+    log(
+      `Reset statement_timeout to ${previousTimeout}. Current statement_timeout: ${JSON.stringify(
+        await prisma.$queryRaw(Prisma.sql`SHOW statement_timeout;`),
+      )}`,
+    );
+
+    // Disconnect from the database
+    await prisma.$disconnect();
+    log("Disconnected from the database.");
+  }
+};
+
+function parseArgs(args: string[]) {
+  try {
+    const namedArgs: Record<string, string | boolean> = {};
+    for (let i = 0; i < args.length; i++) {
+      if (args[i].startsWith("--")) {
+        const key = args[i].slice(2);
+        const value =
+          args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : true;
+        namedArgs[key] = value;
+        if (value !== true) i++; // Skip the next argument if it was used as a value
+      }
+    }
+
+    return BackfillCalculatedGenerationArgsSchema.parse(namedArgs);
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      console.error("Validation error:", error.errors);
+    } else {
+      console.error("An unexpected error occurred:", error);
+    }
+
+    process.exit(1);
+  }
+}
+
+async function addTemporaryColumnIfNotExists() {
+  const columnExists = await prisma.$queryRaw<{ column_exists: boolean }[]>(
+    Prisma.sql`
+            SELECT EXISTS (
+                SELECT 1
+                FROM information_schema.columns
+                WHERE table_name = 'observations'
+                AND column_name = 'tmp_has_calculated_cost'
+            ) AS column_exists;
+        `,
+  );
+  if (!columnExists[0]?.column_exists) {
+    await prisma.$executeRaw`ALTER TABLE observations ADD COLUMN tmp_has_calculated_cost BOOLEAN DEFAULT FALSE;`;
+    log("✅ Added temporary column tmp_has_calculated_cost");
+  } else {
+    log(
+      "⚠️ Temporary column tmp_has_calculated_cost already exists. Continuing...",
+    );
+  }
+}
+
+type StatementTimeout = {
+  statement_timeout: string;
+};
+
+async function updateStatementTimeout(
+  newTimeout: string,
+  previousTimeout: any,
+) {
+  const [{ statement_timeout: previousTimeoutRead }] = await prisma.$queryRaw<
+    StatementTimeout[]
+  >(Prisma.sql`SHOW statement_timeout;`);
+
+  log(`Current statement_timeout read from DB: ${previousTimeoutRead}`);
+
+  if (!previousTimeoutRead || previousTimeoutRead === newTimeout) {
+    // If the statement_timeout is already set to 19 minutes, assume it was set by this script and reset it to 2 minutes
+    previousTimeout = "2min";
+  } else {
+    previousTimeout = previousTimeoutRead;
+  }
+
+  log(`Setting statement_timeout to ${newTimeout} minutes...`);
+
+  await prisma.$executeRawUnsafe(`SET statement_timeout = '${newTimeout}';`);
+
+  log(
+    `Updated statement_timeout. Current statement_timeout: ${JSON.stringify(
+      await prisma.$queryRaw(Prisma.sql`SHOW statement_timeout;`),
+    )}`,
+  );
+
+  return previousTimeout;
+}
+
+function log(message: string, ...args: any[]) {
+  console.log(new Date().toISOString(), " - ", message, ...args);
+}
+
+// Execute the script
+backfillCalculatedGenerationCost();

web/src/__tests__/datasets.servertest.ts

@@ -1,8 +1,26 @@
 /** @jest-environment node */
 
 import { prisma } from "@langfuse/shared/src/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { v4 } from "uuid";
+import {
+  GetDatasetItemV1Response,
+  GetDatasetItemsV1Response,
+  GetDatasetRunV1Response,
+  GetDatasetRunsV1Response,
+  GetDatasetV1Response,
+  GetDatasetV2Response,
+  GetDatasetsV1Response,
+  GetDatasetsV2Response,
+  PostDatasetItemsV1Response,
+  PostDatasetRunItemsV1Response,
+  PostDatasetsV1Response,
+  PostDatasetsV2Response,
+} from "@/src/features/public-api/types/datasets";
 
 describe("/api/public/datasets and /api/public/dataset-items API Endpoints", () => {
   const traceId = v4();
@@ -49,11 +67,24 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
   });
   afterEach(async () => await pruneDatabase());
 
-  it("should create and get a dataset, include special characters", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
+  it("should create and get a dataset (v1), include special characters", async () => {
+    const createRes = await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset + name",
+        description: "dataset-description",
+        metadata: { foo: "bar" },
+      },
+    );
+    expect(createRes.status).toBe(200);
+    expect(createRes.body).toMatchObject({
       name: "dataset + name",
       description: "dataset-description",
       metadata: { foo: "bar" },
+      items: [],
+      runs: [],
     });
 
     const dbDataset = await prisma.dataset.findMany({
@@ -61,62 +92,132 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
         name: "dataset + name",
       },
     });
-
     expect(dbDataset.length).toBeGreaterThan(0);
 
-    const getDataset = await makeAPICall(
+    // get dataset (v1) excluding items and runs
+    const getDatasetV1 = await makeZodVerifiedAPICall(
+      GetDatasetV1Response,
       "GET",
       `/api/public/datasets/${encodeURIComponent("dataset + name")}`,
     );
-
-    expect(getDataset.status).toBe(200);
-    expect(getDataset.body).toMatchObject({
+    expect(getDatasetV1.status).toBe(200);
+    expect(getDatasetV1.body).toMatchObject({
       name: "dataset + name",
       description: "dataset-description",
       metadata: { foo: "bar" },
+      items: [],
+      runs: [],
     });
   });
 
-  it("GET datasets", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-1",
+  it("should create and get a dataset (v2), include special characters", async () => {
+    const createRes = await makeZodVerifiedAPICall(
+      PostDatasetsV2Response,
+      "POST",
+      "/api/public/v2/datasets",
+      {
+        name: "dataset + name + v2",
+        description: "dataset-description",
+        metadata: { foo: "bar" },
+      },
+    );
+    expect(createRes.status).toBe(200);
+    expect(createRes.body).toMatchObject({
+      name: "dataset + name + v2",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
     });
 
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-2",
+    const dbDataset = await prisma.dataset.findMany({
+      where: {
+        name: "dataset + name + v2",
+      },
     });
+    expect(dbDataset.length).toBeGreaterThan(0);
+
+    // get dataset (v2) excluding items and runs
+    const getDatasetV2 = await makeZodVerifiedAPICall(
+      GetDatasetV2Response,
+      "GET",
+      `/api/public/v2/datasets/${encodeURIComponent("dataset + name + v2")}`,
+    );
+    expect(getDatasetV2.status).toBe(200);
+    expect(getDatasetV2.body).toMatchObject({
+      name: "dataset + name + v2",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
+    });
+    expect(getDatasetV2.body).not.toHaveProperty("items");
+    expect(getDatasetV2.body).not.toHaveProperty("runs");
+  });
+
+  it("GET datasets (v1 & v2)", async () => {
+    // v1 post
+    await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset-name-1",
+        description: "dataset-description-1",
+        metadata: { key: "value" },
+      },
+    );
+    // v2 post
+    await makeZodVerifiedAPICall(
+      PostDatasetsV2Response,
+      "POST",
+      "/api/public/v2/datasets",
+      {
+        name: "dataset-name-2",
+      },
+    );
 
     const datasetItemId = v4();
 
-    const createItemRes = await makeAPICall<{
-      datasetName: string; // field that can break if the API changes as it is not a db column
-    }>("POST", "/api/public/dataset-items", {
-      datasetName: "dataset-name-2",
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-      metadata: { key: "value-dataset-item" },
-      id: datasetItemId,
-    });
+    const createItemRes = await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        datasetName: "dataset-name-2",
+        input: { key: "value" },
+        expectedOutput: { key: "value" },
+        metadata: { key: "value-dataset-item" },
+        id: datasetItemId,
+      },
+    );
 
     expect(createItemRes.status).toBe(200);
     expect(createItemRes.body).toMatchObject({
       datasetName: "dataset-name-2", // not included in db table
     });
 
-    await makeAPICall("POST", "/api/public/dataset-run-items", {
-      datasetItemId: datasetItemId,
-      observationId: observationId,
-      runName: "test-run",
-      metadata: { key: "value" },
-    });
+    await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: datasetItemId,
+        observationId: observationId,
+        runName: "test-run",
+        metadata: { key: "value" },
+      },
+    );
 
-    const getDatasets = await makeAPICall("GET", `/api/public/datasets`);
+    const getDatasetsV1 = await makeZodVerifiedAPICall(
+      GetDatasetsV1Response,
+      "GET",
+      `/api/public/datasets`,
+    );
 
-    expect(getDatasets.status).toBe(200);
-    expect(getDatasets.body).toMatchObject({
+    expect(getDatasetsV1.status).toBe(200);
+    expect(getDatasetsV1.body).toMatchObject({
       data: expect.arrayContaining([
         expect.objectContaining({
           name: "dataset-name-1",
+          description: "dataset-description-1",
+          metadata: { key: "value" },
           items: [],
           runs: [],
         }),
@@ -131,87 +232,257 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
         page: 1,
       }),
     });
+
+    const getDatasetsV2 = await makeZodVerifiedAPICall(
+      GetDatasetsV2Response,
+      "GET",
+      `/api/public/v2/datasets`,
+    );
+
+    expect(getDatasetsV2.status).toBe(200);
+    expect(getDatasetsV2.body).toMatchObject({
+      data: expect.arrayContaining([
+        expect.objectContaining({
+          name: "dataset-name-1",
+          description: "dataset-description-1",
+          metadata: { key: "value" },
+        }),
+        expect.objectContaining({
+          name: "dataset-name-2",
+          description: null,
+          metadata: null,
+        }),
+      ]),
+      meta: expect.objectContaining({
+        totalItems: 2,
+        page: 1,
+      }),
+    });
   });
 
-  it("should create and get a dataset item (via datasets and individually)", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name",
-    });
-    await makeAPICall("POST", "/api/public/dataset-items", {
-      datasetName: "dataset-name",
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-      metadata: { key: "value-dataset-item" },
-      sourceTraceId: traceId,
-      sourceObservationId: observationId,
-    });
-    const dbDatasetItem = await prisma.datasetItem.findFirst({
+  it("should create and get a dataset items (via datasets (v1), individually, and as a list)", async () => {
+    await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset-name",
+      },
+    );
+    for (let i = 0; i < 5; i++) {
+      await makeZodVerifiedAPICall(
+        PostDatasetItemsV1Response,
+        "POST",
+        "/api/public/dataset-items",
+        {
+          datasetName: "dataset-name",
+          input: { key: "value" },
+          expectedOutput: { key: "value" },
+          metadata: { key: "value-dataset-item" },
+          sourceTraceId: i % 2 === 0 ? traceId : undefined,
+          sourceObservationId: i % 2 === 0 ? observationId : undefined,
+        },
+      );
+    }
+    const dbDatasetItems = await prisma.datasetItem.findMany({
       where: {
         dataset: {
           name: "dataset-name",
         },
       },
+      orderBy: {
+        createdAt: "desc",
+      },
     });
+    expect(dbDatasetItems.length).toBe(5);
+    const dbDatasetItemsApiResponseFormat = dbDatasetItems.map((item) => ({
+      ...item,
+      createdAt: item.createdAt.toISOString(),
+      updatedAt: item.updatedAt.toISOString(),
+      datasetName: "dataset-name",
+    }));
 
-    expect(dbDatasetItem).not.toBeNull();
+    // add another dataset to test the list endpoint
+    await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset-name-other",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        datasetName: "dataset-name-other",
+        input: { key: "value" },
+        expectedOutput: { key: "value" },
+      },
+    );
+    const dbDatasetItemsOther = await prisma.datasetItem.findMany({
+      where: {
+        dataset: {
+          name: "dataset-name-other",
+        },
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+    expect(dbDatasetItemsOther.length).toBe(1);
+    const dbDatasetItemsOtherApiResponseFormat = dbDatasetItemsOther.map(
+      (item) => ({
+        ...item,
+        createdAt: item.createdAt.toISOString(),
+        updatedAt: item.updatedAt.toISOString(),
+        datasetName: "dataset-name-other",
+      }),
+    );
+    const dbDatasetItemsAllApiResponseFormat = [
+      ...dbDatasetItemsApiResponseFormat,
+      ...dbDatasetItemsOtherApiResponseFormat,
+    ].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); // createdAt desc
 
-    const getDataset = await makeAPICall(
+    // Get dataset (v1) includes list of items
+    const getDataset = await makeZodVerifiedAPICall(
+      GetDatasetV1Response,
       "GET",
       `/api/public/datasets/dataset-name`,
     );
     expect(getDataset.status).toBe(200);
     expect(getDataset.body).toMatchObject({
       name: "dataset-name",
-      items: [
-        {
-          id: dbDatasetItem!.id,
-          input: { key: "value" },
-          expectedOutput: { key: "value" },
-          metadata: { key: "value-dataset-item" },
-          datasetName: "dataset-name", // not included in db table
-        },
-      ],
+      items: dbDatasetItemsApiResponseFormat,
     });
 
-    const getDatasetItem = await makeAPICall(
+    // Get List
+    const getDatasetItemsAll = await makeZodVerifiedAPICall(
+      GetDatasetItemsV1Response,
       "GET",
-      `/api/public/dataset-items/${dbDatasetItem!.id}`,
+      `/api/public/dataset-items`,
+    );
+    expect(getDatasetItemsAll.status).toBe(200);
+    expect(getDatasetItemsAll.body).toMatchObject({
+      data: dbDatasetItemsAllApiResponseFormat,
+      meta: expect.objectContaining({
+        totalItems: 6,
+        page: 1,
+      }),
+    });
+    // Get List, check pagination
+    const getDatasetItemsAllPage2 = await makeZodVerifiedAPICall(
+      GetDatasetItemsV1Response,
+      "GET",
+      `/api/public/dataset-items?page=2&limit=1`,
+    );
+    expect(getDatasetItemsAllPage2.status).toBe(200);
+    expect(getDatasetItemsAllPage2.body).toMatchObject({
+      data: dbDatasetItemsAllApiResponseFormat.slice(1, 2),
+      meta: expect.objectContaining({
+        totalItems: 6,
+        page: 2,
+        totalPages: 6,
+        limit: 1,
+      }),
+    });
+    // Get filtered list by datasetName
+    const getDatasetItems = await makeZodVerifiedAPICall(
+      GetDatasetItemsV1Response,
+      "GET",
+      `/api/public/dataset-items?datasetName=dataset-name`,
+    );
+    expect(getDatasetItems.status).toBe(200);
+    expect(getDatasetItems.body).toMatchObject({
+      data: dbDatasetItemsApiResponseFormat,
+      meta: expect.objectContaining({
+        totalItems: 5,
+        page: 1,
+      }),
+    });
+    // Get filtered list by sourceTraceId
+    const getDatasetItemsTrace = await makeZodVerifiedAPICall(
+      GetDatasetItemsV1Response,
+      "GET",
+      `/api/public/dataset-items?sourceTraceId=${traceId}`,
+    );
+    expect(getDatasetItemsTrace.status).toBe(200);
+    expect(getDatasetItemsTrace.body).toMatchObject({
+      data: dbDatasetItemsApiResponseFormat.filter(
+        (item) => item.sourceTraceId === traceId,
+      ),
+      meta: expect.objectContaining({
+        totalItems: 3,
+        page: 1,
+      }),
+    });
+    // Get filtered list by sourceObservationId
+    const getDatasetItemsObservation = await makeZodVerifiedAPICall(
+      GetDatasetItemsV1Response,
+      "GET",
+      `/api/public/dataset-items?sourceObservationId=${observationId}`,
+    );
+    expect(getDatasetItemsObservation.status).toBe(200);
+    expect(getDatasetItemsObservation.body).toMatchObject({
+      data: dbDatasetItemsApiResponseFormat.filter(
+        (item) => item.sourceObservationId === observationId,
+      ),
+      meta: expect.objectContaining({
+        totalItems: 3,
+        page: 1,
+      }),
+    });
+
+    // Get single item
+    const singleItem = dbDatasetItemsApiResponseFormat[0];
+    const getDatasetItem = await makeZodVerifiedAPICall(
+      GetDatasetItemV1Response,
+      "GET",
+      `/api/public/dataset-items/${singleItem.id}`,
     );
     expect(getDatasetItem.status).toBe(200);
-    expect(getDatasetItem.body).toMatchObject({
-      id: dbDatasetItem!.id,
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-      metadata: { key: "value-dataset-item" },
-      datasetName: "dataset-name", // not included in db table
-      sourceTraceId: traceId,
-      sourceObservationId: observationId,
-    });
+    expect(getDatasetItem.body).toMatchObject(singleItem);
   });
 
   it("should upsert a dataset item", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name",
-    });
+    await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset-name",
+      },
+    );
 
-    const item1 = await makeAPICall("POST", "/api/public/dataset-items", {
-      id: "dataset-item-id",
-      datasetName: "dataset-name",
-      input: { key: "value" },
-      metadata: { key: "value-dataset-item" },
-    });
+    const item1 = await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        id: "dataset-item-id",
+        datasetName: "dataset-name",
+        input: { key: "value" },
+        metadata: { key: "value-dataset-item" },
+      },
+    );
     expect(item1.status).toBe(200);
     expect(item1.body).toMatchObject({
       id: "dataset-item-id",
     });
 
-    const item2 = await makeAPICall("POST", "/api/public/dataset-items", {
-      id: "dataset-item-id",
-      datasetName: "dataset-name",
-      input: { key: "value2" },
-      metadata: ["hello-world"],
-      status: "ARCHIVED",
-    });
+    const item2 = await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        id: "dataset-item-id",
+        datasetName: "dataset-name",
+        input: { key: "value2" },
+        metadata: ["hello-world"],
+        status: "ARCHIVED",
+      },
+    );
     expect(item2.status).toBe(200);
     expect(item2.body).toMatchObject({
       id: "dataset-item-id",
@@ -230,7 +501,8 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
   });
 
   it("should create and get a dataset run, include special characters", async () => {
-    const dataset = await makeAPICall<{ id: string }>(
+    const dataset = await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
       "POST",
       "/api/public/datasets",
       {
@@ -241,12 +513,17 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dataset.body).toMatchObject({
       name: "dataset name",
     });
-    await makeAPICall("POST", "/api/public/dataset-items", {
-      datasetName: "dataset name",
-      id: "dataset-item-id",
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-    });
+    await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        datasetName: "dataset name",
+        id: "dataset-item-id",
+        input: { key: "value" },
+        expectedOutput: { key: "value" },
+      },
+    );
     const traceId = v4();
     const observationId = v4();
     const response = await makeAPICall("POST", "/api/public/ingestion", {
@@ -285,7 +562,8 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     });
     expect(response.status).toBe(207);
 
-    const runItemObservation = await makeAPICall(
+    const runItemObservation = await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
       "POST",
       "/api/public/dataset-run-items",
       {
@@ -315,9 +593,9 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       traceId: traceId,
     });
 
-    const getRunAPI = await makeAPICall(
+    const getRunAPI = await makeZodVerifiedAPICall(
+      GetDatasetRunV1Response,
       "GET",
-
       `/api/public/datasets/${encodeURIComponent("dataset name")}/runs/${encodeURIComponent("run + only + observation")}`,
     );
     expect(getRunAPI.status).toBe(200);
@@ -337,14 +615,17 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       ]),
     });
 
-    const runItemTrace = await makeAPICall<{
-      datasetRunName: string; // field that can break if the API changes as it is not a db column
-    }>("POST", "/api/public/dataset-run-items", {
-      datasetItemId: "dataset-item-id",
-      traceId: traceId,
-      runName: "run-only-trace",
-      metadata: { key: "value" },
-    });
+    const runItemTrace = await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        traceId: traceId,
+        runName: "run-only-trace",
+        metadata: { key: "value" },
+      },
+    );
 
     expect(runItemTrace.status).toBe(200);
     expect(runItemTrace.body).toMatchObject({
@@ -369,7 +650,8 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       observationId: null,
     });
 
-    const runItemBoth = await makeAPICall(
+    const runItemBoth = await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
       "POST",
       "/api/public/dataset-run-items",
       {
@@ -399,6 +681,149 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     });
   });
 
+  it("GET /api/public/datasets/{datasetName}/runs", async () => {
+    // create multiple runs
+    await makeZodVerifiedAPICall(
+      PostDatasetsV1Response,
+      "POST",
+      "/api/public/datasets",
+      {
+        name: "dataset-name",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostDatasetItemsV1Response,
+      "POST",
+      "/api/public/dataset-items",
+      {
+        datasetName: "dataset-name",
+        id: "dataset-item-id",
+        input: { key: "value" },
+        expectedOutput: { key: "value" },
+      },
+    );
+    const traceId = v4();
+    const observationId = v4();
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+        {
+          id: v4(),
+          type: "observation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: observationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            startTime: "2021-01-01T00:00:00.000Z",
+            endTime: "2021-01-01T00:00:00.000Z",
+            modelParameters: { key: "value" },
+            input: { key: "value" },
+            metadata: { key: "value" },
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(response.status).toBe(207);
+
+    await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        traceId: traceId,
+        observationId: observationId,
+        runName: "run-1",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        traceId: traceId,
+        observationId: observationId,
+        runName: "run-2",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostDatasetRunItemsV1Response,
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        traceId: traceId,
+        observationId: observationId,
+        runName: "run-3",
+      },
+    );
+
+    // check runs in db
+    const dbRuns = await prisma.datasetRuns.findMany({
+      where: {
+        dataset: { name: "dataset-name" },
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+    expect(dbRuns.length).toBe(3);
+    const dbRunsApiResponseFormat = dbRuns.map((run) => ({
+      ...run,
+      createdAt: run.createdAt.toISOString(),
+      updatedAt: run.updatedAt.toISOString(),
+      datasetName: "dataset-name",
+    }));
+
+    // test get runs
+    const getRuns = await makeZodVerifiedAPICall(
+      GetDatasetRunsV1Response,
+      "GET",
+      `/api/public/datasets/dataset-name/runs`,
+    );
+    expect(getRuns.status).toBe(200);
+    expect(getRuns.body).toMatchObject({
+      data: dbRunsApiResponseFormat,
+      meta: expect.objectContaining({
+        totalItems: 3,
+        page: 1,
+      }),
+    });
+
+    // test runs with pagination
+    const getRunsPage2 = await makeZodVerifiedAPICall(
+      GetDatasetRunsV1Response,
+      "GET",
+      `/api/public/datasets/dataset-name/runs?page=2&limit=1`,
+    );
+    expect(getRunsPage2.status).toBe(200);
+    expect(getRunsPage2.body).toMatchObject({
+      data: dbRunsApiResponseFormat.slice(1, 2),
+      meta: expect.objectContaining({
+        totalItems: 3,
+        page: 2,
+        totalPages: 3,
+        limit: 1,
+      }),
+    });
+  });
+
   it("dataset-run-items should fail when neither trace nor observation provided", async () => {
     const response = await makeAPICall(
       "POST",

web/src/__tests__/events.servertest.ts

@@ -1,8 +1,13 @@
 /** @jest-environment node */
 
 import { prisma } from "@langfuse/shared/src/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import { PostEventsV1Response } from "@/src/features/public-api/types/events";
 
 describe("/api/public/events API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -12,15 +17,20 @@ describe("/api/public/events API Endpoint", () => {
 
     const traceId = uuidv4();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -32,16 +42,21 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbTrace[0]?.id).toBe(traceId);
 
     const eventId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceId: traceId,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceId: traceId,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -66,16 +81,21 @@ describe("/api/public/events API Endpoint", () => {
     const traceId = uuidv4();
     const eventId = uuidv4();
 
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceId: traceId,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceId: traceId,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -93,15 +113,20 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbEvent?.metadata).toEqual({ meta: "value" });
     expect(dbEvent?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -118,15 +143,20 @@ describe("/api/public/events API Endpoint", () => {
 
     const traceId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     expect(response.status).toBe(200);
 
@@ -141,17 +171,22 @@ describe("/api/public/events API Endpoint", () => {
     expect(dbTrace[0]?.id).not.toBe(traceId);
 
     const eventId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: eventId,
-      traceIdType: "EXTERNAL",
-      traceId: dbTrace[0]?.id,
-      name: "event-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: eventId,
+        traceIdType: "EXTERNAL",
+        traceId: dbTrace[0]?.id,
+        name: "event-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createEvent.status).toBe(200);
     const dbEvent = await prisma.observation.findUnique({
@@ -174,16 +209,21 @@ describe("/api/public/events API Endpoint", () => {
     const eventName = uuidv4();
 
     const spanId = uuidv4();
-    const createEvent = await makeAPICall("POST", "/api/public/events", {
-      id: spanId,
-      name: eventName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      output: { output: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createEvent = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: spanId,
+        name: eventName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        output: { output: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -215,14 +255,19 @@ describe("/api/public/events API Endpoint", () => {
     const generationName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/events", {
-      id: spanId,
-      name: generationName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      input: { key: "value" },
-      metadata: { key: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostEventsV1Response,
+      "POST",
+      "/api/public/events",
+      {
+        id: spanId,
+        name: generationName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        input: { key: "value" },
+        metadata: { key: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbEvent = await prisma.observation.findFirstOrThrow({
       where: {

web/src/__tests__/generations.servertest.ts

@@ -2,9 +2,19 @@
 
 import { v4 as uuidv4 } from "uuid";
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import {
+  PatchGenerationsV1Response,
+  PostGenerationsV1Body,
+  PostGenerationsV1Response,
+} from "@/src/features/public-api/types/generations";
 
 describe("/api/public/generations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -90,15 +100,20 @@ describe("/api/public/generations API Endpoint", () => {
 
       const traceId = uuidv4();
 
-      await makeAPICall("POST", "/api/public/traces", {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      });
+      await makeZodVerifiedAPICall(
+        PostTracesV1Response,
+        "POST",
+        "/api/public/traces",
+        {
+          id: traceId,
+          name: "trace-name",
+          userId: "user-1",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          metadata: { key: "value" },
+          release: "1.0.0",
+          version: "2.0.0",
+        },
+      );
 
       const dbTrace = await prisma.trace.findMany({
         where: {
@@ -110,7 +125,8 @@ describe("/api/public/generations API Endpoint", () => {
       expect(dbTrace[0]?.id).toBe(traceId);
 
       const generationId = uuidv4();
-      const createGeneration = await makeAPICall(
+      const createGeneration = await makeZodVerifiedAPICall(
+        PostGenerationsV1Response,
         "POST",
         "/api/public/generations",
         {
@@ -164,7 +180,8 @@ describe("/api/public/generations API Endpoint", () => {
     const traceId = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -201,15 +218,20 @@ describe("/api/public/generations API Endpoint", () => {
     expect(dbGeneration?.metadata).toEqual({ key: "value" });
     expect(dbGeneration?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -224,15 +246,20 @@ describe("/api/public/generations API Endpoint", () => {
   it("should create generation after trace ignoring externalId", async () => {
     const traceId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     expect(response.status).toBe(200);
 
@@ -247,7 +274,8 @@ describe("/api/public/generations API Endpoint", () => {
     expect(dbTrace[0]?.id).not.toBe(traceId);
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -290,7 +318,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -340,7 +369,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -368,7 +398,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationId2 = uuidv4();
     const generationName2 = uuidv4();
 
-    const createGeneration2 = await makeAPICall(
+    const createGeneration2 = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -401,7 +432,8 @@ describe("/api/public/generations API Endpoint", () => {
 
     const generationId = uuidv4();
     const externalTraceId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -437,7 +469,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -488,7 +521,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {
@@ -506,7 +540,8 @@ describe("/api/public/generations API Endpoint", () => {
 
     expect(createGeneration.status).toBe(200);
 
-    const updateGeneration = await makeAPICall(
+    const updateGeneration = await makeZodVerifiedAPICall(
+      PatchGenerationsV1Response,
       "PATCH",
       "/api/public/generations",
       {
@@ -540,7 +575,8 @@ describe("/api/public/generations API Endpoint", () => {
     const generationName = uuidv4();
 
     const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
+    const createGeneration = await makeZodVerifiedAPICall(
+      PostGenerationsV1Body,
       "POST",
       "/api/public/generations",
       {

web/src/__tests__/ingestion.servertest.ts

@@ -57,6 +57,17 @@ describe("/api/public/ingestion API Endpoint", () => {
       expectedCompletionTokens: 2,
       expectedTotalTokens: 3,
     },
+    {
+      usage: {
+        input: 1,
+        output: 2,
+        unit: ModelUsageUnit.Requests,
+      },
+      expectedUnit: ModelUsageUnit.Requests,
+      expectedPromptTokens: 1,
+      expectedCompletionTokens: 2,
+      expectedTotalTokens: 3,
+    },
     {
       usage: {
         input: 30,

web/src/__tests__/markdown.clienttest.ts

@@ -0,0 +1,68 @@
+import { containsAnyMarkdown } from "@/src/components/schemas/MarkdownSchema";
+
+describe("containsAnyMarkdown Function", () => {
+  it("Detects simple bold using asterisks", () => {
+    expect(containsAnyMarkdown("This is **bold** text")).toBe(true);
+  });
+
+  it("Detects simple italics using asterisks", () => {
+    expect(containsAnyMarkdown("This is *italic* text")).toBe(true);
+  });
+
+  it("Detects bold using underscores", () => {
+    expect(containsAnyMarkdown("This is __bold__ text")).toBe(true);
+  });
+
+  it("Detects italics using underscores", () => {
+    expect(containsAnyMarkdown("This is _italic_ text")).toBe(true);
+  });
+
+  it("Detects inline code with backticks", () => {
+    expect(containsAnyMarkdown("This is `code` inline")).toBe(true);
+  });
+
+  it("Detects fenced code blocks", () => {
+    const codeBlock = "```\nlet x = 10;\n```";
+    expect(containsAnyMarkdown(codeBlock)).toBe(true);
+  });
+
+  it("Detects headers", () => {
+    expect(containsAnyMarkdown("# Header 1")).toBe(true);
+    expect(containsAnyMarkdown("## Header 2")).toBe(true);
+  });
+
+  it("Detects unordered lists", () => {
+    expect(containsAnyMarkdown("- List item 1\n- List item 2")).toBe(true);
+  });
+
+  it("Detects ordered lists", () => {
+    expect(containsAnyMarkdown("1. First item\n2. Second item")).toBe(true);
+  });
+
+  it("Detects blockquotes", () => {
+    expect(containsAnyMarkdown("> This is a blockquote")).toBe(true);
+  });
+
+  it("Detects links", () => {
+    expect(containsAnyMarkdown("[Google](http://www.google.com)")).toBe(true);
+  });
+
+  it("Detects images", () => {
+    expect(containsAnyMarkdown("![Alt text](http://url/to/img.png)")).toBe(
+      true,
+    );
+  });
+
+  it("Returns false for non-markdown text", () => {
+    expect(
+      containsAnyMarkdown("This is plain text without any markdown syntax"),
+    ).toBe(false);
+    expect(containsAnyMarkdown("12345")).toBe(false);
+    expect(containsAnyMarkdown("Simple text.")).toBe(false);
+  });
+
+  it("Detects combination of markdown elements", () => {
+    const complexMarkdown = "**Bold** and _italic_ and `code`";
+    expect(containsAnyMarkdown(complexMarkdown)).toBe(true);
+  });
+});

web/src/__tests__/metrics-api.servertest.ts

@@ -1,4 +1,10 @@
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import { PostGenerationsV1Response } from "@/src/features/public-api/types/generations";
+import { GetMetricsDailyV1Response } from "@/src/features/public-api/types/metrics";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 import { v4 as uuidv4 } from "uuid";
 
 describe("/api/public/metrics/daily API Endpoint", () => {
@@ -6,53 +12,89 @@ describe("/api/public/metrics/daily API Endpoint", () => {
   afterEach(async () => await pruneDatabase());
 
   it("should handle daily metrics correctly", async () => {
-    await pruneDatabase();
-
     // Create traces with observations on different days
     const traceId1 = uuidv4();
     const traceId2 = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId1,
-      timestamp: "2021-01-01T00:00:00.000Z",
-      name: "trace-day-1",
-      userId: "user-daily-metrics",
-      projectId: "project-daily-metrics",
-    });
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId2,
-      timestamp: "2021-01-02T00:00:00.000Z",
-      name: "trace-day-2",
-      userId: "user-daily-metrics",
-      projectId: "project-daily-metrics",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId1,
+        timestamp: "2021-01-01T00:00:00.000Z",
+        name: "trace-day-1",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId2,
+        timestamp: "2021-01-02T00:00:00.000Z",
+        name: "trace-day-2",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
 
     // Simulate observations with usage metrics on different days
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId1,
-      model: "modelA",
-      usage: { input: 100, output: 200, total: 300 },
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:01:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId2,
-      model: "modelB",
-      usage: { input: 333 },
-      startTime: "2021-01-02T00:00:00.000Z",
-      endTime: "2021-01-02T00:02:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId2,
-      model: "modelC",
-      usage: { input: 666, output: 777, totalCost: 1024.22 },
-      startTime: "2021-01-02T00:00:00.000Z",
-      endTime: "2021-01-02T00:04:00.000Z",
-    });
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId1,
+        model: "modelA",
+        usage: { input: 100, output: 200, total: 300 },
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:01:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        model: "modelB",
+        usage: { input: 333 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:02:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        model: "modelC",
+        usage: { input: 666, output: 777, totalCost: 1024.22 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:04:00.000Z",
+      },
+    );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId2,
+        usage: { output: 300 },
+        startTime: "2021-01-02T00:00:00.000Z",
+        endTime: "2021-01-02T00:04:00.000Z",
+      },
+    );
 
     // Retrieve the daily metrics
-    const dailyMetricsResponse = await makeAPICall<{
-      data: Array<Record<string, unknown>>;
-    }>("GET", `/api/public/metrics/daily`);
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
     const dailyMetricsData = dailyMetricsResponse.body.data;
 
     // Check if the daily metrics are calculated correctly
@@ -81,6 +123,15 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         countTraces: 1,
         totalCost: 1024.22,
       },
+      {
+        model: null,
+        countObservations: 1,
+        countTraces: 1,
+        inputUsage: 0,
+        outputUsage: 300,
+        totalCost: 0,
+        totalUsage: 300,
+      },
     ]);
 
     if (!dailyMetricsData[1])
@@ -100,4 +151,48 @@ describe("/api/public/metrics/daily API Endpoint", () => {
       },
     ]);
   });
+
+  it("should handle daily metrics correctly when there is no data", async () => {
+    // Retrieve the daily metrics
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
+    const dailyMetricsData = dailyMetricsResponse.body.data;
+
+    // Check if the daily metrics are calculated correctly
+    expect(dailyMetricsData).toHaveLength(0); // No data
+  });
+
+  it("should handle daily metrics correctly when there is just a trace", async () => {
+    const traceId1 = uuidv4();
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId1,
+        timestamp: "2021-01-01T00:00:00.000Z",
+        name: "trace-day-1",
+        userId: "user-daily-metrics",
+        projectId: "project-daily-metrics",
+      },
+    );
+
+    // Retrieve the daily metrics
+    const dailyMetricsResponse = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily`,
+    );
+    const dailyMetricsData = dailyMetricsResponse.body.data;
+
+    // Check if the daily metrics are calculated correctly
+    expect(dailyMetricsData).toHaveLength(1);
+    expect(dailyMetricsData[0].date).toBe("2021-01-01");
+    expect(dailyMetricsData[0].countTraces).toBe(1);
+    expect(dailyMetricsData[0].totalCost).toEqual(0);
+    expect(dailyMetricsData[0].usage).toEqual([]);
+  });
 });

web/src/__tests__/model-definitions.servertest.ts

@@ -0,0 +1,221 @@
+/** @jest-environment node */
+
+import { prisma } from "@langfuse/shared/src/db";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  DeleteModelV1Response,
+  GetModelV1Response,
+  GetModelsV1Response,
+  PostModelsV1Response,
+} from "@/src/features/public-api/types/models";
+
+describe("/models API Endpoints", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+    // create some default models that do not belong to a project
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0020",
+        outputPrice: "0.0040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: "TOKENS",
+      },
+    });
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("GET /models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(2);
+    expect(models.body.data[0]).toMatchObject({
+      isLangfuseManaged: true,
+      modelName: "gpt-3.5-turbo",
+    });
+  });
+
+  it("GET /models pagination", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models?page=2&limit=1",
+    );
+    expect(models.status).toBe(200);
+    expect(models.body.data.length).toBe(1);
+    expect(models.body.meta).toMatchObject({
+      page: 2,
+      totalPages: 2,
+      limit: 1,
+      totalItems: 2,
+    });
+  });
+
+  it("Create and get custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+    expect(customModel.body.isLangfuseManaged).toBe(false);
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    const getModel = await makeZodVerifiedAPICall(
+      GetModelV1Response,
+      "GET",
+      `/api/public/models/${customModel.body.id}`,
+    );
+    expect(getModel.body.id).toBe(customModel.body.id);
+    expect(getModel.body).toMatchObject({
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      startDate: new Date("2023-12-01").toISOString(),
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      isLangfuseManaged: false,
+    });
+  });
+
+  it("Post model with invalid matchPattern", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Post model without prices or tokenizer", async () => {
+    await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        unit: "TOKENS",
+      },
+    );
+  });
+
+  it("Post model with missing fields", async () => {
+    const { status } = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+      // missing unit
+    });
+    expect(status).toBe(400);
+  });
+
+  it("Post model with invalid price (input and total cost)", async () => {
+    const customModel = await makeAPICall("POST", "/api/public/models", {
+      modelName: "gpt-3.5-turbo",
+      matchPattern: "[][", // brackets not balanced
+      startDate: "2023-12-01",
+      inputPrice: 0.002,
+      outputPrice: 0.004,
+      totalPrice: 0.1,
+      unit: "TOKENS",
+      tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+    });
+    expect(customModel.status).toBe(400);
+  });
+
+  it("Cannot delete built-in models", async () => {
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(2);
+
+    const deleteModel = await makeAPICall(
+      "DELETE",
+      `/api/public/models/${models.body.data[0].id}`,
+    );
+    expect(deleteModel.status).toBe(404);
+  });
+
+  it("Delete custom model", async () => {
+    const customModel = await makeZodVerifiedAPICall(
+      PostModelsV1Response,
+      "POST",
+      "/api/public/models",
+      {
+        modelName: "gpt-3.5-turbo",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: "2023-12-01",
+        inputPrice: 0.002,
+        outputPrice: 0.004,
+        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+      },
+    );
+
+    const models = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(models.body.data.length).toBe(3);
+
+    await makeZodVerifiedAPICall(
+      DeleteModelV1Response,
+      "DELETE",
+      `/api/public/models/${customModel.body.id}`,
+    );
+
+    const modelsAfterDelete = await makeZodVerifiedAPICall(
+      GetModelsV1Response,
+      "GET",
+      "/api/public/models",
+    );
+    expect(modelsAfterDelete.body.data.length).toBe(2);
+  });
+});

web/src/__tests__/observations.servertest.ts

@@ -2,10 +2,16 @@
 
 import { v4 as uuidv4 } from "uuid";
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared";
+import {
+  GetObservationV1Response,
+  GetObservationsV1Response,
+} from "@/src/features/public-api/types/observations";
 
 describe("/api/public/observations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -79,7 +85,8 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
-    const fetchedObservations = await makeAPICall(
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
       "GET",
       "/api/public/observations",
       undefined,
@@ -87,13 +94,6 @@ describe("/api/public/observations API Endpoint", () => {
 
     expect(fetchedObservations.status).toBe(200);
 
-    if (!isObservationList(fetchedObservations.body)) {
-      throw new Error(
-        "Expected body to be an array of observations" +
-          JSON.stringify(fetchedObservations.body),
-      );
-    }
-
     expect(fetchedObservations.body.data.length).toBe(1);
     expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
     expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
@@ -183,7 +183,8 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
-    const fetchedObservations = await makeAPICall(
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
       "GET",
       "/api/public/observations?type=GENERATION",
       undefined,
@@ -193,10 +194,6 @@ describe("/api/public/observations API Endpoint", () => {
 
     expect(fetchedObservations.status).toBe(200);
 
-    if (!isObservationList(fetchedObservations.body)) {
-      throw new Error("Expected body to be an array of observations");
-    }
-
     expect(fetchedObservations.body.data.length).toBe(1);
     expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
     expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
@@ -205,37 +202,236 @@ describe("/api/public/observations API Endpoint", () => {
   });
 });
 
-const isObservationList = (val: unknown): val is ObservationResponse => {
-  return (
-    typeof val === "object" &&
-    val !== null &&
-    "data" in val &&
-    Array.isArray(val.data) &&
-    val.data.every(
-      (element) =>
-        typeof element === "object" &&
-        element !== null &&
-        "id" in element &&
-        "traceId" in element &&
-        "name" in element &&
-        "startTime" in element &&
-        "endTime" in element &&
-        "model" in element &&
-        "input" in element &&
-        "output" in element &&
-        "metadata" in element &&
-        "version" in element &&
-        "modelId" in element &&
-        "inputPrice" in element &&
-        "outputPrice" in element &&
-        "totalPrice" in element &&
-        "calculatedInputCost" in element &&
-        "calculatedOutputCost" in element &&
-        "calculatedTotalCost" in element,
-    )
-  );
-};
+it("GET /observations with timestamp filters and pagination", async () => {
+  await prisma.trace.create({
+    data: {
+      id: "trace-id",
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+  await prisma.observation.createMany({
+    data: [
+      {
+        id: "observation-2021-01-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-01-01T00:00:00.000Z"),
+        endTime: new Date("2021-01-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "GENERATION",
+      },
+      {
+        id: "observation-2021-02-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-02-01T00:00:00.000Z"),
+        endTime: new Date("2021-02-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "SPAN",
+      },
+      {
+        id: "observation-2021-03-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-03-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "EVENT",
+      },
+      {
+        id: "observation-2021-04-01",
+        traceId: "trace-id",
+        name: "generation-name",
+        startTime: new Date("2021-04-01T00:00:00.000Z"),
+        endTime: new Date("2021-04-01T00:00:00.000Z"),
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "GENERATION",
+      },
+    ],
+  });
 
-type ObservationResponse = {
-  data: ObservationView[];
-};
+  const fromTimestamp = "2021-02-01T00:00:00.000Z";
+  const toTimestamp = "2021-04-01T00:00:00.000Z";
+
+  // Test with both fromTimestamp and toTimestamp
+  let fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(2);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(2);
+
+  // Test with only fromTimestamp
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?fromStartTime=${fromTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(3);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+  // Test with only toTimestamp
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?toStartTime=${toTimestamp}`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(3);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
+  expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+  // test pagination only
+  fetchedObservations = await makeZodVerifiedAPICall(
+    GetObservationsV1Response,
+    "GET",
+    `/api/public/observations?limit=1&page=2`,
+    undefined,
+  );
+
+  expect(fetchedObservations.body.data.length).toBe(1);
+  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+  expect(fetchedObservations.body.meta).toMatchObject({
+    totalItems: 4,
+    totalPages: 4,
+    page: 2,
+    limit: 1,
+  });
+});
+
+it("Get a single EVENT from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  // EVENT
+  const eventId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: eventId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      type: "EVENT",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getEventRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + eventId,
+  );
+  expect(getEventRes.body).toMatchObject({
+    id: eventId,
+    traceId: traceId,
+    type: "EVENT",
+  });
+});
+it("Get a single GENERATION from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  const generationId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: generationId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      endTime: new Date("2021-01-01T00:00:00.000Z"),
+      model: "gpt-3.5-turbo",
+      internalModel: "gpt-3.5-turbo",
+      modelParameters: { key: "value" },
+      input: { key: "input" },
+      output: { key: "output" },
+      promptTokens: 10,
+      completionTokens: 20,
+      totalTokens: 30,
+      version: "2.0.0",
+      unit: ModelUsageUnit.Tokens,
+      type: "GENERATION",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getGenerationRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + generationId,
+  );
+  expect(getGenerationRes.body).toMatchObject({
+    id: generationId,
+    traceId: traceId,
+    input: { key: "input" },
+    output: { key: "output" },
+    type: "GENERATION",
+  });
+});
+it("Get a single SPAN from /observations/:id", async () => {
+  const traceId = uuidv4();
+  await prisma.trace.create({
+    data: {
+      id: traceId,
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    },
+  });
+
+  const spanId = uuidv4();
+  await prisma.observation.create({
+    data: {
+      id: spanId,
+      traceId: traceId,
+      name: "generation-name",
+      startTime: new Date("2021-01-01T00:00:00.000Z"),
+      endTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelParameters: { key: "value" },
+      input: { key: "input" },
+      output: { key: "output" },
+      version: "2.0.0",
+      type: "SPAN",
+      project: {
+        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+      },
+    },
+  });
+  const getSpanRes = await makeZodVerifiedAPICall(
+    GetObservationV1Response,
+    "GET",
+    "/api/public/observations/" + spanId,
+  );
+  expect(getSpanRes.body).toMatchObject({
+    id: spanId,
+    traceId: traceId,
+    input: { key: "input" },
+    output: { key: "output" },
+    type: "SPAN",
+  });
+});

web/src/__tests__/prompts.v1.servertest.ts

@@ -219,7 +219,7 @@ describe("/api/public/prompts API Endpoint", () => {
     });
 
     // Third prompt is deactivated
-    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+    await makeAPICall("POST", "/api/public/prompts", {
       name: "prompt-name",
       projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
       prompt: "prompt3",
@@ -698,7 +698,7 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(postResponse2.status).toBe(400);
     expect(postResponse2.body).toEqual({
-      error: "ValidationError",
+      error: "InvalidRequestError",
       message:
         "Previous versions have different prompt type. Create a new prompt with a different name.",
     });

web/src/__tests__/prompts.v2.servertest.ts

@@ -533,7 +533,7 @@ describe("/api/public/v2/prompts API Endpoint", () => {
 
       expect(postResponse2.status).toBe(400);
       // @ts-expect-error
-      expect(postResponse2.body.error).toBe("ValidationError");
+      expect(postResponse2.body.error).toBe("InvalidRequestError");
 
       // Check if the prompt is still the chat prompt
       const getResponse1 = await makeAPICall(

web/src/__tests__/score-configs.servertest.ts

@@ -1,24 +1,30 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { v4 as uuidv4 } from "uuid";
-import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
-import { type CastedConfig } from "@langfuse/shared";
-
-const CONFIG_ID_ONE = uuidv4();
-const CONFIG_ID_TWO = uuidv4();
-const CONFIG_ID_THREE = uuidv4();
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  type ScoreConfig,
+  prisma,
+  type ScoreDataType,
+} from "@langfuse/shared/src/db";
+import {
+  GetScoreConfigResponse,
+  PostScoreConfigResponse,
+  GetScoreConfigsResponse,
+} from "@/src/features/public-api/types/score-configs";
 
 const configOne = [
   {
-    id: CONFIG_ID_ONE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Boolean Config",
     description: "Test Description",
-    dataType: ScoreDataType.BOOLEAN,
+    dataType: "BOOLEAN" as ScoreDataType,
     categories: [
-      { label: "False", value: 0 },
       { label: "True", value: 1 },
+      { label: "False", value: 0 },
     ],
     createdAt: new Date("2024-05-10T00:00:00.000Z"),
     updatedAt: new Date("2024-05-10T00:00:00.000Z"),
@@ -26,11 +32,10 @@ const configOne = [
 ];
 const configTwo = [
   {
-    id: CONFIG_ID_TWO,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Numeric Config",
     description: "Test Description",
-    dataType: ScoreDataType.NUMERIC,
+    dataType: "NUMERIC" as ScoreDataType,
     minValue: 0,
     createdAt: new Date("2024-05-11T00:00:00.000Z"),
     updatedAt: new Date("2024-05-11T00:00:00.000Z"),
@@ -39,11 +44,10 @@ const configTwo = [
 
 const configThree = [
   {
-    id: CONFIG_ID_THREE,
     projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
     name: "Test Categorical Config",
     description: "Test Description",
-    dataType: ScoreDataType.CATEGORICAL,
+    dataType: "CATEGORICAL" as ScoreDataType,
     categories: [
       { label: "A", value: 0 },
       { label: "B", value: 1 },
@@ -69,11 +73,18 @@ describe("/api/public/score-configs API Endpoint", () => {
   );
 
   it("should GET a score config", async () => {
-    const configId = CONFIG_ID_ONE;
+    const { id: configId } = (await prisma.scoreConfig.findFirst({
+      where: {
+        projectId: configOne[0].projectId,
+        name: configOne[0].name,
+      },
+    })) as ScoreConfig;
 
-    const getScoreConfig = await makeAPICall<{
-      id: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(200);
     expect(getScoreConfig.body).toMatchObject({
@@ -84,24 +95,12 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
-  it("test invalid config id input", async () => {
-    const configId = "invalid-config-id";
-
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
-
-    expect(getScoreConfig.status).toBe(404);
-    expect(getScoreConfig.body).toMatchObject({
-      message: "Score config not found within authorized project",
-    });
-  });
-
   it("should GET all score configs", async () => {
-    const fetchedConfigs = await makeAPICall<{
-      data: CastedConfig[];
-      meta: object;
-    }>("GET", `/api/public/score-configs?limit=50&page=1`);
+    const fetchedConfigs = await makeZodVerifiedAPICall(
+      GetScoreConfigsResponse,
+      "GET",
+      `/api/public/score-configs?limit=50&page=1`,
+    );
 
     expect(fetchedConfigs.status).toBe(200);
     expect(fetchedConfigs.body.meta).toMatchObject({
@@ -120,6 +119,20 @@ describe("/api/public/score-configs API Endpoint", () => {
     });
   });
 
+  it("test invalid config id input", async () => {
+    const configId = "invalid-config-id";
+
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
+
+    expect(getScoreConfig.status).toBe(404);
+    expect(getScoreConfig.body).toMatchObject({
+      message: "Score config not found within authorized project",
+    });
+  });
+
   it("should return 500 when hitting corrupted score config", async () => {
     const configId = "corrupted-config-id";
 
@@ -131,13 +144,234 @@ describe("/api/public/score-configs API Endpoint", () => {
       },
     });
 
-    const getScoreConfig = await makeAPICall<{
-      message: string;
-    }>("GET", `/api/public/score-configs/${configId}`);
+    const getScoreConfig = await makeAPICall(
+      "GET",
+      `/api/public/score-configs/${configId}`,
+    );
 
     expect(getScoreConfig.status).toBe(500);
     expect(getScoreConfig.body).toMatchObject({
-      message: "Internal Server Error",
+      message: "Requested score config is corrupted",
+    });
+  });
+
+  it("should POST a numeric score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "numeric-config-name",
+        dataType: "NUMERIC",
+        maxValue: 0,
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("numeric-config-name");
+    expect(scoreConfig.body.dataType).toBe("NUMERIC");
+    expect(scoreConfig.body.maxValue).toBe(0);
+  });
+
+  it("should POST a boolean score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "boolean-config-name",
+        dataType: "BOOLEAN",
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("boolean-config-name");
+    expect(scoreConfig.body.dataType).toBe("BOOLEAN");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "True", value: 1 },
+      { label: "False", value: 0 },
+    ]);
+  });
+
+  it("should POST a categorical score config", async () => {
+    const postScoreConfig = await makeZodVerifiedAPICall(
+      PostScoreConfigResponse,
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    const scoreConfig = await makeZodVerifiedAPICall(
+      GetScoreConfigResponse,
+      "GET",
+      `/api/public/score-configs/${postScoreConfig.body.id}`,
+    );
+
+    expect(postScoreConfig.status).toBe(200);
+    expect(scoreConfig.body.name).toBe("categorical-config-name");
+    expect(scoreConfig.body.dataType).toBe("CATEGORICAL");
+    expect(scoreConfig.body.categories).toEqual([
+      { label: "Good", value: 1 },
+      { label: "Bad", value: 0 },
+    ]);
+  });
+
+  it("should fail POST of numeric score config with invalid range", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        PostScoreConfigResponse,
+        "POST",
+        "/api/public/score-configs",
+        {
+          name: "invalid-numeric-config-name",
+          dataType: "NUMERIC",
+          maxValue: 0,
+          minValue: 1,
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"custom\",\"message\":\"Maximum value must be greater than Minimum value\",\"path\":[]}]}`,
+      );
+    }
+  });
+
+  it("should fail POST of boolean score config with custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-boolean-config-name",
+        dataType: "BOOLEAN",
+        categories: [
+          { label: "Good", value: 1 },
+          { label: "Bad", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with NO custom categories", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+    });
+  });
+
+  it("should fail POST of categorical score config with invalid custom categories format", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { key: "first", value: 1 },
+          { key: "second", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Category must be an array of objects with label value pairs, where labels and values are unique.",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category label", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "first", value: 0 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category label: first, category labels must be unique",
+          path: ["categories"],
+        },
+      ],
+    });
+  });
+
+  it("should fail POST of categorical score config with duplicated category value", async () => {
+    const postScoreConfig = await makeAPICall(
+      "POST",
+      "/api/public/score-configs",
+      {
+        name: "invalid-categorical-config-name",
+        dataType: "CATEGORICAL",
+        categories: [
+          { label: "first", value: 1 },
+          { label: "second", value: 1 },
+        ],
+      },
+    );
+
+    expect(postScoreConfig.status).toBe(400);
+    expect(postScoreConfig.body).toMatchObject({
+      message: "Invalid request data",
+      error: [
+        {
+          code: "custom",
+          message:
+            "Duplicate category value: 1, category values must be unique",
+          path: ["categories"],
+        },
+      ],
     });
   });
 });

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -19,11 +19,11 @@ describe("Traces TRPC Router", () => {
           id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           role: "ADMIN",
           name: "test",
+          cloudConfig: { defaultLookBackDays: null },
         },
       ],
       featureFlags: {
         templateFlag: true,
-        evals: true,
       },
       admin: true,
     },
@@ -36,7 +36,6 @@ describe("Traces TRPC Router", () => {
     const trace = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };
@@ -57,11 +56,42 @@ describe("Traces TRPC Router", () => {
     expect(traces).toMatchObject({ traces: [trace] });
   });
 
+  test("traces.all RPC must not return input, output, metadata", async () => {
+    const trace = {
+      name: "trace-name",
+      userId: "user-1",
+      input: { a: 1 },
+      output: { b: 2 },
+      metadata: { c: 3 },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    await prisma.trace.create({
+      data: { ...trace, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const traces = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: null,
+    });
+    expect(traces.traces).toBeDefined();
+    expect(traces.traces).toHaveLength(1);
+
+    const returnedTrace = traces.traces[0];
+    expect(returnedTrace).not.toHaveProperty("input");
+    expect(returnedTrace).not.toHaveProperty("output");
+    expect(returnedTrace).not.toHaveProperty("metadata");
+  });
+
   test("traces.all RPC orders traces by userId", async () => {
     const traceTmpl = {
       name: "trace-name",
       userId: "user-1",
-      metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
     };

web/src/__tests__/sessions.servertest.ts

@@ -1,20 +1,33 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import {
+  GetSessionV1Response,
+  GetSessionsV1Response,
+} from "@/src/features/public-api/types/sessions";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 import { prisma } from "@langfuse/shared/src/db";
 
-describe("/api/public/traces API Endpoint", () => {
+describe("Create and get sessions", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
   it("should create a session via a trace", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      sessionId: "session-id",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        name: "trace-name",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        sessionId: "session-id",
+      },
+    );
 
     const dbSession = await prisma.traceSession.findFirst({
       where: {
@@ -32,16 +45,22 @@ describe("/api/public/traces API Endpoint", () => {
   it("should get session including traces", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      id: "trace-id",
-      input: { hello: "world" },
-      output: "hi",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      sessionId: "session-id",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        name: "trace-name",
+        id: "trace-id",
+        input: { hello: "world" },
+        output: "hi",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        sessionId: "session-id",
+      },
+    );
 
-    const response = await makeAPICall(
+    const response = await makeZodVerifiedAPICall(
+      GetSessionV1Response,
       "GET",
       "/api/public/sessions/session-id",
     );
@@ -60,3 +79,122 @@ describe("/api/public/traces API Endpoint", () => {
     });
   });
 });
+
+describe("GET /api/public/sessions API Endpoint", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+    await prisma.traceSession.createMany({
+      data: [
+        {
+          id: "session-2021-01-01",
+          createdAt: new Date("2021-01-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-02-01",
+          createdAt: new Date("2021-02-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-03-01",
+          createdAt: new Date("2021-03-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-04-01",
+          createdAt: new Date("2021-04-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+        {
+          id: "session-2021-05-01",
+          createdAt: new Date("2021-05-01T00:00:00Z"),
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+      ],
+    });
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("should return all sessions", async () => {
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      "/api/public/sessions",
+    );
+    expect(sessions.body.data).toHaveLength(5);
+  });
+
+  it("should return paginated sessions with limit and page number", async () => {
+    const limit = 2;
+    const page = 2;
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?limit=${limit}&page=${page}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-03-01",
+      "session-2021-02-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(5);
+    expect(sessions.body.meta.totalPages).toBe(3);
+    expect(sessions.body.meta.page).toBe(page);
+    expect(sessions.body.meta.limit).toBe(limit);
+  });
+
+  it("should return sessions within a specific date range", async () => {
+    const fromTimestamp = "2021-02-01T00:00:00Z";
+    const toTimestamp = "2021-04-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?fromTimestamp=${fromTimestamp}&toTimestamp=${toTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-03-01",
+      "session-2021-02-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(2);
+  });
+
+  it("should return sessions from a specific date onwards (including the date)", async () => {
+    const fromTimestamp = "2021-03-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?fromTimestamp=${fromTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(3);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-05-01",
+      "session-2021-04-01",
+      "session-2021-03-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(3);
+  });
+
+  it("should return sessions up to a specific date (excluding it)", async () => {
+    const toTimestamp = "2021-03-01T00:00:00Z";
+
+    const sessions = await makeZodVerifiedAPICall(
+      GetSessionsV1Response,
+      "GET",
+      `/api/public/sessions?toTimestamp=${toTimestamp}`,
+    );
+
+    expect(sessions.body.data).toHaveLength(2);
+    expect(sessions.body.data.map((session) => session.id)).toEqual([
+      "session-2021-02-01",
+      "session-2021-01-01",
+    ]);
+    expect(sessions.body.meta.totalItems).toBe(2);
+  });
+});

web/src/__tests__/spans.servertest.ts

@@ -1,27 +1,39 @@
 /** @jest-environment node */
 
 import { prisma } from "@langfuse/shared/src/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeAPICall,
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
+import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
+import {
+  PatchSpansV1Response,
+  PostSpansV1Response,
+} from "@/src/features/public-api/types/spans";
 
 describe("/api/public/spans API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
   it("should create span after trace", async () => {
-    await pruneDatabase();
-
     const traceId = uuidv4();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -33,16 +45,21 @@ describe("/api/public/spans API Endpoint", () => {
     expect(dbTrace[0]?.id).toBe(traceId);
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceId: traceId,
-      name: "span-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceId: traceId,
+        name: "span-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
     const dbSpan = await prisma.observation.findUnique({
@@ -62,21 +79,24 @@ describe("/api/public/spans API Endpoint", () => {
   });
 
   it("should create span before trace", async () => {
-    await pruneDatabase();
-
     const traceId = uuidv4();
     const spanId = uuidv4();
 
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceId: traceId,
-      name: "span-name",
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceId: traceId,
+        name: "span-name",
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
     const dbSpan = await prisma.observation.findUnique({
@@ -94,15 +114,20 @@ describe("/api/public/spans API Endpoint", () => {
     expect(dbSpan?.metadata).toEqual({ meta: "value" });
     expect(dbSpan?.version).toBe("2.0.0");
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -118,21 +143,31 @@ describe("/api/public/spans API Endpoint", () => {
     const traceId = uuidv4();
     const spanId = uuidv4();
 
-    const response = await makeAPICall("POST", "/api/public/traces", {
-      externalId: uuidv4(),
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    });
+    const response = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        externalId: uuidv4(),
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      },
+    );
     expect(response.status).toBe(200);
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: "spanName",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      traceId: traceId,
-      externalTraceIdType: "EXTERNAL",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: "spanName",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        traceId: traceId,
+        externalTraceIdType: "EXTERNAL",
+      },
+    );
     expect(createSpan.status).toBe(200);
 
     const dbSpan = await prisma.observation.findFirstOrThrow({
@@ -148,15 +183,20 @@ describe("/api/public/spans API Endpoint", () => {
     const spanName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -189,17 +229,22 @@ describe("/api/public/spans API Endpoint", () => {
 
     const spanId = uuidv4();
     const traceId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      traceIdType: "EXTERNAL",
-      traceId: traceId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        traceIdType: "EXTERNAL",
+        traceId: traceId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
 
@@ -221,15 +266,20 @@ describe("/api/public/spans API Endpoint", () => {
     const generationName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: generationName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { key: "value" },
-      metadata: { key: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: generationName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { key: "value" },
+        metadata: { key: "value" },
+        version: "2.0.0",
+      },
+    );
 
     const dbSpan = await prisma.observation.findFirstOrThrow({
       where: {
@@ -262,22 +312,32 @@ describe("/api/public/spans API Endpoint", () => {
     const spanName = uuidv4();
 
     const spanId = uuidv4();
-    const createSpan = await makeAPICall("POST", "/api/public/spans", {
-      id: spanId,
-      name: spanName,
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:00:00.000Z",
-      input: { input: "value" },
-      metadata: { meta: "value" },
-      version: "2.0.0",
-    });
+    const createSpan = await makeZodVerifiedAPICall(
+      PostSpansV1Response,
+      "POST",
+      "/api/public/spans",
+      {
+        id: spanId,
+        name: spanName,
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:00:00.000Z",
+        input: { input: "value" },
+        metadata: { meta: "value" },
+        version: "2.0.0",
+      },
+    );
 
     expect(createSpan.status).toBe(200);
 
-    const updatedSpan = await makeAPICall("PATCH", "/api/public/spans", {
-      spanId: spanId,
-      output: { key: "this is a great gpt output" },
-    });
+    const updatedSpan = await makeZodVerifiedAPICall(
+      PatchSpansV1Response,
+      "PATCH",
+      "/api/public/spans",
+      {
+        spanId: spanId,
+        output: { key: "this is a great gpt output" },
+      },
+    );
     expect(updatedSpan.status).toBe(200);
 
     const dbSpan = await prisma.observation.findUnique({

web/src/__tests__/test-utils.ts

@@ -3,6 +3,7 @@ import { hash } from "bcryptjs";
 import { env } from "@/src/env.mjs";
 import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
+import { type z } from "zod";
 
 export const pruneDatabase = async () => {
   if (!env.DATABASE_URL.includes("localhost:5432")) {
@@ -13,6 +14,7 @@ export const pruneDatabase = async () => {
   await prisma.scoreConfig.deleteMany();
   await prisma.observation.deleteMany();
   await prisma.trace.deleteMany();
+  await prisma.traceSession.deleteMany();
   await prisma.datasetItem.deleteMany();
   await prisma.dataset.deleteMany();
   await prisma.datasetRuns.deleteMany();
@@ -73,6 +75,29 @@ export async function makeAPICall<T = IngestionAPIResponse>(
   return { body: responseBody, status: response.status };
 }
 
+export async function makeZodVerifiedAPICall<T extends z.ZodTypeAny>(
+  responseZodSchema: T,
+  method: "POST" | "GET" | "PUT" | "DELETE" | "PATCH",
+  url: string,
+  body?: unknown,
+  auth?: string,
+): Promise<{ body: z.infer<T>; status: number }> {
+  const { body: resBody, status } = await makeAPICall(method, url, body, auth);
+  if (status !== 200) {
+    throw new Error(
+      `API call did not return 200, returned status ${status}, body ${JSON.stringify(resBody)}`,
+    );
+  }
+  const typeCheckResult = responseZodSchema.safeParse(resBody);
+  if (!typeCheckResult.success) {
+    console.error(typeCheckResult.error);
+    throw new Error(
+      `API call (${method} ${url}) did not return valid response, returned status ${status}, body ${JSON.stringify(resBody)}, error ${typeCheckResult.error}`,
+    );
+  }
+  return { body: resBody, status };
+}
+
 export const setupUserAndProject = async () => {
   const user = await prisma.user.create({
     data: {

web/src/__tests__/tokenCost.servertest.ts

@@ -126,32 +126,119 @@ describe("Token Cost Calculation", () => {
 
   it("should correctly calculate token costs when only some user provided costs are given", async () => {
     const model = {
-      inputPrice: new Decimal(0.01),
-      outputPrice: new Decimal(0.02),
-      totalPrice: new Decimal(0.03),
+      inputPrice: new Decimal(1),
+      outputPrice: new Decimal(1),
     };
 
-    const tokenCounts = {
-      input: 100,
-      output: 200,
-      total: undefined,
-    };
+    const data = [
+      // missing total
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: 2,
+          total: undefined,
+        },
+        expectedCost: {
+          input: 1,
+          output: 2,
+          total: 3,
+        },
+      },
+      // only total
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: undefined,
+          total: 2,
+        },
+        expectedCost: {
+          input: undefined,
+          output: undefined,
+          total: 2,
+        },
+      },
+      // missing input
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+        expectedCost: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+      },
+      // only input
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: undefined,
+          total: undefined,
+        },
+        expectedCost: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+      },
 
-    const userProvidedCosts = {
-      inputCost: null,
-      outputCost: new Decimal(3.0),
-      totalCost: null,
-    };
+      // missing output
+      {
+        userProvidedCosts: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+        expectedCost: {
+          input: 1,
+          output: undefined,
+          total: 1,
+        },
+      },
 
-    const costs = ObservationProcessor.calculateTokenCosts(
-      model as any,
-      userProvidedCosts,
-      tokenCounts,
-    );
+      // only output
+      {
+        userProvidedCosts: {
+          input: undefined,
+          output: 2,
+          total: undefined,
+        },
+        expectedCost: {
+          input: undefined,
+          output: 2,
+          total: 2,
+        },
+      },
+    ];
 
-    expect(costs.inputCost?.toNumber()).toBe(1.0); // Calculated based on model price
-    expect(costs.outputCost?.toNumber()).toBe(3.0); // Overridden by user provided cost
-    expect(costs.totalCost?.toNumber()).toBe(4.0); // Sum of input and output costs
+    for (const { userProvidedCosts, expectedCost } of data) {
+      const tokenCounts = {
+        input: 0,
+        output: 0,
+        total: 0,
+      };
+
+      const userProvidedCostsDecimal = {
+        inputCost:
+          userProvidedCosts.input && new Decimal(userProvidedCosts.input),
+        outputCost:
+          userProvidedCosts.output && new Decimal(userProvidedCosts.output),
+        totalCost:
+          userProvidedCosts.total && new Decimal(userProvidedCosts.total),
+      };
+
+      const costs = ObservationProcessor.calculateTokenCosts(
+        model as any,
+        userProvidedCostsDecimal as any,
+        tokenCounts,
+      );
+
+      expect(costs.inputCost?.toNumber()).toBe(expectedCost.input);
+      expect(costs.outputCost?.toNumber()).toBe(expectedCost.output);
+      expect(costs.totalCost?.toNumber()).toBe(expectedCost.total);
+    }
   });
 
   it("should return empty costs if no model is provided", async () => {
@@ -842,6 +929,83 @@ describe("Token Cost Calculation", () => {
     expect(generation?.totalTokens).toBe(generationUsage2.usage.total);
   });
 
+  it("should overwrite costs if new costs are user provided and only partial", async () => {
+    const generationUsage1 = {
+      model: modelName,
+      usage: {
+        input: 1,
+        output: 2,
+        total: 3,
+        unit: ModelUsageUnit.Tokens,
+      },
+    };
+
+    const generationUsage2 = {
+      model: modelName,
+      usage: {
+        outputCost: 1,
+        unit: ModelUsageUnit.Tokens,
+      },
+    };
+
+    const events = [
+      {
+        id: uuidv4(),
+        type: "generation-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: generationId,
+          ...generationUsage1,
+        },
+      },
+      {
+        id: uuidv4(),
+        type: "generation-update",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: generationId,
+          ...generationUsage2,
+        },
+      },
+    ];
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: events,
+    });
+
+    expect(response.status).toBe(207);
+    const generation = await prisma.observation.findFirst({
+      where: {
+        id: generationId,
+      },
+    });
+
+    expect(generation).toBeDefined();
+    expect(generation?.type).toBe("GENERATION");
+
+    // Model name should be matched
+    expect(generation?.internalModel).toBe(tokenModelData.modelName);
+    expect(generation?.unit).toEqual(tokenModelData.unit);
+    expect(generation?.internalModelId).toBe(tokenModelData.id);
+
+    // User provided cost
+    expect(generation?.inputCost?.toNumber()).toBe(undefined);
+    expect(generation?.outputCost?.toNumber()).toBe(
+      generationUsage2.usage.outputCost,
+    );
+    expect(generation?.totalCost?.toNumber()).toBe(undefined);
+
+    // Calculated cost
+    expect(generation?.calculatedInputCost?.toNumber()).toBe(undefined);
+    expect(generation?.calculatedOutputCost?.toNumber()).toBe(
+      generationUsage2.usage.outputCost,
+    );
+    expect(generation?.calculatedTotalCost?.toNumber()).toBe(1);
+    expect(generation?.promptTokens).toBe(generationUsage1.usage.input);
+    expect(generation?.completionTokens).toBe(generationUsage1.usage.output);
+    expect(generation?.totalTokens).toBe(generationUsage1.usage.total);
+  });
+
   it("should not overwrite costs if previous cost were user provided", async () => {
     const generationUsage1 = {
       model: modelName,

web/src/__tests__/traces.servertest.ts

@@ -1,58 +1,70 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  makeZodVerifiedAPICall,
+  pruneDatabase,
+} from "@/src/__tests__/test-utils";
+import { PostGenerationsV1Response } from "@/src/features/public-api/types/generations";
+import { PostScoresResponse } from "@/src/features/public-api/types/scores";
+import {
+  GetTraceV1Response,
+  GetTracesV1Response,
+  PostTracesV1Response,
+} from "@/src/features/public-api/types/traces";
 import { prisma } from "@langfuse/shared/src/db";
 import { v4 as uuidv4 } from "uuid";
 
-interface GetTracesAPIResponse {
-  data: Array<{
-    id: string;
-    [key: string]: unknown;
-  }>;
-}
-
 describe("/api/public/traces API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
-  it("should create", async () => {
+  it("should create and get a trace via /traces", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
+    const traceCreate = await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
         name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
       },
-    });
+       );
 
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.name).toBe("trace-name");
-    expect(dbTrace[0]?.release).toBe("1.0.0");
-    expect(dbTrace[0]?.externalId).toBeNull();
-    expect(dbTrace[0]?.version).toBe("2.0.0");
-    expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    const trace = await makeZodVerifiedAPICall(
+      GetTraceV1Response,
+      "GET",
+      "/api/public/traces/" + traceCreate.body.id,
+    );
+
+    expect(trace.body.name).toBe("trace-name");
+    expect(trace.body.release).toBe("1.0.0");
+    expect(trace.body.externalId).toBeNull();
+    expect(trace.body.version).toBe("2.0.0");
+    expect(trace.body.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
   });
 
   it("should upsert second trace", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-id",
-      name: "trace-name",
-      userId: "user-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-      public: true,
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-id",
+        name: "trace-name",
+        userId: "user-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+        public: true,
+      },
+       );
 
     const dbTrace1 = await prisma.trace.findFirst({
       where: {
@@ -70,14 +82,19 @@ describe("/api/public/traces API Endpoint", () => {
       userId: "user-1",
     });
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-id",
-      metadata: { key: "value" },
-      timestamp: "2021-01-01T00:00:00.000Z",
-      release: "1.0.0",
-      version: "5.0.0",
-      public: false,
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-id",
+        metadata: { key: "value" },
+        timestamp: "2021-01-01T00:00:00.000Z",
+        release: "1.0.0",
+        version: "5.0.0",
+        public: false,
+      },
+       );
 
     const dbTrace2 = await prisma.trace.findFirst({
       where: {
@@ -100,23 +117,39 @@ describe("/api/public/traces API Endpoint", () => {
   it("should use tags correctly on POST and GET", async () => {
     await pruneDatabase();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-1",
-      tags: ["tag-1", "tag-2", "tag-3"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-1",
+        tags: ["tag-1", "tag-2", "tag-3"],
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-2",
-      tags: ["tag-1"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-2",
+        tags: ["tag-1"],
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-3",
-      tags: ["tag-2", "tag-3"],
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-3",
+        tags: ["tag-2", "tag-3"],
+      },
+       );
 
     // multiple tags
-    const traces = await makeAPICall<GetTracesAPIResponse>(
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-2&tags=tag-3",
     );
@@ -125,7 +158,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds).toEqual(["trace-3", "trace-1"]);
 
     // single tag
-    const traces2 = await makeAPICall<GetTracesAPIResponse>(
+    const traces2 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-1",
     );
@@ -134,7 +168,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds2).toEqual(["trace-2", "trace-1"]);
 
     // wrong tag
-    const traces3 = await makeAPICall<GetTracesAPIResponse>(
+    const traces3 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=tag-10",
     );
@@ -143,7 +178,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds3).toEqual([]);
 
     // no tag
-    const traces4 = await makeAPICall<GetTracesAPIResponse>(
+    const traces4 = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       "/api/public/traces?tags=",
     );
@@ -157,60 +193,94 @@ describe("/api/public/traces API Endpoint", () => {
 
     // Create a trace with some observations that have costs and latencies
     const traceId = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-with-costs",
-      userId: "user-costs",
-      projectId: "project-costs",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: traceId,
+        name: "trace-with-costs",
+        userId: "user-costs",
+        projectId: "project-costs",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+       );
     console.log(traceId);
 
     // Simulate observations with costs and latencies
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId,
-      usage: { totalCost: 10.5 },
-      startTime: "2021-01-01T00:00:00.000Z",
-      endTime: "2021-01-01T00:10:00.000Z",
-    });
-    await makeAPICall("POST", "/api/public/generations", {
-      traceId: traceId,
-      usage: { totalCost: 5.25 },
-      startTime: "2021-01-01T00:10:00.000Z",
-      endTime: "2021-01-01T00:20:00.000Z",
-    });
+    const generationId = uuidv4();
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId,
+        id: generationId,
+        name: "Generation1",
+        usage: { totalCost: 10.5 },
+        startTime: "2021-01-01T00:00:00.000Z",
+        endTime: "2021-01-01T00:10:00.000Z",
+      },
+       );
+    await makeZodVerifiedAPICall(
+      PostGenerationsV1Response,
+      "POST",
+      "/api/public/generations",
+      {
+        traceId: traceId,
+        usage: { totalCost: 5.25 },
+        startTime: "2021-01-01T00:10:00.000Z",
+        endTime: "2021-01-01T00:20:00.000Z",
+      },
+       );
 
     // Simulate scores on the trace
     const scoreId1 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId1,
-      name: "score-1",
-      value: 75.0,
-      traceId: traceId,
-      comment: "First score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId1,
+        name: "score-1",
+        value: 75.0,
+        traceId: traceId,
+        comment: "First score",
+      },
+       );
     const scoreId2 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId2,
-      name: "score-2",
-      value: 85.5,
-      traceId: traceId,
-      comment: "Second score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId2,
+        name: "score-2",
+        value: 85.5,
+        traceId: traceId,
+        comment: "Second score",
+      },
+       );
     const scoreId3 = uuidv4();
-    await makeAPICall("POST", "/api/public/scores", {
-      id: scoreId3,
-      name: "score-3",
-      value: 95.0,
-      traceId: traceId,
-      comment: "Third score",
-    });
+    await makeZodVerifiedAPICall(
+      PostScoresResponse,
+      "POST",
+      "/api/public/scores",
+      {
+        id: scoreId3,
+        name: "score-3",
+        value: 95.0,
+        traceId: traceId,
+        comment: "Third score",
+      },
+       );
 
     // GET traces
     // Retrieve the trace with totalCost and latency
-    const traces = await makeAPICall<GetTracesAPIResponse>(
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces`,
     );
@@ -226,17 +296,23 @@ describe("/api/public/traces API Endpoint", () => {
 
     // GET trace
     // Retrieve the trace with total
-    const trace = await makeAPICall<{
-      id: string;
-      totalCost: number;
-      htmlPath: string;
-    }>("GET", `/api/public/traces/${traceId}`);
+    const trace = await makeZodVerifiedAPICall(
+      GetTraceV1Response,
+      "GET",
+      `/api/public/traces/${traceId}`,
+    );
     console.log(trace.body);
     expect(trace.body.totalCost).toBeCloseTo(15.75);
     expect(trace.body.id).toBe(traceId);
     expect(trace.body.id).toBe(traceId);
     expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
     expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
+    expect(trace.body.scores).toHaveLength(3);
+    expect(trace.body.scores[0].id).toBe(scoreId3);
+    expect(trace.body.scores[0].name).toBe("score-3");
+    expect(trace.body.observations).toHaveLength(2);
+    expect(trace.body.observations[0].id).toBe(generationId);
+    expect(trace.body.observations[0].name).toBe("Generation1");
   });
 
   it("should filter traces by session ID", async () => {
@@ -244,30 +320,41 @@ describe("/api/public/traces API Endpoint", () => {
     const anotherSessionId = "another-session-id";
 
     // Create traces with different session IDs
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-1",
-      name: "test-trace-1",
-      sessionId,
-      userId: "user-1",
-      projectId: "project-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "1.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-1",
+        name: "test-trace-1",
+        sessionId,
+        userId: "user-1",
+        projectId: "project-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "1.0.0",
+      },
+       );
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: "trace-2",
-      name: "test-trace-2",
-      sessionId: anotherSessionId,
-      userId: "user-2",
-      projectId: "project-1",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "1.0.0",
-    });
+    await makeZodVerifiedAPICall(
+      PostTracesV1Response,
+      "POST",
+      "/api/public/traces",
+      {
+        id: "trace-2",
+        name: "test-trace-2",
+        sessionId: anotherSessionId,
+        userId: "user-2",
+        projectId: "project-1",
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "1.0.0",
+      },
+       );
 
     // Filter by session ID
-    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+    const tracesBySessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces?sessionId=${sessionId}`,
     );
@@ -277,7 +364,8 @@ describe("/api/public/traces API Endpoint", () => {
     expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
 
     // Filter by another session ID
-    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+    const tracesByAnotherSessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
       "GET",
       `/api/public/traces?sessionId=${anotherSessionId}`,
     );
@@ -287,11 +375,11 @@ describe("/api/public/traces API Endpoint", () => {
     expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
 
     // Filter by non-existent session ID
-    const tracesByNonExistentSessionId =
-      await makeAPICall<GetTracesAPIResponse>(
-        "GET",
-        `/api/public/traces?sessionId=non-existent-session-id`,
-      );
+    const tracesByNonExistentSessionId = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      `/api/public/traces?sessionId=non-existent-session-id`,
+    );
 
     expect(tracesByNonExistentSessionId.status).toBe(200);
     expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);

web/src/components/BatchExportTableButton.tsx

@@ -15,6 +15,7 @@ import {
 import React from "react";
 import { api } from "@/src/utils/api";
 import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 
 export type BatchExportTableButtonProps = {
@@ -31,6 +32,10 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
   const [isExporting, setIsExporting] = React.useState(false);
   const createExport = api.batchExport.create.useMutation();
   const isEeEnabled = useIsEeEnabled();
+  const hasAccess = useHasAccess({
+    projectId: props.projectId,
+    scope: "batchExport:create",
+  });
 
   const handleExport = async (format: BatchExportFileFormat) => {
     setIsExporting(true);
@@ -51,7 +56,7 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
     });
   };
 
-  if (!isEeEnabled) return null;
+  if (!isEeEnabled || !hasAccess) return null;
 
   return (
     <DropdownMenu>

web/src/components/date-picker.tsx

@@ -29,6 +29,7 @@ import { useMediaQuery } from "react-responsive";
 import { type DashboardDateRange } from "@/src/pages/project/[projectId]";
 import { isValidOption } from "@/src/utils/types";
 import { setBeginningOfDay, setEndOfDay } from "@/src/utils/dates";
+import { TimePicker } from "@/src/components/ui/time-picker";
 
 export const DEFAULT_DATE_RANGE_SELECTION = "Date range" as const;
 export type AvailableDateRangeSelections =
@@ -41,12 +42,14 @@ export function DatePicker({
   clearable = false,
   className,
   disabled,
+  includeTimePicker,
 }: {
   date?: Date | undefined;
   onChange: (date: Date | undefined) => void;
   clearable?: boolean;
   className?: string;
   disabled?: boolean;
+  includeTimePicker?: boolean;
 }) {
   return (
     <div className="flex flex-row gap-2 align-middle">
@@ -62,7 +65,11 @@ export function DatePicker({
             )}
           >
             <CalendarIcon className="mr-2 h-4 w-4" />
-            {date ? format(date, "PPP") : <span>Pick a date</span>}
+            {date ? (
+              format(date, includeTimePicker ? "PPP pp" : "PPP")
+            ) : (
+              <span>Pick a date</span>
+            )}
           </Button>
         </PopoverTrigger>
         <PopoverContent className="w-auto p-0">
@@ -72,6 +79,9 @@ export function DatePicker({
             onSelect={(d) => onChange(d)}
             initialFocus
           />
+          {includeTimePicker && (
+            <TimePicker date={date} setDate={(d) => onChange(d)} />
+          )}
         </PopoverContent>
       </Popover>
       {date && clearable && (

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
         <span key={i} className="group/score ml-1 first:ml-0">
           {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
             ? s.stringValue
-            : s.value.toFixed(2)}
+            : s.value?.toFixed(2)}
           {s.comment && (
             <HoverCard>
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">

web/src/components/schemas/ChatMlSchema.ts

@@ -0,0 +1,25 @@
+import { z } from "zod";
+
+export const ChatMlMessageSchema = z
+  .object({
+    role: z.string().optional(),
+    name: z.string().optional(),
+    content: z
+      .union([z.record(z.any()), z.string(), z.array(z.any())])
+      .nullish(),
+    additional_kwargs: z.record(z.any()).optional(),
+  })
+  .passthrough()
+  .refine((value) => value.content !== null || value.role !== undefined)
+  .transform(({ additional_kwargs, ...other }) => ({
+    ...other,
+    ...additional_kwargs,
+  }))
+  .transform(({ role, name, content, ...other }) => ({
+    role,
+    name,
+    content,
+    json: Object.keys(other).length === 0 ? undefined : other,
+  }));
+
+export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);

web/src/components/schemas/MarkdownSchema.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+const MARKDOWN_PATTERNS = [
+  "(\\*\\*?|__?)(.*?)\\1", // Matches bold (** or __) and italic (* or _) with proper escaping
+  "`{3}[\\s\\S]*?`{3}", // Matches fenced code blocks with triple backticks
+  "`[\\s\\S]*?`", // Matches inline code with single backticks
+  "(^|\\s)[-+*]\\s", // Matches unordered lists that start with -, +, or *
+  "^\\s*#{1,6}\\s", // Matches headers that start with # to ######
+  "^>\\s+", // Matches blockquotes starting with >
+  "^\\d+\\.\\s", // Matches ordered lists starting with 1. or 2. etc
+  "!\\[.*?\\]\\(.*?\\)", // Matches images ![Alt text](URL)
+  "\\[.*?\\]\\(.*?\\)", // Matches links [Link text](URL)
+].join("|");
+
+const MARKDOWN_REGEX = new RegExp(MARKDOWN_PATTERNS, "gm");
+
+export function containsAnyMarkdown(...texts: string[]): boolean {
+  MARKDOWN_REGEX.lastIndex = 0;
+  return texts.some((text) => MARKDOWN_REGEX.test(text));
+}
+
+export const MarkdownSchema = z.string().refine(containsAnyMarkdown);

web/src/components/session/index.tsx

@@ -132,6 +132,7 @@ export const SessionPage: React.FC<{
                 variant="badge"
                 type="session"
                 source="SessionDetail"
+                key={"annotation-drawer" + trace.id}
               />
             </div>
           </Card>

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -38,7 +38,7 @@ export function TraceTableMultiSelectAction({
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
     onSuccess: () => {
       onDeleteSuccess();
-      void utils.traces.invalidate();
+      void utils.traces.all.invalidate();
     },
   });
 

web/src/components/table/use-cases/generations.tsx

@@ -21,18 +21,20 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import {
-  type Prisma,
   type ObservationLevel,
   type FilterState,
   type ObservationOptions,
 } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type BatchExportFileFormat,
@@ -44,7 +46,7 @@ import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAl
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 
 export type GenerationsTableRow = {
   id: string;
@@ -58,14 +60,14 @@ export type GenerationsTableRow = {
   timeToFirstToken?: number;
   name?: string;
   model?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
+  metadata?: unknown;
   inputCost?: Decimal;
   outputCost?: Decimal;
   totalCost?: Decimal;
   traceName?: string;
-  metadata?: Prisma.JsonValue;
   scores?: ScoreSimplified[];
   usage: {
     promptTokens: number;
@@ -113,7 +115,7 @@ export default function GenerationsTable({
         column: "Start Time",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "generations",
@@ -454,7 +456,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
     },
     {
@@ -470,7 +472,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
     },
     {
@@ -486,7 +488,7 @@ export default function GenerationsTable({
           completionTokens: number;
           totalTokens: number;
         } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
     },
     {
@@ -519,10 +521,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -538,10 +540,10 @@ export default function GenerationsTable({
         const observationId: string = row.getValue("id");
         const traceId: string = row.getValue("traceId");
         return (
-          <GenerationsIOCell
+          <GenerationsDynamicCell
             observationId={observationId}
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -553,12 +555,16 @@ export default function GenerationsTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values = row.getValue(
-          "metadata",
-        ) as GenerationsTableRow["metadata"];
-        return !!values ? (
-          <IOTableCell data={values} singleLine={rowHeight === "s"} />
-        ) : null;
+        const observationId: string = row.getValue("id");
+        const traceId: string = row.getValue("traceId");
+        return (
+          <GenerationsDynamicCell
+            observationId={observationId}
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
       defaultHidden: true,
@@ -617,7 +623,6 @@ export default function GenerationsTable({
           model: generation.model ?? "",
           scores: generation.scores,
           level: generation.level,
-          metadata: generation.metadata,
           statusMessage: generation.statusMessage ?? undefined,
           usage: {
             promptTokens: generation.promptTokens,
@@ -712,15 +717,15 @@ export default function GenerationsTable({
   );
 }
 
-const GenerationsIOCell = ({
+const GenerationsDynamicCell = ({
   traceId,
   observationId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
   observationId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine: boolean;
 }) => {
   const observation = api.observations.byId.useQuery(
@@ -742,9 +747,13 @@ const GenerationsIOCell = ({
     <IOTableCell
       isLoading={observation.isLoading}
       data={
-        io === "output" ? observation.data?.output : observation.data?.input
+        col === "output"
+          ? observation.data?.output
+          : col === "input"
+            ? observation.data?.input
+            : observation.data?.metadata
       }
-      className={cn(io === "output" && "bg-accent-light-green")}
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/table/use-cases/models.tsx

@@ -43,7 +43,7 @@ const modelConfigDescriptions = {
   outputPrice: "Price per 1000 units of output",
   totalPrice:
     "Price per 1000 units, for models that don't have input/output specific prices",
-  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, or IMAGES.",
+  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, REQUESTS or IMAGES.",
   tokenizerId:
     "Tokenizer used for this model to calculate token counts if none are ingested. Pick from list of supported tokenizers.",
   config:

web/src/components/table/use-cases/score-configs.tsx

@@ -6,11 +6,7 @@ import { api } from "@/src/utils/api";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import { DataTable } from "@/src/components/table/data-table";
-import {
-  type ScoreDataType,
-  type Prisma,
-  type ConfigCategory,
-} from "@langfuse/shared";
+import { type ScoreDataType, type Prisma } from "@langfuse/shared";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 import {
@@ -28,6 +24,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { type ConfigCategory } from "@/src/features/public-api/types/score-configs";
 
 type ScoreConfigTableRow = {
   id: string;
@@ -38,7 +35,7 @@ type ScoreConfigTableRow = {
   range: {
     maxValue?: number | null;
     minValue?: number | null;
-    categories?: Prisma.JsonValue | null;
+    categories?: ConfigCategory[] | null;
   };
   description?: string | null;
   isArchived: boolean;
@@ -46,7 +43,7 @@ type ScoreConfigTableRow = {
 
 function getConfigRange(
   originalRow: ScoreConfigTableRow,
-): Prisma.JsonValue | undefined {
+): undefined | Prisma.JsonValue {
   const { range, dataType } = originalRow;
 
   if (isNumericDataType(dataType)) {
@@ -57,7 +54,7 @@ function getConfigRange(
   }
 
   if (isCategoricalDataType(dataType) || isBooleanDataType(dataType)) {
-    const configCategories = (range.categories as ConfigCategory[]) ?? [];
+    const configCategories = range.categories ?? [];
 
     return configCategories.reduce(
       (acc, category) => {

web/src/components/table/use-cases/scores.tsx

@@ -9,13 +9,14 @@ import useColumnVisibility from "@/src/features/column-visibility/hooks/useColum
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { isNumericDataType } from "@/src/features/manual-scoring/lib/helpers";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 import {
   type ScoreOptions,
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { utcDateOffsetByDays } from "@/src/utils/dates";
+import { localtimeDateOffsetByDays } from "@/src/utils/dates";
+import { isPresent } from "@/src/utils/typeChecks";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import type { FilterState, ScoreDataType } from "@langfuse/shared";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
@@ -90,7 +91,7 @@ export default function ScoresTable({
         column: "Timestamp",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "scores",
@@ -116,9 +117,18 @@ export default function ScoresTable({
   });
   const totalCount = scores.data?.totalCount ?? 0;
 
-  const filterOptions = api.scores.filterOptions.useQuery({
-    projectId,
-  });
+  const filterOptions = api.scores.filterOptions.useQuery(
+    {
+      projectId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
 
   const rawColumns: LangfuseColumnDef<ScoresTableRow>[] = [
     {
@@ -310,11 +320,12 @@ export default function ScoresTable({
       source: score.source,
       name: score.name,
       dataType: score.dataType,
-      value: isNumericDataType(score.dataType)
-        ? score.value % 1 === 0
-          ? String(score.value)
-          : score.value.toFixed(4)
-        : score.stringValue ?? "",
+      value:
+        isNumericDataType(score.dataType) && isPresent(score.value)
+          ? score.value % 1 === 0
+            ? String(score.value)
+            : score.value.toFixed(4)
+          : score.stringValue ?? "",
       author: {
         image: score.authorUserImage ?? undefined,
         name: score.authorUserName ?? undefined,

web/src/components/table/use-cases/sessions.tsx

@@ -14,13 +14,16 @@ import {
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
-import { usdFormatter } from "@/src/utils/numbers";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
 import type Decimal from "decimal.js";
 import { useEffect } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
 import { BatchExportTableButton } from "@/src/components/BatchExportTableButton";
 
 export type SessionTableRow = {
@@ -57,7 +60,7 @@ export default function SessionsTable({
         column: "Created At",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "sessions",
@@ -276,7 +279,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value: number | undefined = row.getValue("inputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -289,7 +294,9 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value = row.getValue("outputTokens");
 
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {
@@ -301,7 +308,9 @@ export default function SessionsTable({
       enableSorting: true,
       cell: ({ row }) => {
         const value = row.getValue("totalTokens");
-        return value ? <span>{Number(value)}</span> : undefined;
+        return value ? (
+          <span>{numberFormatter(Number(value), 0)}</span>
+        ) : undefined;
       },
     },
     {

web/src/components/table/use-cases/traces.tsx

@@ -11,7 +11,10 @@ import { Checkbox } from "@/src/components/ui/checkbox";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
+import {
+  formatIntervalSeconds,
+  localtimeDateOffsetByDays,
+} from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
@@ -23,7 +26,7 @@ import {
   withDefault,
 } from "use-query-params";
 import type Decimal from "decimal.js";
-import { usdFormatter } from "@/src/utils/numbers";
+import { numberFormatter, usdFormatter } from "@/src/utils/numbers";
 import { DeleteButton } from "@/src/components/deleteButton";
 import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
@@ -34,11 +37,11 @@ import {
   type TraceOptions,
   tracesTableColsWithOptions,
   type ObservationLevel,
-  type Score,
 } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
-import { useLookBackDays } from "@/src/hooks/useLookBackDays";
+import { useTableLookBackDays } from "@/src/hooks/useTableLookBackDays";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -46,22 +49,22 @@ export type TracesTableRow = {
   timestamp: string;
   name: string;
   userId: string;
-  metadata?: string;
   level: ObservationLevel;
   observationCount: number;
   latency?: number;
   release?: string;
   version?: string;
   sessionId?: string;
-  // i/o not set explicitly, but fetched from the server from the cell
+  // i/o and metadata not set explicitly, but fetched from the server from the cell
   input?: unknown;
   output?: unknown;
-  scores: Score[];
+  metadata?: unknown;
+  scores: APIScore[];
   tags: string[];
   usage: {
-    promptTokens: number;
-    completionTokens: number;
-    totalTokens: number;
+    promptTokens: bigint;
+    completionTokens: bigint;
+    totalTokens: bigint;
   };
   inputCost?: Decimal;
   outputCost?: Decimal;
@@ -95,7 +98,7 @@ export default function TracesTable({
         column: "Timestamp",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-useLookBackDays(projectId)),
+        value: localtimeDateOffsetByDays(-useTableLookBackDays(projectId)),
       },
     ],
     "traces",
@@ -128,7 +131,6 @@ export default function TracesTable({
     filter: filterState,
     searchQuery,
     orderBy: orderByState,
-    returnIO: false,
   };
   const traces = api.traces.all.useQuery(tracesAllQueryFilter);
 
@@ -180,7 +182,6 @@ export default function TracesTable({
       name: trace.name ?? "",
       level: trace.level,
       observationCount: trace.observationCount,
-      metadata: JSON.stringify(trace.metadata),
       release: trace.release ?? undefined,
       version: trace.version ?? undefined,
       userId: trace.userId ?? "",
@@ -238,7 +239,8 @@ export default function TracesTable({
       header: undefined,
       id: "bookmarked",
       cell: ({ row }) => {
-        const bookmarked = row.getValue("bookmarked");
+        const bookmarked: TracesTableRow["bookmarked"] =
+          row.getValue("bookmarked");
         const traceId = row.getValue("id");
         return typeof traceId === "string" &&
           typeof bookmarked === "boolean" ? (
@@ -258,7 +260,7 @@ export default function TracesTable({
       header: "ID",
       id: "id",
       cell: ({ row }) => {
-        const value = row.getValue("id");
+        const value: TracesTableRow["id"] = row.getValue("id");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/traces/${value}`}
@@ -287,7 +289,7 @@ export default function TracesTable({
       header: "User ID",
       id: "userId",
       cell: ({ row }) => {
-        const value = row.getValue("userId");
+        const value: TracesTableRow["userId"] = row.getValue("userId");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/users/${encodeURIComponent(value)}`}
@@ -305,7 +307,7 @@ export default function TracesTable({
       id: "sessionId",
       header: "Session ID",
       cell: ({ row }) => {
-        const value = row.getValue("sessionId");
+        const value: TracesTableRow["sessionId"] = row.getValue("sessionId");
         return value && typeof value === "string" ? (
           <TableLink
             path={`/project/${projectId}/sessions/${encodeURIComponent(value)}`}
@@ -323,7 +325,7 @@ export default function TracesTable({
       header: "Latency",
       // add seconds to the end of the latency
       cell: ({ row }) => {
-        const value: number | undefined = row.getValue("latency");
+        const value: TracesTableRow["latency"] = row.getValue("latency");
         return value !== undefined ? formatIntervalSeconds(value) : undefined;
       },
       enableHiding: true,
@@ -334,12 +336,8 @@ export default function TracesTable({
       id: "inputTokens",
       header: "Input Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.promptTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.promptTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -350,12 +348,8 @@ export default function TracesTable({
       id: "outputTokens",
       header: "Output Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.completionTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.completionTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -366,12 +360,8 @@ export default function TracesTable({
       id: "totalTokens",
       header: "Total Tokens",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
-        return <span>{value.totalTokens}</span>;
+        const value: TracesTableRow["usage"] = row.getValue("usage");
+        return <span>{numberFormatter(value.totalTokens, 0)}</span>;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -383,11 +373,7 @@ export default function TracesTable({
       header: "Usage",
       id: "usage",
       cell: ({ row }) => {
-        const value: {
-          promptTokens: number;
-          completionTokens: number;
-          totalTokens: number;
-        } = row.getValue("usage");
+        const value: TracesTableRow["usage"] = row.getValue("usage");
         return (
           <TokenUsageBadge
             promptTokens={value.promptTokens}
@@ -405,7 +391,7 @@ export default function TracesTable({
       id: "inputCost",
       header: "Input Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("inputCost");
+        const cost: TracesTableRow["inputCost"] = row.getValue("inputCost");
         return (
           <div>
             {cost ? (
@@ -425,7 +411,7 @@ export default function TracesTable({
       id: "outputCost",
       header: "Output Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("outputCost");
+        const cost: TracesTableRow["outputCost"] = row.getValue("outputCost");
         return (
           <div>
             {cost ? (
@@ -445,7 +431,7 @@ export default function TracesTable({
       id: "totalCost",
       header: "Total Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("totalCost");
+        const cost: TracesTableRow["totalCost"] = row.getValue("totalCost");
         return (
           <div>
             {cost ? (
@@ -465,7 +451,7 @@ export default function TracesTable({
       header: "Scores",
       enableColumnFilter: !omittedFilter.find((f) => f === "scores"),
       cell: ({ row }) => {
-        const values: Score[] = row.getValue("scores");
+        const values: TracesTableRow["scores"] = row.getValue("scores");
         return <GroupedScoreBadges scores={values} variant="headings" />;
       },
       enableHiding: true,
@@ -475,11 +461,11 @@ export default function TracesTable({
       header: "Input",
       id: "input",
       cell: ({ row }) => {
-        const traceId: string = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="input"
+            col="input"
             singleLine={rowHeight === "s"}
           />
         );
@@ -492,11 +478,11 @@ export default function TracesTable({
       header: "Output",
       id: "output",
       cell: ({ row }) => {
-        const traceId: string = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return (
-          <TracesIOCell
+          <TracesDynamicCell
             traceId={traceId}
-            io="output"
+            col="output"
             singleLine={rowHeight === "s"}
           />
         );
@@ -508,17 +494,24 @@ export default function TracesTable({
       accessorKey: "metadata",
       header: "Metadata",
       cell: ({ row }) => {
-        const values: string = row.getValue("metadata");
-        return <IOTableCell data={values} singleLine={rowHeight === "s"} />;
+        const traceId: TracesTableRow["id"] = row.getValue("id");
+        return (
+          <TracesDynamicCell
+            traceId={traceId}
+            col="metadata"
+            singleLine={rowHeight === "s"}
+          />
+        );
       },
       enableHiding: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "level",
       id: "level",
       header: "Level",
       cell: ({ row }) => {
-        const value: ObservationLevel = row.getValue("level");
+        const value: TracesTableRow["level"] = row.getValue("level");
         return (
           <span
             className={cn(
@@ -563,8 +556,8 @@ export default function TracesTable({
       id: "tags",
       header: "Tags",
       cell: ({ row }) => {
-        const tags: string[] = row.getValue("tags");
-        const traceId: string = row.getValue("id");
+        const tags: TracesTableRow["tags"] = row.getValue("tags");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         const filterOptionTags = traceFilterOptions.data?.tags ?? [];
         const allTags = filterOptionTags.map((t) => t.value);
         return (
@@ -583,13 +576,13 @@ export default function TracesTable({
       accessorKey: "action",
       header: "Action",
       cell: ({ row }) => {
-        const traceId = row.getValue("id");
+        const traceId: TracesTableRow["id"] = row.getValue("id");
         return traceId && typeof traceId === "string" ? (
           <DeleteButton
             itemId={traceId}
             projectId={projectId}
             scope="traces:delete"
-            invalidateFunc={() => void utils.traces.invalidate()}
+            invalidateFunc={() => void utils.traces.all.invalidate()}
             type="trace"
             isTableAction={true}
           />
@@ -668,13 +661,13 @@ export default function TracesTable({
   );
 }
 
-const TracesIOCell = ({
+const TracesDynamicCell = ({
   traceId,
-  io,
+  col,
   singleLine = false,
 }: {
   traceId: string;
-  io: "input" | "output";
+  col: "input" | "output" | "metadata";
   singleLine?: boolean;
 }) => {
   const trace = api.traces.byId.useQuery(
@@ -692,8 +685,14 @@ const TracesIOCell = ({
   return (
     <IOTableCell
       isLoading={trace.isLoading}
-      data={io === "output" ? trace.data?.output : trace.data?.input}
-      className={cn(io === "output" && "bg-accent-light-green")}
+      data={
+        col === "output"
+          ? trace.data?.output
+          : col === "input"
+            ? trace.data?.input
+            : trace.data?.metadata
+      }
+      className={cn(col === "output" && "bg-accent-light-green")}
       singleLine={singleLine}
     />
   );

web/src/components/token-usage-badge.tsx

@@ -1,5 +1,6 @@
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+import { numberFormatter } from "@/src/utils/numbers";
 import { type Observation } from "@langfuse/shared";
 
 export const TraceAggUsageBadge = (props: {
@@ -25,9 +26,9 @@ export const TokenUsageBadge = (
         observation: Observation;
       }
     | {
-        promptTokens: number;
-        completionTokens: number;
-        totalTokens: number;
+        promptTokens: number | bigint;
+        completionTokens: number | bigint;
+        totalTokens: number | bigint;
       }
   ) & {
     inline?: boolean;
@@ -52,13 +53,13 @@ export const TokenUsageBadge = (
   if (props.inline)
     return (
       <span>
-        {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+        {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
       </span>
     );
 
   return (
     <Badge variant="outline">
-      {usage.promptTokens} → {usage.completionTokens} (∑ {usage.totalTokens})
+      {`${numberFormatter(usage.promptTokens, 0)} → ${numberFormatter(usage.completionTokens, 0)} (∑ ${numberFormatter(usage.totalTokens, 0)})`}
     </Badge>
   );
 };

web/src/components/trace/IOPreview.tsx

@@ -1,16 +1,60 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { z } from "zod";
-import { deepParseJson } from "@langfuse/shared";
+import { type Prisma, deepParseJson } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
-import { useState } from "react";
+import { useMemo, useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { Fragment } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { MarkdownView } from "@/src/components/ui/MarkdownViewer";
+import { MarkdownSchema } from "@/src/components/schemas/MarkdownSchema";
+import {
+  ChatMlArraySchema,
+  ChatMlMessageSchema,
+} from "@/src/components/schemas/ChatMlSchema";
+import { useMarkdownContext } from "@/src/features/theming/useMarkdownContext";
+
+// MarkdownOrJsonView will render markdown if `isMarkdownEnabled` (global context) is true and the content is valid markdown
+// otherwise, if content is valid markdown will render JSON with switch to enable markdown globally
+function MarkdownOrJsonView({
+  content,
+  title,
+  className,
+  customCodeHeaderClassName,
+}: {
+  content?: unknown;
+  title?: string;
+  className?: string;
+  customCodeHeaderClassName?: string;
+}) {
+  const validatedMarkdown = useMemo(
+    () => MarkdownSchema.safeParse(content),
+    [content],
+  );
+
+  const { isMarkdownEnabled } = useMarkdownContext();
+
+  return validatedMarkdown.success && isMarkdownEnabled ? (
+    <MarkdownView
+      markdown={validatedMarkdown.data}
+      title={title}
+      className={className}
+      customCodeHeaderClassName={customCodeHeaderClassName}
+    />
+  ) : (
+    <JSONView
+      json={content}
+      canEnableMarkdown={validatedMarkdown.success}
+      title={title}
+      className={className}
+    />
+  );
+}
 
 export const IOPreview: React.FC<{
-  input?: unknown;
-  output?: unknown;
+  input?: Prisma.JsonValue;
+  output?: Prisma.JsonValue;
   isLoading?: boolean;
   hideIfNull?: boolean;
 }> = ({ isLoading = false, hideIfNull = false, ...props }) => {
@@ -59,43 +103,67 @@ export const IOPreview: React.FC<{
     Array.isArray(output) ? output : [output],
   );
 
-  // Pretty view available
-  const isPrettyViewAvailable = inChatMlArray.success;
+  const inMarkdown = MarkdownSchema.safeParse(input);
+  const outMarkdown = MarkdownSchema.safeParse(output);
+
+  const isPrettyViewAvailable =
+    inChatMlArray.success || inMarkdown.success || outMarkdown.success;
 
   // default I/O
   return (
     <>
       {isPrettyViewAvailable ? (
-        <Tabs
-          value={currentView}
-          onValueChange={(v) => {
-            setCurrentView(v as "pretty" | "json"),
-              capture("trace_detail:io_mode_switch", { view: v });
-          }}
-        >
-          <TabsList>
-            <TabsTrigger value="pretty">Pretty ✨</TabsTrigger>
-            <TabsTrigger value="json">JSON</TabsTrigger>
-          </TabsList>
-        </Tabs>
+        <div className="flex flex-row justify-between">
+          <Tabs
+            value={currentView}
+            onValueChange={(v) => {
+              setCurrentView(v as "pretty" | "json"),
+                capture("trace_detail:io_mode_switch", { view: v });
+            }}
+          >
+            <TabsList>
+              <TabsTrigger value="pretty">Pretty ✨</TabsTrigger>
+              <TabsTrigger value="json">JSON</TabsTrigger>
+            </TabsList>
+          </Tabs>
+        </div>
       ) : null}
       {isPrettyViewAvailable && currentView === "pretty" ? (
-        <OpenAiMessageView
-          messages={[
-            ...inChatMlArray.data,
-            ...(outChatMlArray.success
-              ? outChatMlArray.data.map((m) => ({
-                  ...m,
-                  role: m.role ?? "assistant",
-                }))
-              : [
-                  ChatMlMessageSchema.parse({
-                    role: "assistant",
-                    content: outputClean ? JSON.stringify(outputClean) : null,
-                  }),
-                ]),
-          ]}
-        />
+        <>
+          {inChatMlArray.success ? (
+            <OpenAiMessageView
+              messages={[
+                ...inChatMlArray.data,
+                ...(outChatMlArray.success
+                  ? outChatMlArray.data.map((m) => ({
+                      ...m,
+                      role: m.role ?? "assistant",
+                    }))
+                  : [
+                      ChatMlMessageSchema.parse({
+                        role: "assistant",
+                        content: outputClean,
+                      }),
+                    ]),
+              ]}
+              shouldRenderMarkdown
+            />
+          ) : (
+            <>
+              {!(hideIfNull && !input) ? (
+                <MarkdownOrJsonView title="Input" content={input} />
+              ) : null}
+              {!(hideIfNull && !output) ? (
+                <MarkdownOrJsonView
+                  title="Output"
+                  content={output}
+                  className="bg-accent-light-green dark:border-accent-dark-green"
+                  customCodeHeaderClassName="bg-muted-green dark:bg-secondary"
+                />
+              ) : null}
+            </>
+          )}
+        </>
       ) : null}
       {currentView === "json" || !isPrettyViewAvailable ? (
         <>
@@ -121,57 +189,23 @@ export const IOPreview: React.FC<{
   );
 };
 
-const ChatMlMessageSchema = z
-  .object({
-    role: z
-      .enum(["system", "user", "assistant", "function", "tool"])
-      .optional(),
-    name: z.string().optional(),
-    content: z
-      .union([z.record(z.any()), z.string(), z.array(z.any())])
-      .nullish(),
-    additional_kwargs: z.record(z.any()).optional(),
-  })
-  .passthrough()
-  .refine((value) => value.content !== null || value.role !== undefined)
-  .transform(({ additional_kwargs, ...other }) => ({
-    ...other,
-    ...additional_kwargs,
-  }))
-  .transform(({ role, name, content, ...other }) => ({
-    role,
-    name,
-    content,
-    json: Object.keys(other).length === 0 ? undefined : other,
-  }));
-export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
-
 export const OpenAiMessageView: React.FC<{
-  title?: string;
   messages: z.infer<typeof ChatMlArraySchema>;
-}> = ({ title, messages }) => {
+  title?: string;
+  shouldRenderMarkdown?: boolean;
+}> = ({ title, messages, shouldRenderMarkdown = false }) => {
   const COLLAPSE_THRESHOLD = 3;
   const [isCollapsed, setCollapsed] = useState(
     messages.length > COLLAPSE_THRESHOLD ? true : null,
   );
 
-  const transformedMessages = messages;
-  // const transformedMessages = messages.map(
-  //   ({ role, name, content, ...rest }) => ({
-  //     role,
-  //     name,
-  //     content,
-  //     json: rest,
-  //   }),
-  // );
-
   return (
     <div className="rounded-md border">
       {title && (
         <div className="border-b px-3 py-1 text-xs font-medium">{title}</div>
       )}
       <div className="flex flex-col gap-2 p-3">
-        {transformedMessages
+        {messages
           .filter(
             (_, i) =>
               // show all if not collapsed or null; show first and last n if collapsed
@@ -182,20 +216,38 @@ export const OpenAiMessageView: React.FC<{
           .map((message, index) => (
             <Fragment key={index}>
               <div>
-                {!!message.content && (
-                  <JSONView
-                    title={message.name ?? message.role}
-                    json={message.content}
-                    className={cn(
-                      "bg-muted",
-                      message.role === "system" && "bg-primary-foreground",
-                      message.role === "assistant" &&
-                        "bg-accent-light-green dark:border-accent-dark-green",
-                      message.role === "user" && "bg-background",
-                      !!message.json && "rounded-b-none",
-                    )}
-                  />
-                )}
+                {!!message.content &&
+                  (shouldRenderMarkdown ? (
+                    <MarkdownOrJsonView
+                      title={message.name ?? message.role}
+                      content={message.content}
+                      className={cn(
+                        "bg-muted",
+                        message.role === "system" && "bg-primary-foreground",
+                        message.role === "assistant" &&
+                          "bg-accent-light-green dark:border-accent-dark-green",
+                        message.role === "user" && "bg-background",
+                        !!message.json && "rounded-b-none",
+                      )}
+                      customCodeHeaderClassName={cn(
+                        message.role === "assistant" &&
+                          "bg-muted-green dark:bg-secondary",
+                      )}
+                    />
+                  ) : (
+                    <JSONView
+                      title={message.name ?? message.role}
+                      json={message.content}
+                      className={cn(
+                        "bg-muted",
+                        message.role === "system" && "bg-primary-foreground",
+                        message.role === "assistant" &&
+                          "bg-accent-light-green dark:border-accent-dark-green",
+                        message.role === "user" && "bg-background",
+                        !!message.json && "rounded-b-none",
+                      )}
+                    />
+                  ))}
                 {!!message.json && (
                   <JSONView
                     title={

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type ScoreSource, type Score } from "@langfuse/shared";
+import { type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -22,11 +22,12 @@ import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const ObservationPreview = (props: {
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string;
   traceId: string;
 }) => {
@@ -59,7 +60,7 @@ export const ObservationPreview = (props: {
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, APIScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
@@ -147,7 +148,11 @@ export const ObservationPreview = (props: {
                     .filter(Boolean)
                     .map(([key, value]) => (
                       <Badge variant="outline" key={key}>
-                        {key}: {value?.toString()}
+                        {key}:{" "}
+                        {Object.prototype.toString.call(value) ===
+                        "[object Object]"
+                          ? JSON.stringify(value)
+                          : value?.toString()}
                       </Badge>
                     ))
                 : null}
@@ -160,6 +165,7 @@ export const ObservationPreview = (props: {
               observationId={preloadedObservation.id}
               scores={props.scores}
               type="observation"
+              key={"annotation-drawer" + preloadedObservation.id}
             />
             {observationWithInputAndOutput.data?.type === "GENERATION" && (
               <JumpToPlaygroundButton

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,7 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, type $Enums } from "@langfuse/shared";
+import { type Trace, type $Enums } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -22,7 +23,7 @@ export const ObservationTree = (props: {
   collapseAll: () => void;
   expandAll: () => void;
   trace: Trace;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics: boolean;
@@ -61,7 +62,7 @@ const ObservationTreeTraceNode = (props: {
   trace: Trace & { latency?: number };
   expandAll: () => void;
   collapseAll: () => void;
-  scores: Score[];
+  scores: APIScore[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics?: boolean;
@@ -119,7 +120,7 @@ const ObservationTreeNode = (props: {
   observations: NestedObservation[];
   collapsedObservations: string[];
   toggleCollapsedObservation: (id: string) => void;
-  scores: Score[];
+  scores: APIScore[];
   indentationLevel: number;
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;

web/src/components/trace/ScoresPreview.tsx

@@ -1,10 +1,10 @@
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { type Score } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const ScoresPreview = ({
   itemScoresBySource,
 }: {
-  itemScoresBySource: Map<string, Score[]>;
+  itemScoresBySource: Map<string, APIScore[]>;
 }) => {
   if (!Boolean(itemScoresBySource.size)) return null;
 

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
-import { type Trace, type Score, type ScoreSource } from "@langfuse/shared";
+import { type Trace, type ScoreSource } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -19,6 +19,7 @@ import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
 import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/manual-scoring/components/AnnotateDrawer";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export const TracePreview = ({
   trace,
@@ -27,7 +28,7 @@ export const TracePreview = ({
 }: {
   trace: Trace & { latency?: number };
   observations: ObservationReturnType[];
-  scores: Score[];
+  scores: APIScore[];
 }) => {
   const [selectedTab, setSelectedTab] = useQueryParam(
     "view",
@@ -41,7 +42,7 @@ export const TracePreview = ({
     }
     acc.get(score.source)?.push(score);
     return acc;
-  }, new Map<ScoreSource, Score[]>());
+  }, new Map<ScoreSource, APIScore[]>());
 
   return (
     <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
@@ -99,6 +100,7 @@ export const TracePreview = ({
               projectId={trace.projectId}
               traceId={trace.id}
               scores={scores}
+              key={"annotation-drawer" + trace.id}
             />
             <NewDatasetItemFromTrace
               traceId={trace.id}

web/src/components/trace/TraceTimelineView.tsx

@@ -1,12 +1,19 @@
 import { Card } from "@/src/components/ui/card";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Score, type Trace } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
-import React, { useEffect, useRef, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";
 import { SimpleTreeView } from "@mui/x-tree-view/SimpleTreeView";
 import { TreeItem } from "@mui/x-tree-view/TreeItem";
 
-import { MinusIcon, PlusIcon, PanelRightOpen } from "lucide-react";
+import {
+  MinusIcon,
+  PlusIcon,
+  PanelRightOpen,
+  PlusSquareIcon,
+  MinusSquare,
+} from "lucide-react";
 import { nestObservations } from "@/src/components/trace/lib/helpers";
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
@@ -36,6 +43,24 @@ const PREDEFINED_STEP_SIZES = [
   0.25, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10,
 ];
 
+const getNestedObservationKeys = (
+  observations: NestedObservation[],
+): string[] => {
+  const keys: string[] = [];
+
+  const collectKeys = (obs: NestedObservation[]) => {
+    obs.forEach((observation) => {
+      keys.push(`observation-${observation.id}`);
+      if (observation.children) {
+        collectKeys(observation.children);
+      }
+    });
+  };
+
+  collectKeys(observations);
+  return keys;
+};
+
 const calculateStepSize = (latency: number, scaleWidth: number) => {
   const calculatedStepSize = latency / (scaleWidth / STEP_SIZE);
   return (
@@ -163,7 +188,7 @@ function TraceTreeItem({
   traceStartTime: Date;
   totalScaleSpan: number;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
   observations: Array<ObservationReturnType>;
   cardWidth: number;
 }) {
@@ -184,8 +209,8 @@ function TraceTreeItem({
         selected: "!bg-background !important hover:!bg-muted",
         label: "!min-w-fit",
       }}
-      key={observation.id}
-      itemId={observation.id}
+      key={`observation-${observation.id}`}
+      itemId={`observation-${observation.id}`}
       label={
         <TreeItemInner
           latency={latency}
@@ -217,7 +242,7 @@ function TraceTreeItem({
       {Array.isArray(observation.children)
         ? observation.children.map((child) => (
             <TraceTreeItem
-              key={child.id}
+              key={`observation-${child.id}`}
               observation={child}
               level={level + 1}
               traceStartTime={traceStartTime}
@@ -242,13 +267,13 @@ export function TraceTimelineView({
   trace: Trace & { latency?: number };
   observations: Array<ObservationReturnType>;
   projectId: string;
-  scores: Score[];
+  scores: APIScore[];
 }) {
   const { latency, name, id } = trace;
   const [backgroundColor, setBackgroundColor] = useState("");
   const [expandedItems, setExpandedItems] = useSessionStorage<string[]>(
-    `${trace.id}-expanded`,
-    [trace.id],
+    `${id}-expanded`,
+    [`trace-${id}`],
   );
 
   const [cardWidth, setCardWidth] = useState(0);
@@ -270,9 +295,17 @@ export function TraceTimelineView({
     };
   }, [parentRef]);
 
+  const nestedObservations = useMemo(
+    () => nestObservations(observations),
+    [observations],
+  );
+  const nestedObservationKeys = useMemo(
+    () => getNestedObservationKeys(nestedObservations),
+    [nestedObservations],
+  );
+
   if (!latency) return null;
 
-  const nestedObservations = nestObservations(observations);
   const stepSize = calculateStepSize(latency, SCALE_WIDTH);
   const totalScaleSpan = stepSize * (SCALE_WIDTH / STEP_SIZE);
 
@@ -283,14 +316,39 @@ export function TraceTimelineView({
         style={{ width: cardWidth }}
       >
         <div className="grid w-full grid-cols-[1fr,auto] items-center p-2">
-          <h3
-            className="p-2 text-2xl font-semibold tracking-tight"
+          <div
+            className="flex flex-row items-center gap-2"
             style={{
               minWidth: `${MIN_LABEL_WIDTH}px`,
             }}
           >
-            Trace Timeline
-          </h3>
+            <h3 className="text-2xl font-semibold tracking-tight">
+              Trace Timeline
+            </h3>
+            <div className="flex h-full items-center">
+              <Button
+                onClick={() =>
+                  setExpandedItems([
+                    `trace-${trace.id}`,
+                    ...nestedObservationKeys,
+                  ])
+                }
+                size="xs"
+                variant="ghost"
+                title="Expand all"
+              >
+                <PlusSquareIcon className="h-4 w-4" />
+              </Button>
+              <Button
+                onClick={() => setExpandedItems([])}
+                size="xs"
+                variant="ghost"
+                title="Collapse all"
+              >
+                <MinusSquare className="h-4 w-4" />
+              </Button>
+            </div>
+          </div>
           <div
             className="relative mr-2 h-4"
             style={{ width: `${SCALE_WIDTH}px` }}
@@ -333,8 +391,8 @@ export function TraceTimelineView({
             itemChildrenIndentation={TREE_INDENTATION}
           >
             <TreeItem
-              key={id}
-              itemId={id}
+              key={`trace-${id}`}
+              itemId={`trace-${id}`}
               classes={{
                 content: `${backgroundColor} !min-w-fit !hover:bg-muted`,
                 selected: "!bg-background !important hover:!bg-muted",
@@ -365,7 +423,7 @@ export function TraceTimelineView({
               {Boolean(nestedObservations.length)
                 ? nestedObservations.map((observation) => (
                     <TraceTreeItem
-                      key={observation.id}
+                      key={`observation-${observation.id}`}
                       observation={observation}
                       level={1}
                       traceStartTime={nestedObservations[0].startTime}

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared";
+import { type Trace } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";
@@ -34,11 +34,12 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePos
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
 import { Alert, AlertDescription, AlertTitle } from "@/src/components/ui/alert";
+import { type APIScore } from "@/src/features/public-api/types/scores";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
   trace: Trace;
-  scores: Score[];
+  scores: APIScore[];
   projectId: string;
 }) {
   const capture = usePostHogClientCapture();
@@ -101,6 +102,7 @@ export function Trace(props: {
   const expandAll = useCallback(() => {
     capture("trace_detail:observation_tree_expand", { type: "all" });
     setCollapsedObservations([]);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
   return (
@@ -260,7 +262,7 @@ export function TracePage({ traceId }: { traceId: string }) {
               itemId={traceId}
               projectId={trace.data.projectId}
               scope="traces:delete"
-              invalidateFunc={() => void utils.traces.invalidate()}
+              invalidateFunc={() => void utils.traces.all.invalidate()}
               type="trace"
               redirectUrl={`/project/${router.query.projectId as string}/traces`}
             />
@@ -293,7 +295,7 @@ export function TracePage({ traceId }: { traceId: string }) {
           </Badge>
         ) : undefined}
       </div>
-      <div className="mt-5 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
+      <div className="mt-4 rounded-lg border bg-card font-semibold text-card-foreground shadow-sm">
         <div className="flex flex-row items-center gap-3 p-2.5">
           Tags
           <TagTraceDetailsPopover
@@ -311,19 +313,19 @@ export function TracePage({ traceId }: { traceId: string }) {
           setSelectedTab(tab);
           capture("trace_detail:display_mode_switch", { view: tab });
         }}
-        className="flex w-full justify-end border-b bg-background"
+        className="mt-2 flex w-full justify-end border-b bg-transparent"
       >
-        <TabsList className="bg-background py-0">
+        <TabsList className="bg-transparent py-0">
           <TabsTrigger
             value="details"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <Network className="mr-1 h-4 w-4"></Network>
             Tree
           </TabsTrigger>
           <TabsTrigger
             value="timeline"
-            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:shadow-none"
+            className="h-full rounded-none border-b-4 border-transparent data-[state=active]:border-primary-accent data-[state=active]:bg-transparent data-[state=active]:shadow-none"
           >
             <ListTree className="mr-1 h-4 w-4"></ListTree>
             Timeline
@@ -332,7 +334,7 @@ export function TracePage({ traceId }: { traceId: string }) {
         </TabsList>
       </Tabs>
       {selectedTab === "details" && (
-        <div className="mt-5 flex-1 overflow-hidden border-t pt-5">
+        <div className="mt-5 flex-1 overflow-hidden">
           <Trace
             key={trace.data.id}
             trace={trace.data}

web/src/components/trace/lib/helpers.ts

@@ -16,8 +16,19 @@ export function nestObservations(
 ): NestedObservation[] {
   if (list.length === 0) return [];
 
+  // Data prep: Remove parentObservationId attribute from observations if the id does not exist in the list of observations
+  const mutableList = list.map((o) => ({ ...o }));
+  mutableList.forEach((observation) => {
+    if (
+      observation.parentObservationId &&
+      !list.find((o) => o.id === observation.parentObservationId)
+    ) {
+      observation.parentObservationId = null;
+    }
+  });
+
   // Step 0: Sort the list by start time to ensure observations are in right order
-  const sortedObservations = list.sort(
+  const sortedObservations = mutableList.sort(
     (a, b) => a.startTime.getTime() - b.startTime.getTime(),
   );