chore: release v2.11.3

push

.env.prod.example

@@ -29,7 +29,7 @@ SALT="salt"
 # LANGFUSE_DEFAULT_PROJECT_ROLE=
 
 # Enable experimental features, optional
-# NEXT_PUBLIC_ENABLE_EXPERIMENTAL_FEATURES
+# LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
@@ -38,11 +38,14 @@ SALT="salt"
 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
 # AUTH_GOOGLE_CLIENT_SECRET=
+# AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
+# AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
+# AUTH_AZURE_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.

.github/dependabot.yml

@@ -6,7 +6,34 @@
 version: 2
 updates:
   - package-ecosystem: npm
-    directory: "/" # Location of package manifests
+    directory: "/worker" # Location of package manifests
+    schedule:
+      interval: "daily"
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
+    commit-message:
+      prefix: chore
+      prefix-development: chore
+      include: scope
+    ignore:
+      - dependency-name: "@types/node"
+      - dependency-name: "@trpc/*"
+    groups:
+      sentry:
+        patterns:
+          - "@sentry/*"
+      prisma:
+        patterns:
+          - "prisma"
+          - "@prisma/*"
+      next:
+        patterns:
+          - "eslint-config-next"
+          - "next"
+      patches:
+        update-types:
+          - "patch"
+  - package-ecosystem: npm
+    directory: "/web" # Location of package manifests
     schedule:
       interval: "daily"
     rebase-strategy: "disabled" # use dependabot-rebase-stale

.github/workflows/pipeline.yml

@@ -18,6 +18,9 @@ defaults:
 
 jobs:
   lint:
+    defaults:
+      run:
+        working-directory: .
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -30,10 +33,13 @@ jobs:
         run: |
           npm ci
       - name: Load default env
-        working-directory: .
         run: |
           cp .env.dev.example .env
-      - name: lint
+      - name: lint web
+        working-directory: ./web
+        run: npm run lint
+      - name: lint worker
+        working-directory: ./worker
         run: npm run lint
 
   test-docker-build:
@@ -206,6 +212,7 @@ jobs:
             type=sha
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
 
       - name: Build and push Docker image
         uses: docker/build-push-action@v4

CONTRIBUTING.md

@@ -91,7 +91,7 @@ flowchart TB
 
 The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
 
-Full database schema: [prisma/schema.prisma](prisma/schema.prisma)
+Full database schema: [web/prisma/schema.prisma](web/prisma/schema.prisma)
 
 <img src="./web/prisma/database.svg">
 
@@ -110,7 +110,14 @@ flowchart LR
    App --- DB
 ```
 
-## Development Setup
+## Repository Structure
+
+This repository contains the following packages:
+
+- `web`: is the main application package providing Frontend and Backend APIs for Langfuse
+- `worker` (no production yet): contains an application for asynchronous processing of tasks. This package is not yet used in production.
+
+## Development Setup (web package only)
 
 Requirements
 
@@ -146,6 +153,8 @@ Requirements
 
 6. Run the migrations
 
+   The `schema.prisma` file is in the `web` directory and hence, migrations are applied from there.
+
    ```bash
    npm run db:migrate
 

README.md

@@ -21,21 +21,12 @@
    </div>
    </br>
    <div>
-      <a href="https://langfuse.com/docs">
-         <strong>Docs</strong>
-      </a> ·
-      <a href="https://langfuse.com/issue">
-         <strong>Report Bug</strong>
-      </a> ·
-      <a href="https://langfuse.com/idea">
-         <strong>Feature Request</strong>
-      </a> ·
-      <a href="https://langfuse.com/changelog">
-         <strong>Changelog</strong>
-      </a> ·
-      <a href="https://langfuse.com/discord">
-         <strong>Discord</strong>
-      </a> 
+      <a href="https://langfuse.com/docs"><strong>Docs</strong></a> ·
+      <a href="https://langfuse.com/issues"><strong>Report Bug</strong></a> ·
+      <a href="https://langfuse.com/ideas"><strong>Feature Request</strong></a> ·
+      <a href="https://langfuse.com/changelog"><strong>Changelog</strong></a> ·
+      <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
+      <a href="https://langfuse.com/discord"><strong>Discord</strong></a> 
    </div>
    </br>
       <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
@@ -119,20 +110,22 @@ See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get start
 
 ### Integrations
 
-| Integration                                              | Supports      | Description                                                                     |
-| -------------------------------------------------------- | ------------- | ------------------------------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk) | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.                     |
-| [OpenAI](https://langfuse.com/docs/openai)               | Python        | Automated instrumentation using drop-in replacement of OpenAI SDK.              |
-| [Langchain](https://langfuse.com/docs/langchain)         | Python, JS/TS | Automated instrumentation by passing callback handler to Langchain application. |
-| [API](https://langfuse.com/docs/api)                     |               | Directly call the public API. OpenAPI spec available.                           |
+| Integration                                                      | Supports      | Description                                                      |
+| ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
+| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.      |
+| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python        | Automated instrumentation via drop-in replacement of OpenAI SDK. |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Automated instrumentation via LlamaIndex callback system.        |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python, JS/TS | Automated instrumentation via LlamaIndex callback system.        |
+| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.            |
 
 External projects/packages that integrate with Langfuse:
 
-| Name                                           | Description                                                                                                                                      |
-| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](/https://langfuse.comdocs/litellm)   | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/flowise)   | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/langflow) | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
+| Name                                                            | Description                                                                                                                                      |
+| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
+| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
+| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
 
 ## Questions and feedback
 
@@ -140,6 +133,7 @@ External projects/packages that integrate with Langfuse:
 
 - [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
 - [Feature Requests](https://langfuse.com/idea)
+- [Roadmap](https://langfuse.com/roadmap)
 
 ### Support and feedback
 
@@ -178,6 +172,6 @@ This helps us to:
 1. Understand how Langfuse is used and improve the most relevant features.
 2. Track overall usage for internal and external (e.g. fundraising) reporting.
 
-None of the data is shared with third parties and does not include any sensitive information. We want to be super transparent about this and you can find the exact data we collect [here](/src/features/telemetry/index.ts).
+None of the data is shared with third parties and does not include any sensitive information. We want to be super transparent about this and you can find the exact data we collect [here](/web/src/features/telemetry/index.ts).
 
 You can opt-out by setting `TELEMETRY_ENABLED=false`.

fern/apis/server/definition/commons.yml

@@ -90,20 +90,35 @@ types:
       latency: optional<double>
 
   Usage:
+    docs: Standard interface for usage and cost
     properties:
-      input: optional<integer>
-      output: optional<integer>
-      total: optional<integer>
+      input:
+        docs: Number of input units (e.g. tokens)
+        type: optional<integer>
+      output:
+        docs: Number of output units (e.g. tokens)
+        type: optional<integer>
+      total:
+        docs: Defaults to input+output if not set
+        type: optional<integer>
       unit: optional<ModelUsageUnit>
-      inputCost: optional<double>
-      outputCost: optional<double>
-      totalCost: optional<double>
+      inputCost:
+        docs: USD input cost
+        type: optional<double>
+      outputCost:
+        docs: USD output cost
+        type: optional<double>
+      totalCost:
+        docs: USD total cost, defaults to input+output
+        type: optional<double>
+
   Score:
     properties:
       id: string
       traceId: string
       name: string
       value: double
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
@@ -145,6 +160,7 @@ types:
 
   # Utilities
   ModelUsageUnit:
+    docs: Unit of usage in Langfuse
     enum:
       - CHARACTERS
       - TOKENS
@@ -168,6 +184,10 @@ types:
     enum:
       - ACTIVE
       - ARCHIVED
+  ScoreSource:
+    enum:
+      - API
+      - REVIEW
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -7,12 +7,12 @@ service:
   endpoints:
     create:
       method: POST
-      docs: Create a dataset item, upserts on id
+      docs: Create a dataset item
       path: /dataset-items
       request: CreateDatasetItemRequest
       response: commons.DatasetItem
     get:
-      docs: Get a specific dataset item
+      docs: Get a dataset item
       method: GET
       path: /dataset-items/{id}
       path-parameters:
@@ -26,4 +26,6 @@ types:
       datasetName: string
       input: unknown
       expectedOutput: optional<unknown>
-      id: optional<string>
+      id:
+        type: optional<string>
+        docs: Dataset items are upserted on their id

fern/apis/server/definition/ingestion.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     batch:
-      docs: Ingest multiple events to Langfuse
+      docs: Batched ingestion for Langfuse Tracing
       method: POST
       path: /ingestion
       request:
@@ -31,8 +31,12 @@ types:
       sdk-log: SDKLogEvent
 
       # both are legacy
-      observation-create: CreateObservationEvent
-      observation-update: UpdateObservationEvent
+      observation-create:
+        type: CreateObservationEvent
+        docs: Deprecated event type
+      observation-update:
+        type: UpdateObservationEvent
+        docs: Deprecated event type
 
   ObservationType:
     enum:
@@ -47,6 +51,7 @@ types:
       - OpenAIUsage
 
   OpenAIUsage:
+    docs: Usage interface of OpenAI for improved compatibility.
     properties:
       promptTokens: optional<integer>
       completionTokens: optional<integer>

fern/apis/server/definition/observations.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a specific observation
+      docs: Get a observation
       method: GET
       path: /observations/{observationId}
       path-parameters:

fern/apis/server/definition/projects.yml

@@ -7,6 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
+      docs: Get Project associated with API key
       method: GET
       path: /projects
       response: Projects

fern/apis/server/definition/prompts.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a specific prompt
+      docs: Get a prompt
       method: GET
       path: /prompts
       request:
@@ -17,7 +17,7 @@ service:
           version: optional<integer>
       response: Prompt
     create:
-      docs: Create a specific prompt
+      docs: Create a prompt
       method: POST
       path: /prompts
       request: CreatePromptRequest
@@ -27,7 +27,9 @@ types:
   CreatePromptRequest:
     properties:
       name: string
-      isActive: boolean
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
       prompt: string
       config: optional<unknown>
 

fern/apis/server/definition/score.yml

@@ -7,13 +7,13 @@ service:
   base-path: /api/public
   endpoints:
     create:
-      docs: Add a score to the database, upserts on id
+      docs: Create a score
       method: POST
       path: /scores
       request: CreateScoreRequest
       response: commons.Score
     get:
-      docs: Get scores
+      docs: Get a list of scores
       method: GET
       path: /scores
       request:
@@ -24,6 +24,15 @@ service:
           userId: optional<string>
           name: optional<string>
       response: Scores
+    get-by-id:
+      docs: Get a score
+      method: GET
+      path: /scores/{scoreId}
+      path-parameters:
+        scoreId:
+          type: string
+          docs: The unique langfuse identifier of a score
+      response: commons.Score
     delete:
       docs: Delete a score
       method: DELETE

fern/apis/server/definition/trace.yml

@@ -27,8 +27,8 @@ service:
           userId: optional<string>
           name: optional<string>
           orderBy:
-            type: string
-            docs: Format of the string sort_by=timestamp.asc (id, timestamp, name, userId, release, version, public, bookmarked, sessionId)
+            type: optional<string>
+            docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"
           tags:
             type: optional<string>
             allow-multiple: true

generated/postman/collection.json

@@ -46,7 +46,7 @@
           "_type": "endpoint",
           "name": "Create",
           "request": {
-            "description": "Create a dataset item, upserts on id",
+            "description": "Create a dataset item",
             "url": {
               "raw": "{{baseUrl}}/api/public/dataset-items",
               "host": [
@@ -79,7 +79,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a specific dataset item",
+            "description": "Get a dataset item",
             "url": {
               "raw": "{{baseUrl}}/api/public/dataset-items/:id",
               "host": [
@@ -301,7 +301,7 @@
           "_type": "endpoint",
           "name": "Batch",
           "request": {
-            "description": "Ingest multiple events to Langfuse",
+            "description": "Batched ingestion for Langfuse Tracing",
             "url": {
               "raw": "{{baseUrl}}/api/public/ingestion",
               "host": [
@@ -400,7 +400,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a specific observation",
+            "description": "Get a observation",
             "url": {
               "raw": "{{baseUrl}}/api/public/observations/:observationId",
               "host": [
@@ -500,7 +500,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": null,
+            "description": "Get Project associated with API key",
             "url": {
               "raw": "{{baseUrl}}/api/public/projects",
               "host": [
@@ -532,7 +532,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get a specific prompt",
+            "description": "Get a prompt",
             "url": {
               "raw": "{{baseUrl}}/api/public/prompts?name=&version=",
               "host": [
@@ -568,7 +568,7 @@
           "_type": "endpoint",
           "name": "Create",
           "request": {
-            "description": "Create a specific prompt",
+            "description": "Create a prompt",
             "url": {
               "raw": "{{baseUrl}}/api/public/prompts",
               "host": [
@@ -608,7 +608,7 @@
           "_type": "endpoint",
           "name": "Create",
           "request": {
-            "description": "Add a score to the database, upserts on id",
+            "description": "Create a score",
             "url": {
               "raw": "{{baseUrl}}/api/public/scores",
               "host": [
@@ -641,7 +641,7 @@
           "_type": "endpoint",
           "name": "Get",
           "request": {
-            "description": "Get scores",
+            "description": "Get a list of scores",
             "url": {
               "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=",
               "host": [
@@ -683,6 +683,38 @@
           },
           "response": []
         },
+        {
+          "_type": "endpoint",
+          "name": "Get By Id",
+          "request": {
+            "description": "Get a score",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scores/:scoreId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scores",
+                ":scoreId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "scoreId",
+                  "value": "",
+                  "description": "The unique langfuse identifier of a score"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Delete",
@@ -832,7 +864,7 @@
                 {
                   "key": "orderBy",
                   "value": "",
-                  "description": "Format of the string sort_by=timestamp.asc (id, timestamp, name, userId, release, version, public, bookmarked, sessionId)"
+                  "description": "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"
                 },
                 {
                   "key": "tags",

package-lock.json

@@ -14,6 +14,7 @@
         "@types/jest": "^29.5.12",
         "@typescript-eslint/eslint-plugin": "^7.0.2",
         "@typescript-eslint/parser": "^7.0.2",
+        "dotenv-cli": "^7.3.0",
         "eslint": "^8.56.0",
         "prettier": "^3.2.5"
       }
@@ -926,6 +927,42 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/dotenv": {
+      "version": "16.4.5",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
+      "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/dotenv-cli": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/dotenv-cli/-/dotenv-cli-7.3.0.tgz",
+      "integrity": "sha512-314CA4TyK34YEJ6ntBf80eUY+t1XaFLyem1k9P0sX1gn30qThZ5qZr/ZwE318gEnzyYP9yj9HJk6SqwE0upkfw==",
+      "dev": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.3",
+        "dotenv": "^16.3.0",
+        "dotenv-expand": "^10.0.0",
+        "minimist": "^1.2.6"
+      },
+      "bin": {
+        "dotenv": "cli.js"
+      }
+    },
+    "node_modules/dotenv-expand": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz",
+      "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/escape-string-regexp": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@@ -1601,6 +1638,15 @@
         "node": "*"
       }
     },
+    "node_modules/minimist": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",

package.json

@@ -2,10 +2,12 @@
   "name": "langfuse",
   "version": "2.7.0",
   "scripts": {
-    "infra:dev:up": "docker-compose -f ../docker-compose.dev.yml up -d",
-    "infra:dev:down": "docker-compose -f ../docker-compose.dev.yml down",
+    "infra:dev:up": "docker-compose -f ./docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker-compose -f ./docker-compose.dev.yml down",
     "dev": "npm run dev --prefix worker/ & npm run dev --prefix web/",
-    "postinstall": "cd worker && npm install && cd ../web && npm install"
+    "postinstall": "cd worker && npm install && cd ../web && npm install",
+    "nuke": "bash ./scripts/nuke.sh",
+    "dx": "dotenv -e .env -- npm i && npm run infra:dev:up && cd web && npm run db:reset && npm run db:seed:examples && cd .. && npm run dev"
   },
   "author": "",
   "license": "MIT",
@@ -14,6 +16,7 @@
     "@types/jest": "^29.5.12",
     "@typescript-eslint/eslint-plugin": "^7.0.2",
     "@typescript-eslint/parser": "^7.0.2",
+    "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
     "prettier": "^3.2.5"
   }

scripts/nuke.sh

@@ -0,0 +1,9 @@
+
+#  used to nuke the dev environment for engineers
+
+find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \;
+find . -name '.next' -type d -prune -print -exec rm -rf '{}' \;
+find . -iname "bin" -type d -prune -print -exec rm -rf '{}' \;
+find . -iname "dist" -type d -prune -print -exec rm -rf '{}' \;
+find . -iname "out" -type d -prune -print -exec rm -rf '{}' \;
+find . -iname ".turbo" -type d -prune -print -exec rm -rf '{}' \;

web/generated/openapi-server/openapi.yml

@@ -16,7 +16,7 @@ info:
 paths:
   /api/public/dataset-items:
     post:
-      description: Create a dataset item, upserts on id
+      description: Create a dataset item
       operationId: datasetItems_create
       tags:
         - DatasetItems
@@ -63,7 +63,7 @@ paths:
               $ref: '#/components/schemas/CreateDatasetItemRequest'
   /api/public/dataset-items/{id}:
     get:
-      description: Get a specific dataset item
+      description: Get a dataset item
       operationId: datasetItems_get
       tags:
         - DatasetItems
@@ -336,7 +336,7 @@ paths:
           description: ''
   /api/public/ingestion:
     post:
-      description: Ingest multiple events to Langfuse
+      description: Batched ingestion for Langfuse Tracing
       operationId: ingestion_batch
       tags:
         - Ingestion
@@ -464,7 +464,7 @@ paths:
       security: *ref_0
   /api/public/observations/{observationId}:
     get:
-      description: Get a specific observation
+      description: Get a observation
       operationId: observations_get
       tags:
         - Observations
@@ -594,6 +594,7 @@ paths:
       security: *ref_0
   /api/public/projects:
     get:
+      description: Get Project associated with API key
       operationId: projects_get
       tags:
         - Projects
@@ -633,7 +634,7 @@ paths:
       security: *ref_0
   /api/public/prompts:
     get:
-      description: Get a specific prompt
+      description: Get a prompt
       operationId: prompts_get
       tags:
         - Prompts
@@ -683,7 +684,7 @@ paths:
               schema: {}
       security: *ref_0
     post:
-      description: Create a specific prompt
+      description: Create a prompt
       operationId: prompts_create
       tags:
         - Prompts
@@ -729,7 +730,7 @@ paths:
               $ref: '#/components/schemas/CreatePromptRequest'
   /api/public/scores:
     post:
-      description: Add a score to the database, upserts on id
+      description: Create a score
       operationId: score_create
       tags:
         - Score
@@ -774,7 +775,7 @@ paths:
             schema:
               $ref: '#/components/schemas/CreateScoreRequest'
     get:
-      description: Get scores
+      description: Get a list of scores
       operationId: score_get
       tags:
         - Score
@@ -837,6 +838,51 @@ paths:
               schema: {}
       security: *ref_0
   /api/public/scores/{scoreId}:
+    get:
+      description: Get a score
+      operationId: score_get-by-id
+      tags:
+        - Score
+      parameters:
+        - name: scoreId
+          in: path
+          description: The unique langfuse identifier of a score
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Score'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
     delete:
       description: Delete a score
       operationId: score_delete
@@ -1004,11 +1050,13 @@ paths:
         - name: orderBy
           in: query
           description: >-
-            Format of the string sort_by=timestamp.asc (id, timestamp, name,
-            userId, release, version, public, bookmarked, sessionId)
-          required: true
+            Format of the string [field].[asc/desc]. Fields: id, timestamp,
+            name, userId, release, version, public, bookmarked, sessionId.
+            Example: timestamp.asc
+          required: false
           schema:
             type: string
+            nullable: true
         - name: tags
           in: query
           description: Only traces that include all of these tags will be returned.
@@ -1283,16 +1331,20 @@ components:
     Usage:
       title: Usage
       type: object
+      description: Standard interface for usage and cost
       properties:
         input:
           type: integer
           nullable: true
+          description: Number of input units (e.g. tokens)
         output:
           type: integer
           nullable: true
+          description: Number of output units (e.g. tokens)
         total:
           type: integer
           nullable: true
+          description: Defaults to input+output if not set
         unit:
           $ref: '#/components/schemas/ModelUsageUnit'
           nullable: true
@@ -1300,14 +1352,17 @@ components:
           type: number
           format: double
           nullable: true
+          description: USD input cost
         outputCost:
           type: number
           format: double
           nullable: true
+          description: USD output cost
         totalCost:
           type: number
           format: double
           nullable: true
+          description: USD total cost, defaults to input+output
     Score:
       title: Score
       type: object
@@ -1321,6 +1376,8 @@ components:
         value:
           type: number
           format: double
+        source:
+          $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
           nullable: true
@@ -1335,6 +1392,7 @@ components:
         - traceId
         - name
         - value
+        - source
         - timestamp
     Dataset:
       title: Dataset
@@ -1458,6 +1516,7 @@ components:
         - MILLISECONDS
         - SECONDS
         - IMAGES
+      description: Unit of usage in Langfuse
     ObservationLevel:
       title: ObservationLevel
       type: string
@@ -1485,6 +1544,12 @@ components:
       enum:
         - ACTIVE
         - ARCHIVED
+    ScoreSource:
+      title: ScoreSource
+      type: string
+      enum:
+        - API
+        - REVIEW
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
       type: object
@@ -1497,6 +1562,7 @@ components:
         id:
           type: string
           nullable: true
+          description: Dataset items are upserted on their id
       required:
         - datasetName
         - input
@@ -1664,6 +1730,7 @@ components:
     OpenAIUsage:
       title: OpenAIUsage
       type: object
+      description: Usage interface of OpenAI for improved compatibility.
       properties:
         promptTokens:
           type: integer
@@ -2193,6 +2260,7 @@ components:
           type: string
         isActive:
           type: boolean
+          description: Should the prompt be promoted to production immediately?
         prompt:
           type: string
         config:

web/next.config.mjs

@@ -34,26 +34,26 @@ const nextConfig = {
         ],
       },
       // Required to check authentication status from langfuse.com
-      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ?
-        [
-          {
-            source: "/api/auth/session",
-            headers: [
-              {
-                key: "Access-Control-Allow-Origin",
-                value: "https://langfuse.com",
-              },
-              { key: "Access-Control-Allow-Credentials", value: "true" },
-              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-              {
-                key: "Access-Control-Allow-Headers",
-                value: "Content-Type, Authorization",
-              },
-            ]
-          },
-        ] : []
-      )
-    ]
+      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
+        ? [
+            {
+              source: "/api/auth/session",
+              headers: [
+                {
+                  key: "Access-Control-Allow-Origin",
+                  value: "https://langfuse.com",
+                },
+                { key: "Access-Control-Allow-Credentials", value: "true" },
+                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+                {
+                  key: "Access-Control-Allow-Headers",
+                  value: "Content-Type, Authorization",
+                },
+              ],
+            },
+          ]
+        : []),
+    ];
   },
 
   // webassembly support for @dqbd/tiktoken

web/package-lock.json

@@ -1,23 +1,23 @@
 {
   "name": "langfuse-core",
-  "version": "2.7.2",
+  "version": "2.11.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "langfuse-core",
-      "version": "2.7.2",
+      "version": "2.11.3",
       "hasInstallScript": true,
       "dependencies": {
         "@anthropic-ai/tokenizer": "^0.0.4",
         "@aws-sdk/client-s3": "^3.507.0",
-        "@aws-sdk/lib-storage": "^3.511.0",
+        "@aws-sdk/lib-storage": "^3.515.0",
         "@aws-sdk/s3-request-presigner": "^3.507.0",
         "@headlessui/react": "^1.7.18",
         "@heroicons/react": "^2.1.1",
         "@hookform/resolvers": "^3.3.4",
         "@next-auth/prisma-adapter": "^1.0.7",
-        "@prisma/client": "^5.9.1",
+        "@prisma/client": "^5.10.2",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-alert-dialog": "^1.0.5",
         "@radix-ui/react-avatar": "^1.0.4",
@@ -40,8 +40,8 @@
         "@radix-ui/react-tooltip": "^1.0.7",
         "@react-email/components": "^0.0.14",
         "@react-email/render": "^0.0.12",
-        "@sentry/nextjs": "^7.101.1",
-        "@sentry/profiling-node": "^7.101.1",
+        "@sentry/nextjs": "^7.102.1",
+        "@sentry/profiling-node": "^7.102.1",
         "@sentry/types": "^7.88.0",
         "@t3-oss/env-nextjs": "^0.8.0",
         "@tailwindcss/forms": "^0.5.7",
@@ -116,9 +116,9 @@
         "postcss": "^8.4.35",
         "prettier": "^3.2.5",
         "prettier-plugin-tailwindcss": "^0.5.11",
-        "prisma": "^5.9.1",
+        "prisma": "^5.10.2",
         "prisma-erd-generator": "^1.11.2",
-        "release-it": "^17.0.5",
+        "release-it": "^17.1.1",
         "tailwindcss": "^3.4.1",
         "ts-node": "^10.9.2",
         "tsconfig-paths": "^4.2.0",
@@ -683,9 +683,9 @@
       }
     },
     "node_modules/@aws-sdk/lib-storage": {
-      "version": "3.511.0",
-      "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.511.0.tgz",
-      "integrity": "sha512-inEbSyqzGxiQs8aEnkGdxw9ZDn370mRHOdE1TB/GvVe9buQVyZ2hQvOY5WBVOaIGDIxGpuUzVvr4o89XreU19w==",
+      "version": "3.515.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.515.0.tgz",
+      "integrity": "sha512-/7z/3KnMs1ODNS9c8Skj/DFTsy6/v7n17clh1IGOcTYhhioCMA3MIzIZecWFeLjPYcUSkNQHIIjKFQt1nhZkwA==",
       "dependencies": {
         "@smithy/abort-controller": "^2.1.1",
         "@smithy/middleware-endpoint": "^2.4.1",
@@ -3459,9 +3459,9 @@
       }
     },
     "node_modules/@prisma/client": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.9.1.tgz",
-      "integrity": "sha512-caSOnG4kxcSkhqC/2ShV7rEoWwd3XrftokxJqOCMVvia4NYV/TPtJlS9C2os3Igxw/Qyxumj9GBQzcStzECvtQ==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.10.2.tgz",
+      "integrity": "sha512-ef49hzB2yJZCvM5gFHMxSFL9KYrIP9udpT5rYo0CsHD4P9IKj473MbhU1gjKKftiwWBTIyrt9jukprzZXazyag==",
       "hasInstallScript": true,
       "engines": {
         "node": ">=16.13"
@@ -3482,45 +3482,45 @@
       "dev": true
     },
     "node_modules/@prisma/engines": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.9.1.tgz",
-      "integrity": "sha512-gkdXmjxQ5jktxWNdDA5aZZ6R8rH74JkoKq6LD5mACSvxd2vbqWeWIOV0Py5wFC8vofOYShbt6XUeCIUmrOzOnQ==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.10.2.tgz",
+      "integrity": "sha512-HkSJvix6PW8YqEEt3zHfCYYJY69CXsNdhU+wna+4Y7EZ+AwzeupMnUThmvaDA7uqswiHkgm5/SZ6/4CStjaGmw==",
       "devOptional": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@prisma/debug": "5.9.1",
-        "@prisma/engines-version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
-        "@prisma/fetch-engine": "5.9.1",
-        "@prisma/get-platform": "5.9.1"
+        "@prisma/debug": "5.10.2",
+        "@prisma/engines-version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
+        "@prisma/fetch-engine": "5.10.2",
+        "@prisma/get-platform": "5.10.2"
       }
     },
     "node_modules/@prisma/engines-version": {
-      "version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
-      "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64.tgz",
-      "integrity": "sha512-HFl7275yF0FWbdcNvcSRbbu9JCBSLMcurYwvWc8WGDnpu7APxQo2ONtZrUggU3WxLxUJ2uBX+0GOFIcJeVeOOQ==",
+      "version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
+      "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9.tgz",
+      "integrity": "sha512-uCy/++3Jx/O3ufM+qv2H1L4tOemTNqcP/gyEVOlZqTpBvYJUe0tWtW0y3o2Ueq04mll4aM5X3f6ugQftOSLdFQ==",
       "devOptional": true
     },
     "node_modules/@prisma/engines/node_modules/@prisma/debug": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
-      "integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
+      "integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
       "devOptional": true
     },
     "node_modules/@prisma/fetch-engine": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.9.1.tgz",
-      "integrity": "sha512-l0goQOMcNVOJs1kAcwqpKq3ylvkD9F04Ioe1oJoCqmz05mw22bNAKKGWuDd3zTUoUZr97va0c/UfLNru+PDmNA==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.10.2.tgz",
+      "integrity": "sha512-dSmXcqSt6DpTmMaLQ9K8ZKzVAMH3qwGCmYEZr/uVnzVhxRJ1EbT/w2MMwIdBNq1zT69Rvh0h75WMIi0mrIw7Hg==",
       "devOptional": true,
       "dependencies": {
-        "@prisma/debug": "5.9.1",
-        "@prisma/engines-version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
-        "@prisma/get-platform": "5.9.1"
+        "@prisma/debug": "5.10.2",
+        "@prisma/engines-version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
+        "@prisma/get-platform": "5.10.2"
       }
     },
     "node_modules/@prisma/fetch-engine/node_modules/@prisma/debug": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
-      "integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
+      "integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
       "devOptional": true
     },
     "node_modules/@prisma/generator-helper": {
@@ -3533,18 +3533,18 @@
       }
     },
     "node_modules/@prisma/get-platform": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.9.1.tgz",
-      "integrity": "sha512-6OQsNxTyhvG+T2Ksr8FPFpuPeL4r9u0JF0OZHUBI/Uy9SS43sPyAIutt4ZEAyqWQt104ERh70EZedkHZKsnNbg==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.10.2.tgz",
+      "integrity": "sha512-nqXP6vHiY2PIsebBAuDeWiUYg8h8mfjBckHh6Jezuwej0QJNnjDiOq30uesmg+JXxGk99nqyG3B7wpcOODzXvg==",
       "devOptional": true,
       "dependencies": {
-        "@prisma/debug": "5.9.1"
+        "@prisma/debug": "5.10.2"
       }
     },
     "node_modules/@prisma/get-platform/node_modules/@prisma/debug": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
-      "integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
+      "integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
       "devOptional": true
     },
     "node_modules/@prisma/internals": {
@@ -5291,57 +5291,57 @@
       }
     },
     "node_modules/@sentry-internal/feedback": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-7.101.1.tgz",
-      "integrity": "sha512-fOKDMVvLX+FuJHJszKBvRg1m7+fd4hchqRnZ9DDfitT6P5Ppl0gbEt/LStqu8Wq5M0tna+hpdwHlVEt7gZVKzw==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-7.102.1.tgz",
+      "integrity": "sha512-vY4hpLLMNLjICtWiizc7KeGbWOTUMGrF7C+9dPCztZww3CLgzWy9A7DvPj5hodRiYzpdRnAMl8yQnMFbYXh7bA==",
       "dependencies": {
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=12"
       }
     },
     "node_modules/@sentry-internal/replay-canvas": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-7.101.1.tgz",
-      "integrity": "sha512-09l6nD+lxWvwkpXLlIZuzj/z79Llbo6mcH33TJvxrUTjAqSGF/i3Pd5bTLWro9atippOyQgIV/yTGG4Bc5FhyQ==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-7.102.1.tgz",
+      "integrity": "sha512-GUX4RWI10uRjdjeyvCLtAAhWRVqnAnG6+yNxWfqUQ3qMA7B7XxG43KT2UhSnulmErNzODQ6hA68rGPwwYeRIww==",
       "dependencies": {
-        "@sentry/core": "7.101.1",
-        "@sentry/replay": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry/core": "7.102.1",
+        "@sentry/replay": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=12"
       }
     },
     "node_modules/@sentry-internal/tracing": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.101.1.tgz",
-      "integrity": "sha512-ihjWG8x4x0ozx6t+EHoXLKbsPrgzYLCpeBLWyS+M6n3hn6cmHM76c8nZw3ldhUQi5UYL3LFC/JZ50b4oSxtlrg==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.102.1.tgz",
+      "integrity": "sha512-RkFlFyAC0fQOvBbBqnq0CLmFW5m3JJz9pKbZd5vXPraWAlniKSb1bC/4DF9SlNx0FN1LWG+IU3ISdpzwwTeAGg==",
       "dependencies": {
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=8"
       }
     },
     "node_modules/@sentry/browser": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-7.101.1.tgz",
-      "integrity": "sha512-+rIFoWPdO29AHVYsAwq8QEl2Ihv17Xh9Bt2aPFvLTGDA0caHjnx98g2jSOvLIOah6HI7Nwp3Njg2zBEzDtHkNw==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-7.102.1.tgz",
+      "integrity": "sha512-7BOfPBiM7Kp6q/iy0JIbsBTxIASV+zWXByqqjuEMWGj3X2u4oRIfm3gv4erPU/l+CORQUVQZLSPGoIoM1gbB/A==",
       "dependencies": {
-        "@sentry-internal/feedback": "7.101.1",
-        "@sentry-internal/replay-canvas": "7.101.1",
-        "@sentry-internal/tracing": "7.101.1",
-        "@sentry/core": "7.101.1",
-        "@sentry/replay": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry-internal/feedback": "7.102.1",
+        "@sentry-internal/replay-canvas": "7.102.1",
+        "@sentry-internal/tracing": "7.102.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/replay": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=8"
@@ -5368,25 +5368,25 @@
       }
     },
     "node_modules/@sentry/core": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.101.1.tgz",
-      "integrity": "sha512-XSmXXeYT1d4O14eDF3OXPJFUgaN2qYEeIGUztqPX9nBs9/ij8y/kZOayFqlIMnfGvjOUM+63sy/2xDBOpFn6ug==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.102.1.tgz",
+      "integrity": "sha512-QjY+LSP3du3J/C8x/FfEbRxgZgsWd0jfTJ4P7s9f219I1csK4OeBMC3UA1HwEa0pY/9OF6H/egW2CjOcMM5Pdg==",
       "dependencies": {
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=8"
       }
     },
     "node_modules/@sentry/integrations": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.101.1.tgz",
-      "integrity": "sha512-0kk6773Lg2Pa1cUmK1VtxaLLAmpIXcT3qYPlYxRXZQJEUpGcjZZ704V7i8SFLhJSsHkXrL/sAGSyM1p+NDF+QA==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.102.1.tgz",
+      "integrity": "sha512-Its3Ru6xCAqpaLE3cTxW/b91js2SIFoXa8LWtQDJ7tmTdwPAbT8Pij1F4bOhhaqLYbjLtCXGl/NR2cffsiRLww==",
       "dependencies": {
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1",
         "localforage": "^1.8.1"
       },
       "engines": {
@@ -5394,18 +5394,18 @@
       }
     },
     "node_modules/@sentry/nextjs": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-7.101.1.tgz",
-      "integrity": "sha512-qIT0jByxaAbbmX5Gl//M9+d+Pi1znk9aWoIGZyakpK+seGQ3D+UjrznxxmwvjcqQgVpbo0SJSq+3yQv7eAh2Tg==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-7.102.1.tgz",
+      "integrity": "sha512-gOI/GD7DWhc3WucyYnepl8Nu5jmpa1YfR6jWDzTkPE2CV9zKK9zulTdqk+Aig9ch62SAmJOGkgejm5k9PE2XzQ==",
       "dependencies": {
         "@rollup/plugin-commonjs": "24.0.0",
-        "@sentry/core": "7.101.1",
-        "@sentry/integrations": "7.101.1",
-        "@sentry/node": "7.101.1",
-        "@sentry/react": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1",
-        "@sentry/vercel-edge": "7.101.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/integrations": "7.102.1",
+        "@sentry/node": "7.102.1",
+        "@sentry/react": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1",
+        "@sentry/vercel-edge": "7.102.1",
         "@sentry/webpack-plugin": "1.21.0",
         "chalk": "3.0.0",
         "resolve": "1.22.8",
@@ -5427,23 +5427,23 @@
       }
     },
     "node_modules/@sentry/node": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.101.1.tgz",
-      "integrity": "sha512-iXSxUT6Zbt/KUY0+fRcW5II6Tgp2zdTfhBW+fQuDt/UUZt7Ypvb+6n4U2oom3LJfttmD7mdjQuT4+vsNImDjTQ==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.102.1.tgz",
+      "integrity": "sha512-mb3vmM3SGuCruckPiv/Vafeh89UQavTfpPFoU6Jwe6dSpQ39BO8fO8k8Zev+/nP6r/FKLtX17mJobErHECXsYw==",
       "dependencies": {
-        "@sentry-internal/tracing": "7.101.1",
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry-internal/tracing": "7.102.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=8"
       }
     },
     "node_modules/@sentry/profiling-node": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/profiling-node/-/profiling-node-7.101.1.tgz",
-      "integrity": "sha512-buM+HZZW7jf3mqJgByVGPwytvmHcbjxRT8CkjO5UDO7stRTsq3iT/hK7qt0bQEDQwhmKeVCvfqryg2nKkPq3RA==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/profiling-node/-/profiling-node-7.102.1.tgz",
+      "integrity": "sha512-eqOUdu04eI4ODqeh/nHvC/mdwm3tWkqm02anR2ITEjKVJxHliHH6+jr+3M2X56e1hIxOibtL+JrR89Du9HEl9w==",
       "hasInstallScript": true,
       "dependencies": {
         "detect-libc": "^2.0.2",
@@ -5457,14 +5457,14 @@
       }
     },
     "node_modules/@sentry/react": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/react/-/react-7.101.1.tgz",
-      "integrity": "sha512-CwaBXntX2e3XHZQZVuv/tcfm5H+UHcS6aVChGfUiBHIBi2JpAqdnLdQIFGTkE8BSnKyolKgIsnvIU3BQ//QTig==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/react/-/react-7.102.1.tgz",
+      "integrity": "sha512-X4j2DgbktlEifnd21YJKCayAmff5hnaS+9MNz9OonEwD0ARi0ks7bo0wtWHMjPK20992MO+JwczVg/1BXJYDdQ==",
       "dependencies": {
-        "@sentry/browser": "7.101.1",
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1",
+        "@sentry/browser": "7.102.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1",
         "hoist-non-react-statics": "^3.3.2"
       },
       "engines": {
@@ -5475,47 +5475,47 @@
       }
     },
     "node_modules/@sentry/replay": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/replay/-/replay-7.101.1.tgz",
-      "integrity": "sha512-l4jmj2Rf/myzk3TA83PdMiomassG8okdBh1b2Hp1+ycBRVZFDmsR81gKPvnefSXwGwGNGKEmp6Q2bdGzekpp3Q==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/replay/-/replay-7.102.1.tgz",
+      "integrity": "sha512-HR/j9dGIvbrId8fh8mQlODx7JrhRmawEd9e9P3laPtogWCg/5TI+XPb2VGSaXOX9VWtb/6Z2UjHsaGjgg6YcuA==",
       "dependencies": {
-        "@sentry-internal/tracing": "7.101.1",
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry-internal/tracing": "7.102.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=12"
       }
     },
     "node_modules/@sentry/types": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.101.1.tgz",
-      "integrity": "sha512-bwtkQvrCZ6JGc7vqX7TEAKBgkbQFORt84FFS3JQQb8G3efTt9fZd2ReY4buteKQdlALl8h1QWVngTLmI+kyUuw==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.102.1.tgz",
+      "integrity": "sha512-htKorf3t/D0XYtM7foTcmG+rM47rDP6XdbvCcX5gBCuCYlzpM1vqCt2rl3FLktZC6TaIpFRJw1TLfx6m+x5jdA==",
       "engines": {
         "node": ">=8"
       }
     },
     "node_modules/@sentry/utils": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.101.1.tgz",
-      "integrity": "sha512-Nrg0nrEI3nrOCd9SLJ/WGzxS5KMQE4cryLOvrDcHJRWpsSyGBF1hLLerk84Nsw/0myMsn7zTYU+xoq7idNsX5A==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.102.1.tgz",
+      "integrity": "sha512-+8WcFjHVV/HROXSAwMuUzveElBFC43EiTG7SNEBNgOUeQzQVTmbUZXyTVgLrUmtoWqvnIxCacoLxtZo1o67kdg==",
       "dependencies": {
-        "@sentry/types": "7.101.1"
+        "@sentry/types": "7.102.1"
       },
       "engines": {
         "node": ">=8"
       }
     },
     "node_modules/@sentry/vercel-edge": {
-      "version": "7.101.1",
-      "resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-7.101.1.tgz",
-      "integrity": "sha512-xs6Xq910CR9rWHCYJEoEbfIfH60Oce+09NQTS5h3R/SOmrrAv4/mmZyblhunIS8TJqABeGRtEPKt1lOqO20O/A==",
+      "version": "7.102.1",
+      "resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-7.102.1.tgz",
+      "integrity": "sha512-iB6KCSxrvO172VjfQHGiYpyXPKNbx6Cz01GA1YDByRiUSgSpAq0qAFRY4lvd736w5KX4s9hen+XmZ7Gmqj8Pag==",
       "dependencies": {
-        "@sentry-internal/tracing": "7.101.1",
-        "@sentry/core": "7.101.1",
-        "@sentry/types": "7.101.1",
-        "@sentry/utils": "7.101.1"
+        "@sentry-internal/tracing": "7.102.1",
+        "@sentry/core": "7.102.1",
+        "@sentry/types": "7.102.1",
+        "@sentry/utils": "7.102.1"
       },
       "engines": {
         "node": ">=8"
@@ -16299,13 +16299,13 @@
       "dev": true
     },
     "node_modules/prisma": {
-      "version": "5.9.1",
-      "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.9.1.tgz",
-      "integrity": "sha512-Hy/8KJZz0ELtkw4FnG9MS9rNWlXcJhf98Z2QMqi0QiVMoS8PzsBkpla0/Y5hTlob8F3HeECYphBjqmBxrluUrQ==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.10.2.tgz",
+      "integrity": "sha512-hqb/JMz9/kymRE25pMWCxkdyhbnIWrq+h7S6WysJpdnCvhstbJSNP/S6mScEcqiB8Qv2F+0R3yG+osRaWqZacQ==",
       "devOptional": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@prisma/engines": "5.9.1"
+        "@prisma/engines": "5.10.2"
       },
       "bin": {
         "prisma": "build/index.js"
@@ -17237,9 +17237,9 @@
       }
     },
     "node_modules/release-it": {
-      "version": "17.0.5",
-      "resolved": "https://registry.npmjs.org/release-it/-/release-it-17.0.5.tgz",
-      "integrity": "sha512-97FcBu/2IjPd4qMvliZSnavpCO0+BWi8lZr+BgVJTXs6ihX8HLPucYR5RsgUUcZoGKj3beO8cunrsMuvEWTzGQ==",
+      "version": "17.1.1",
+      "resolved": "https://registry.npmjs.org/release-it/-/release-it-17.1.1.tgz",
+      "integrity": "sha512-b+4Tu2eb5f2wIdIe5E9hre0evbMQrXp/kRq0natHsHYJVqu1Bd4/h2a+swFi0faGmC3cJdB16uYR6LscG9SchQ==",
       "dev": true,
       "funding": [
         {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse-core",
-  "version": "2.7.2",
+  "version": "2.11.3",
   "private": true,
   "scripts": {
     "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
@@ -16,12 +16,15 @@
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "db:migrate": "DISABLE_ERD=false dotenv -e ../.env -- npx prisma migrate dev",
+    "db:migrate": "DISABLE_ERD=false dotenv -e ../.env -- npx prisma migrate dev && npm run db:worker",
     "db:reset": "dotenv -e ../.env npx -- prisma migrate reset",
+    "db:deploy": "dotenv -e ../.env npx -- prisma migrate deploy",
     "db:seed": "dotenv -e ../.env -- npx prisma db seed",
     "db:seed:examples": "dotenv -e ../.env -- npx prisma db seed -- --environment examples",
+    "db:seed:load": "dotenv -e ../.env -- npx prisma db seed -- --environment load",
     "release": "dotenv -e ../.env -- release-it",
-    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
+    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts",
+    "db:worker": "cp -r prisma/generated ../worker"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
@@ -29,13 +32,13 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.511.0",
+    "@aws-sdk/lib-storage": "^3.515.0",
     "@aws-sdk/s3-request-presigner": "^3.507.0",
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.1",
     "@hookform/resolvers": "^3.3.4",
     "@next-auth/prisma-adapter": "^1.0.7",
-    "@prisma/client": "^5.9.1",
+    "@prisma/client": "^5.10.2",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -58,8 +61,8 @@
     "@radix-ui/react-tooltip": "^1.0.7",
     "@react-email/components": "^0.0.14",
     "@react-email/render": "^0.0.12",
-    "@sentry/nextjs": "^7.101.1",
-    "@sentry/profiling-node": "^7.101.1",
+    "@sentry/nextjs": "^7.102.1",
+    "@sentry/profiling-node": "^7.102.1",
     "@sentry/types": "^7.88.0",
     "@t3-oss/env-nextjs": "^0.8.0",
     "@tailwindcss/forms": "^0.5.7",
@@ -134,9 +137,9 @@
     "postcss": "^8.4.35",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.11",
-    "prisma": "^5.9.1",
+    "prisma": "^5.10.2",
     "prisma-erd-generator": "^1.11.2",
-    "release-it": "^17.0.5",
+    "release-it": "^17.1.1",
     "tailwindcss": "^3.4.1",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",

web/prisma/generated/types.ts

@@ -24,6 +24,11 @@ export const ObservationLevel = {
     ERROR: "ERROR"
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
+export const ScoreSource = {
+    API: "API",
+    REVIEW: "REVIEW"
+} as const;
+export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
     PER_1000_TOKENS: "PER_1000_TOKENS",
     PER_1000_CHARS: "PER_1000_CHARS"
@@ -225,7 +230,7 @@ export type ObservationView = {
     calculated_input_cost: string | null;
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
-    latency: string | null;
+    latency: number | null;
 };
 export type Pricing = {
     id: string;
@@ -259,6 +264,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     name: string;
     value: number;
+    source: ScoreSource;
     comment: string | null;
     trace_id: string;
     observation_id: string | null;

web/prisma/migrations/20240226165118_add_observations_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_start_time_type_idx" ON "observations"("project_id", "start_time", "type");

web/prisma/migrations/20240226182815_add_model_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "models_project_id_model_name_start_date_unit_idx" ON "models"("project_id", "model_name", "start_date", "unit");

web/prisma/migrations/20240226183642_add_observations_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_internal_model_start_time_unit_idx" ON "observations"("project_id", "internal_model", "start_time", "unit");

web/prisma/migrations/20240226202040_add_observations_trace_id_project_id_start_time_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_trace_id_project_id_start_time_idx" ON "observations"("trace_id", "project_id", "start_time");

web/prisma/migrations/20240226202041_add_observations_trace_id_project_id_type_start_time_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_trace_id_project_id_type_start_time_idx" ON "observations"("trace_id", "project_id", "type", "start_time");

web/prisma/migrations/20240226203642_rewrite_observations_view/migration.sql

@@ -0,0 +1,50 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE

web/prisma/migrations/20240227112101_index_prompt_id_in_observations/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "observations_prompt_id_idx" ON "observations"("prompt_id");

web/prisma/migrations/20240228103642_observations_view_cte/migration.sql

@@ -0,0 +1,54 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+WITH model_ranked AS (
+    SELECT
+        *,
+        ROW_NUMBER() OVER (
+            PARTITION BY project_id, model_name, unit
+            ORDER BY project_id ASC, start_date DESC NULLS LAST
+        ) AS rn
+        -- adds a new column to the models table ordering the rows by project_id, model_name, and unit
+        -- each query should take the first row within a partition
+    FROM
+        models
+)
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
+FROM
+    observations o
+LEFT JOIN model_ranked m ON
+    m.rn = 1 AND
+    (m.project_id = o.project_id OR m.project_id IS NULL) AND
+    m.model_name = o.internal_model AND
+    (m.start_date < o.start_time OR m.start_date IS NULL) AND
+    o.unit::TEXT = m.unit;

web/prisma/migrations/20240228123642_observations_view_fix/migration.sql

@@ -0,0 +1,59 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

web/prisma/migrations/20240304123642_traces_view/migration.sql

@@ -0,0 +1,8 @@
+CREATE VIEW trace_metrics_view AS
+SELECT
+    t.id,
+    EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+FROM
+    traces t
+    LEFT JOIN observations o ON t.id = o.trace_id
+group by t.project_id, t.id

web/prisma/migrations/20240304123642_traces_view_improvement/migration.sql

@@ -0,0 +1,20 @@
+DROP VIEW trace_metrics_view;
+
+
+CREATE OR REPLACE VIEW traces_view AS
+WITH observations_metrics AS (
+    SELECT
+        trace_id,
+        project_id,
+        EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+    FROM
+        observations o
+    GROUP BY
+        project_id, trace_id
+)
+SELECT
+    t.*,
+    o.duration
+FROM
+    traces t
+    LEFT JOIN observations_metrics o ON t.id = o.trace_id and t.project_id = o.project_id

web/prisma/migrations/20240304222519_scores_add_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_timestamp_idx" ON "scores"("timestamp");

web/prisma/migrations/20240305095119_add_observations_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_trace_id_project_id_idx" ON "observations"("trace_id", "project_id");

web/prisma/migrations/20240305100713_traces_add_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "traces_id_user_id_idx" ON "traces"("id", "user_id");

web/prisma/migrations/20240307090110_claude_model_three/migration.sql

@@ -0,0 +1,18 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://docs.anthropic.com/claude/docs/models-overview, https://docs.anthropic.com/claude/docs/quickstart-guide
+  ('cltgy0iuw000008le3vod1hhy', NULL, 'claude-3-opus-20240229', '(?i)^(claude-3-opus-20240229)$', NULL, 0.000015, 0.000075, NULL, 'TOKENS', 'claude', NULL),
+  ('cltgy0pp6000108le56se7bl3', NULL, 'claude-3-sonnet-20240229', '(?i)^(claude-3-sonnet-20240229)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude', NULL)
+

web/prisma/migrations/20240307185543_score_add_source_nullable/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "ScoreSource" AS ENUM ('API', 'REVIEW');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "source" "ScoreSource" NOT NULL DEFAULT 'API';

web/prisma/migrations/20240307185544_score_add_name_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_source_idx" ON "scores"("source");

web/prisma/migrations/20240307185725_backfill_score_source/migration.sql

@@ -0,0 +1,4 @@
+-- scores previously created in the Langfuse UI all had the name 'manual-score'
+UPDATE "scores"
+SET "source" = 'REVIEW'::"ScoreSource"
+WHERE "name" = 'manual-score';

web/prisma/migrations/20240312195727_score_source_drop_default/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "source" DROP DEFAULT;

web/prisma/migrations/20240314090110_claude_model/migration.sql

@@ -0,0 +1,17 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://docs.anthropic.com/claude/docs/models-overview, https://docs.anthropic.com/claude/docs/quickstart-guide
+  ('cltr0w45b000008k1407o9qv1', NULL, 'claude-3-haiku-20240307', '(?i)^(claude-3-haiku-20240307)$', NULL, 0.00000025, 0.00000125, NULL, 'TOKENS', 'claude', NULL)
+

web/prisma/schema.prisma

@@ -216,6 +216,7 @@ model Trace {
     @@index([sessionId])
     @@index([name])
     @@index([userId])
+    @@index([id, userId])
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
@@ -261,6 +262,10 @@ model Observation {
     prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
 
     @@unique([id, projectId])
+    @@index([projectId, internalModel, startTime, unit])
+    @@index([traceId, projectId, type, startTime])
+    @@index([traceId, projectId, startTime])
+    @@index([traceId, projectId])
     @@index([traceId])
     @@index([type])
     @@index(startTime)
@@ -268,6 +273,8 @@ model Observation {
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
+    @@index(promptId)
+    @@index([projectId, startTime, type])
     @@map("observations")
 }
 
@@ -289,6 +296,7 @@ model Score {
     timestamp     DateTime     @default(now())
     name          String
     value         Float
+    source        ScoreSource
     comment       String?
     traceId       String       @map("trace_id")
     trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
@@ -296,12 +304,19 @@ model Score {
     observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
 
     @@unique([id, traceId]) // used for upsert
+    @@index(timestamp)
     @@index([value])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
+    @@index([source])
     @@map("scores")
 }
 
+enum ScoreSource {
+    API
+    REVIEW
+}
+
 enum PricingUnit {
     PER_1000_TOKENS
     PER_1000_CHARS
@@ -461,6 +476,7 @@ model Model {
     tokenizerConfig Json?     @map("tokenizer_config")
 
     @@unique([projectId, modelName, startDate, unit])
+    @@index([projectId, modelName, startDate, unit])
     @@index([projectId, modelName])
     @@map("models")
 }
@@ -495,7 +511,7 @@ view ObservationView {
 
     promptId String? @map("prompt_id")
 
-    // Model
+    // model fields
     modelId     String?  @map("model_id")
     inputPrice  Decimal? @map("input_price")
     outputPrice Decimal? @map("output_price")
@@ -505,7 +521,7 @@ view ObservationView {
     calculatedInputCost  Decimal? @map("calculated_input_cost")
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
-    latency              Decimal? @map("latency")
+    latency              Float?   @map("latency")
 
     @@map("observations_view")
 }

web/sentry.server.config.ts

@@ -2,7 +2,8 @@ import { prisma } from "@/src/server/db";
 import { jsonSchema } from "@/src/utils/zod";
 import * as Sentry from "@sentry/nextjs";
 import { ProfilingIntegration } from "@sentry/profiling-node";
-import type { TransactionEvent } from "@sentry/types";
+
+import type { SamplingContext, TransactionEvent } from "@sentry/types";
 
 if (process.env.NEXT_PUBLIC_SENTRY_DSN)
   Sentry.init({
@@ -12,9 +13,20 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.3,
+    tracesSampleRate: 1.0,
+    tracesSampler: (samplingContext: SamplingContext) => {
+      if (
+        samplingContext.request &&
+        samplingContext.request.method === "POST" &&
+        samplingContext.request.url &&
+        samplingContext.request.url.includes("api/trpc")
+      ) {
+        return 1.0;
+      }
+      return 0.2;
+    },
 
-    profilesSampleRate: 0.3, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

web/src/__tests__/cost.servertest.ts

@@ -320,6 +320,85 @@ describe("cost retrieval tests", () => {
     expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
   });
 
+  it(`should take old model for old observations`, async () => {
+    await prisma.model.create({
+      data: {
+        id: "model-0",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000000",
+        outputPrice: "0.0000000",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        startDate: null,
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000020",
+        outputPrice: "0.0000040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2023-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-0");
+    expect(view?.calculatedInputCost?.toString()).toBe("0");
+    expect(view?.calculatedOutputCost?.toString()).toBe("0");
+    expect(view?.calculatedTotalCost?.toString()).toBe("0");
+  });
+
   it(`should prioritize own models`, async () => {
     await pruneDatabase();
     await prisma.model.create({
@@ -385,8 +464,6 @@ describe("cost retrieval tests", () => {
   });
 
   it(`should prioritize old model if the latest model is not own one`, async () => {
-    await pruneDatabase();
-
     await prisma.model.create({
       data: {
         id: "model-1",

web/src/__tests__/generation-exports.servertest.ts

@@ -95,7 +95,7 @@ describe("observations.export RPC", () => {
       orderBy: { column: "id", order: "ASC" },
       filter: [
         {
-          column: "start_time",
+          column: "Start Time",
           type: "datetime",
           operator: ">",
           value: new Date("1990-01-01"),
@@ -123,7 +123,7 @@ describe("observations.export RPC", () => {
       orderBy: { column: "id", order: "ASC" },
       filter: [
         {
-          column: "start_time",
+          column: "Start Time",
           type: "datetime",
           operator: ">",
           value: new Date("1990-01-01"),
@@ -150,7 +150,7 @@ describe("observations.export RPC", () => {
       orderBy: { column: "id", order: "ASC" },
       filter: [
         {
-          column: "start_time",
+          column: "Start Time",
           type: "datetime",
           operator: ">",
           value: new Date("1990-01-01"),
@@ -179,7 +179,7 @@ describe("observations.export RPC", () => {
       orderBy: { column: "id", order: "ASC" },
       filter: [
         {
-          column: "start_time",
+          column: "Start Time",
           type: "datetime",
           operator: ">",
           value: new Date("1990-01-01"),

web/src/__tests__/ingestion.servertest.ts

@@ -768,6 +768,7 @@ describe("/api/public/ingestion API Endpoint", () => {
 
   it("should fail for wrong event formats", async () => {
     const traceId = v4();
+    const scoreId = v4();
 
     const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
       batch: [
@@ -787,13 +788,23 @@ describe("/api/public/ingestion API Endpoint", () => {
             version: "2.0.0",
           },
         },
+        {
+          id: v4(),
+          type: "score-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: scoreId,
+            name: "",
+            traceId: traceId,
+          },
+        },
       ],
     });
     expect(responseOne.status).toBe(207);
 
     expect("errors" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.errors.length).toBe(1);
+    expect(responseOne.body.errors.length).toBe(2);
     expect("successes" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
     expect(responseOne.body.successes.length).toBe(1);
@@ -1763,4 +1774,47 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
 
     expect(dbScore).toBeNull();
   });
+
+  it("should error on wrong input", async () => {
+    const traceId = "trace_id";
+    const bearerAuth = "Bearer pk-lf-1234567890";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    const response = await makeAPICall(
+      "POST",
+      "/api/public/ingestion",
+      {
+        // sending data instead of batch
+        data: [
+          {
+            id: scoreEventId,
+            type: "score-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: scoreId,
+              name: scoreName,
+              value: scoreValue,
+              traceId: traceId,
+            },
+          },
+        ],
+      },
+      bearerAuth,
+    );
+
+    expect(response.status).toBe(400);
+    expect(await prisma.trace.count()).toBe(0);
+
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+
+    expect(dbScore).toBeNull();
+  });
 });

web/src/__tests__/query-builder.servertest.ts

@@ -18,9 +18,23 @@ describe("Build valid SQL queries", () => {
         values: ["project-id"],
         strings: [' FROM  traces t  WHERE  t."project_id" = ', ";"],
       } as const,
+      {
+        table: "traces_metrics",
+        values: ["project-id"],
+        strings: [' FROM traces_view t  WHERE  t."project_id" = ', ";"],
+      } as const,
       {
         table: "traces_observations",
         values: ["project-id", "project-id"],
+        strings: [
+          ' FROM  traces t LEFT JOIN observations o ON t.id = o.trace_id  WHERE  t."project_id" = ',
+          ' AND o."project_id" = ',
+          ";",
+        ],
+      } as const,
+      {
+        table: "traces_observationsview",
+        values: ["project-id", "project-id"],
         strings: [
           ' FROM  traces t LEFT JOIN observations_view o ON t.id = o.trace_id  WHERE  t."project_id" = ',
           ' AND o."project_id" = ',
@@ -142,7 +156,7 @@ describe("Build valid SQL queries", () => {
           from: "traces",
           select: [{ column: "unknown" }],
         }),
-      ).toThrow("Column unknown not found");
+      ).toThrow('Column "unknown" not found in table traces');
     });
 
     it("should not group by an unknown column", () => {
@@ -152,7 +166,7 @@ describe("Build valid SQL queries", () => {
           groupBy: [{ column: "unknown", type: "string" }],
           select: [],
         }),
-      ).toThrow("Column unknown not found");
+      ).toThrow('Column "unknown" not found in table traces');
     });
 
     it("should not order by an unknown column", () => {
@@ -162,7 +176,7 @@ describe("Build valid SQL queries", () => {
           select: [],
           orderBy: [{ column: "unknown", direction: "ASC" }],
         }),
-      ).toThrow("Column unknown not found");
+      ).toThrow('Column "unknown" not found in table traces');
     });
   });
 

web/src/__tests__/scores.servertest.ts

@@ -54,6 +54,7 @@ describe("/api/public/scores API Endpoint", () => {
     expect(dbScore?.value).toBe(100.5);
     expect(dbScore?.observationId).toBeNull();
     expect(dbScore?.comment).toBe("comment");
+    expect(dbScore?.source).toBe("API");
   });
 
   it("should create score for a trace with int", async () => {
@@ -270,4 +271,49 @@ describe("/api/public/scores API Endpoint", () => {
     });
     expect(deletedScore).toBeNull();
   });
+
+  it("should GET a score", async () => {
+    await pruneDatabase();
+
+    const traceId = uuidv4();
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: traceId,
+    });
+    const generationId = uuidv4();
+    await makeAPICall("POST", "/api/public/generations", {
+      id: generationId,
+    });
+
+    const scoreId = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId,
+      observationId: generationId,
+      name: "score-name",
+      value: 100.5,
+      traceId: traceId,
+      comment: "comment",
+    });
+
+    const getScore = await makeAPICall<{
+      id: string;
+      name: string;
+      value: number;
+      comment: string;
+      traceId: string;
+      observationId: string;
+      source: string;
+    }>("GET", `/api/public/scores/${scoreId}`);
+
+    expect(getScore.status).toBe(200);
+    expect(getScore.body).toMatchObject({
+      id: scoreId,
+      name: "score-name",
+      value: 100.5,
+      comment: "comment",
+      source: "API",
+      traceId,
+      observationId: generationId,
+    });
+  });
 });

web/src/components/LangfuseLogo.tsx

@@ -13,7 +13,7 @@ export const LangfuseIcon = ({
 }) => (
   // eslint-disable-next-line @next/next/no-img-element
   <img
-    src="/icon256.png"
+    src="/icon.svg"
     width={size}
     height={size}
     alt="Langfuse Icon"
@@ -56,8 +56,8 @@ export const LangfuseLogo = ({
       <LangfuseIcon size={size === "sm" ? 16 : 20} />
       <span
         className={cn(
-          "font-mono font-semibold",
-          size === "sm" ? "ml-2 text-sm" : "ml-3 text-xl",
+          "ml-2 font-mono font-semibold",
+          size === "sm" ? "text-sm" : "text-xl",
         )}
       >
         Langfuse

web/src/components/date-picker.tsx

@@ -22,7 +22,6 @@ import {
 import { useEffect, useState } from "react";
 import {
   type DateTimeAggregationOption,
-  findClosestInterval,
   dateTimeAggregationSettings,
   dateTimeAggregationOptions,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
@@ -89,7 +88,6 @@ export function DatePicker({
 export type DatePickerWithRangeProps = {
   dateRange?: DashboardDateRange;
   className?: string;
-  setAgg: (agg: DateTimeAggregationOption) => void;
   selectedOption: AvailableDateRangeSelections;
   setDateRangeAndOption: (
     option: AvailableDateRangeSelections,
@@ -101,7 +99,6 @@ export function DatePickerWithRange({
   className,
   dateRange,
   selectedOption,
-  setAgg,
   setDateRangeAndOption,
 }: DatePickerWithRangeProps) {
   const [internalDateRange, setInternalDateRange] = useState<
@@ -112,12 +109,6 @@ export function DatePickerWithRange({
     setInternalDateRange(dateRange);
   }, [dateRange]);
 
-  const closestInterval = dateRange
-    ? findClosestInterval(dateRange)
-    : undefined;
-
-  closestInterval ? setAgg(closestInterval) : null;
-
   const onDropDownSelection = (value: string) => {
     if (isValidOption(value)) {
       const setting = dateTimeAggregationSettings[value];
@@ -171,8 +162,8 @@ export function DatePickerWithRange({
             {internalDateRange?.from ? (
               internalDateRange.to ? (
                 <>
-                  {format(internalDateRange.from, "LLL dd, yy : hh:mm")} -{" "}
-                  {format(internalDateRange.to, "LLL dd, yy : hh:mm")}
+                  {format(internalDateRange.from, "LLL dd, yy : HH:mm")} -{" "}
+                  {format(internalDateRange.to, "LLL dd, yy : HH:mm")}
                 </>
               ) : (
                 format(internalDateRange.from, "LLL dd, y")

web/src/components/delete-trace.tsx

@@ -1,84 +0,0 @@
-import { TrashIcon } from "lucide-react";
-import { useRouter } from "next/router";
-import { useState } from "react";
-import { Button } from "@/src/components/ui/button";
-import { api } from "@/src/utils/api";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/src/components/ui/popover";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { usePostHog } from "posthog-js/react";
-
-export function DeleteTrace({
-  traceId,
-  projectId,
-  isTableAction,
-}: {
-  traceId: string;
-  projectId: string;
-  isTableAction?: boolean;
-}) {
-  const [isDeleted, setIsDeleted] = useState(false);
-  const router = useRouter();
-  const utils = api.useUtils();
-  const posthog = usePostHog();
-
-  const hasAccess = useHasAccess({ projectId, scope: "traces:delete" });
-
-  const mutDeleteTraces = api.traces.deleteMany.useMutation({
-    onSuccess: () => {
-      setIsDeleted(true);
-      void utils.traces.invalidate();
-      if (!isTableAction) {
-        void router.push(`/project/${projectId}/traces`);
-      }
-    },
-  });
-
-  if (!hasAccess) {
-    return null;
-  }
-
-  return (
-    <Popover key={traceId}>
-      <PopoverTrigger asChild>
-        {isTableAction ? (
-          <Button variant="ghost" size="xs">
-            <TrashIcon className="h-4 w-4" />
-          </Button>
-        ) : (
-          <Button variant="outline" type="button" size="icon">
-            <TrashIcon className="h-4 w-4" />
-          </Button>
-        )}
-      </PopoverTrigger>
-      <PopoverContent>
-        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
-        <p className="mb-3 text-sm">
-          This action cannot be undone and removes all the data associated with
-          this trace.
-        </p>
-        <div className="flex justify-end space-x-4">
-          <Button
-            type="button"
-            variant="destructive"
-            loading={mutDeleteTraces.isLoading || isDeleted}
-            onClick={() => {
-              void mutDeleteTraces.mutateAsync({
-                traceIds: [traceId],
-                projectId,
-              });
-              posthog.capture("trace:delete", {
-                source: isTableAction ? "table-single-row" : "trace",
-              });
-            }}
-          >
-            Delete trace
-          </Button>
-        </div>
-      </PopoverContent>
-    </Popover>
-  );
-}

web/src/components/deleteButton.tsx

@@ -0,0 +1,116 @@
+import { useState } from "react";
+import { useRouter } from "next/router";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { Button } from "@/src/components/ui/button";
+import { TrashIcon } from "lucide-react";
+import { usePostHog } from "posthog-js/react";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { type Scope } from "@/src/features/rbac/constants/roleAccessRights";
+import { api } from "@/src/utils/api";
+
+interface DeleteButtonProps {
+  itemId: string;
+  projectId: string;
+  isTableAction?: boolean;
+  scope: Scope;
+  invalidateFunc: () => void;
+  type: "trace" | "dataset";
+  redirectUrl?: string;
+}
+
+export function DeleteButton({
+  itemId,
+  projectId,
+  isTableAction = false,
+  scope,
+  invalidateFunc,
+  type,
+  redirectUrl,
+}: DeleteButtonProps) {
+  const [isDeleted, setIsDeleted] = useState(false);
+  const router = useRouter();
+  const posthog = usePostHog();
+
+  const hasAccess = useHasAccess({ projectId, scope: scope });
+  const traceMutation = api.traces.deleteMany.useMutation({
+    onSuccess: () => {
+      setIsDeleted(true);
+      !isTableAction && redirectUrl
+        ? void router.push(redirectUrl)
+        : invalidateFunc();
+    },
+  });
+  const datasetMutation = api.datasets.deleteDataset.useMutation({
+    onSuccess: () => {
+      setIsDeleted(true);
+      !isTableAction && redirectUrl
+        ? void router.push(redirectUrl)
+        : invalidateFunc();
+    },
+  });
+
+  if (!hasAccess) {
+    return null;
+  }
+
+  return (
+    <Popover key={itemId}>
+      <PopoverTrigger asChild>
+        <Button
+          variant={isTableAction ? "ghost" : "outline"}
+          size={isTableAction ? "xs" : "icon"}
+        >
+          <TrashIcon className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action cannot be undone and removes all the data associated with
+          this {type}.
+        </p>
+        <div className="flex justify-end space-x-4">
+          {type === "trace" ? (
+            <Button
+              type="button"
+              variant="destructive"
+              loading={traceMutation.isLoading || isDeleted}
+              onClick={() => {
+                void traceMutation.mutateAsync({
+                  traceIds: [itemId],
+                  projectId,
+                });
+                posthog.capture("trace:delete", {
+                  source: isTableAction ? "table-single-row" : "trace",
+                });
+              }}
+            >
+              Delete trace
+            </Button>
+          ) : (
+            <Button
+              type="button"
+              variant="destructive"
+              loading={datasetMutation.isLoading || isDeleted}
+              onClick={() => {
+                void datasetMutation.mutateAsync({
+                  projectId,
+                  datasetId: itemId,
+                });
+                posthog.capture("dataset:delete", {
+                  source: isTableAction ? "table-single-row" : "dataset",
+                });
+              }}
+            >
+              Delete dataset
+            </Button>
+          )}
+        </div>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/components/grouped-score-badge.tsx

@@ -4,15 +4,10 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
 import { cn } from "@/src/utils/tailwind";
 import { MessageCircle } from "lucide-react";
 
-type ScoreSimplified = {
-  name: string;
-  value: number;
-  comment?: string | null;
-};
-
 export const GroupedScoreBadges = ({
   scores,
   variant = "badge",

web/src/components/layouts/doc-popup.tsx

@@ -75,3 +75,27 @@ export default function DocPopup({
     </HoverCard>
   );
 }
+
+export type PopupProps = {
+  triggerContent: React.ReactNode;
+  description: React.ReactNode;
+};
+
+export function Popup({ triggerContent, description }: PopupProps) {
+  return (
+    <HoverCard openDelay={200}>
+      <HoverCardTrigger className="mx-1 cursor-pointer" asChild>
+        <div>{triggerContent}</div>
+      </HoverCardTrigger>
+      <HoverCardContent>
+        {typeof description === "string" ? (
+          <div className="whitespace-break-spaces text-xs font-normal text-gray-800 sm:pl-0">
+            {description}
+          </div>
+        ) : (
+          description
+        )}
+      </HoverCardContent>
+    </HoverCard>
+  );
+}

web/src/components/layouts/routes.ts

@@ -62,7 +62,6 @@ export const ROUTES: Route[] = [
     name: "Prompts",
     pathname: "/project/[projectId]/prompts",
     icon: PenSquareIcon,
-    label: "Beta",
     rbacScope: "prompts:read",
   },
   {

web/src/components/table/data-table-pagination.tsx

@@ -17,10 +17,12 @@ import {
 
 interface DataTablePaginationProps<TData> {
   table: Table<TData>;
+  paginationOptions?: number[];
 }
 
 export function DataTablePagination<TData>({
   table,
+  paginationOptions = [10, 20, 30, 40, 50],
 }: DataTablePaginationProps<TData>) {
   return (
     <div className="mt-3 flex items-center justify-between overflow-x-auto px-2">
@@ -41,7 +43,7 @@ export function DataTablePagination<TData>({
               <SelectValue placeholder={table.getState().pagination.pageSize} />
             </SelectTrigger>
             <SelectContent side="top">
-              {[10, 20, 30, 40, 50].map((pageSize) => (
+              {paginationOptions.map((pageSize) => (
                 <SelectItem key={pageSize} value={`${pageSize}`}>
                   {pageSize}
                 </SelectItem>

web/src/components/table/data-table.tsx

@@ -34,6 +34,7 @@ interface DataTableProps<TData, TValue> {
     pageCount: number;
     onChange: OnChangeFn<PaginationState>;
     state: PaginationState;
+    options?: number[];
   };
   rowSelection?: RowSelectionState;
   setRowSelection?: OnChangeFn<RowSelectionState>;
@@ -215,7 +216,12 @@ export function DataTable<TData extends object, TValue>({
           </Table>
         </div>
       </div>
-      {pagination !== undefined ? <DataTablePagination table={table} /> : null}
+      {pagination !== undefined ? (
+        <DataTablePagination
+          table={table}
+          paginationOptions={pagination.options}
+        />
+      ) : null}
     </>
   );
 }

web/src/components/table/use-cases/generations.tsx

@@ -23,40 +23,55 @@ import {
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
-import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
+import {
+  formatIntervalSeconds,
+  intervalInSeconds,
+  utcDateOffsetByDays,
+} from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { JSONView } from "@/src/components/ui/code";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type Score, type ObservationLevel } from "@prisma/client";
+import { type ObservationLevel } from "@prisma/client";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
-import { usdFormatter } from "@/src/utils/numbers";
+import { randomIntFromInterval, usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type ExportFileFormats,
 } from "@/src/server/api/interfaces/exportTypes";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
+import type Decimal from "decimal.js";
+import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import React from "react";
 
 export type GenerationsTableRow = {
   id: string;
-  traceId: string;
-  startTime: string;
+  traceId?: string;
+  startTime: Date;
   level?: ObservationLevel;
   statusMessage?: string;
   endTime?: string;
+  completionStartTime?: Date;
   latency?: number;
   name?: string;
   model?: string;
   input?: unknown;
   output?: unknown;
+  inputCost?: Decimal;
+  outputCost?: Decimal;
+  totalCost?: Decimal;
   traceName?: string;
   metadata?: string;
-  scores: Score[];
+  scores?: ScoreSimplified[];
   usage: {
     promptTokens: number;
     completionTokens: number;
     totalTokens: number;
   };
+  promptId?: string;
+  promptName?: string;
+  promptVersion?: string;
 };
 
 export type GenerationsTableProps = {
@@ -78,7 +93,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
 
   const [filterState, setFilterState] = useQueryFilterState([
     {
-      column: "start_time",
+      column: "Start Time",
       type: "datetime",
       operator: ">",
       value: utcDateOffsetByDays(-14),
@@ -119,6 +134,10 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
 
     setIsExporting(true);
     posthog.capture("generations:export", { file_format: fileFormat });
+    if (fileFormat === "OPENAI-JSONL")
+      alert(
+        "When exporting in OpenAI-JSONL, only generations that exactly match the `ChatML` format will be exported. For any questions, reach out to support.",
+      );
     try {
       const fileData = await directApi.generations.export.query({
         projectId,
@@ -181,7 +200,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
     {
       accessorKey: "name",
       id: "name",
-      header: "name",
+      header: "Name",
       enableSorting: true,
     },
     {
@@ -212,14 +231,52 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       header: "Start Time",
       enableHiding: true,
       enableSorting: true,
+      cell: ({ row }) => {
+        const value: Date = row.getValue("startTime");
+        return value.toLocaleString();
+      },
+    },
+    {
+      accessorKey: "endTime",
+      id: "endTime",
+      header: "End Time",
+      enableHiding: true,
+      enableSorting: true,
+    },
+    {
+      accessorKey: "timeToFirstToken",
+      id: "timeToFirstToken",
+      header: "Time to First Token",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const startTime: Date = row.getValue("startTime");
+        const completionStartTime: Date | undefined =
+          row.getValue("timeToFirstToken");
+
+        if (!completionStartTime) {
+          return undefined;
+        }
+
+        const latencyInSeconds =
+          intervalInSeconds(startTime, completionStartTime) || "-";
+        return (
+          <span>
+            {typeof latencyInSeconds === "number"
+              ? formatIntervalSeconds(latencyInSeconds)
+              : latencyInSeconds}
+          </span>
+        );
+      },
     },
     {
       accessorKey: "scores",
       id: "scores",
       header: "Scores",
       cell: ({ row }) => {
-        const values: Score[] = row.getValue("scores");
-        return <GroupedScoreBadges scores={values} variant="headings" />;
+        const values: ScoreSimplified[] | undefined = row.getValue("scores");
+        return (
+          values && <GroupedScoreBadges scores={values} variant="headings" />
+        );
       },
       enableHiding: true,
     },
@@ -228,22 +285,71 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       id: "latency",
       header: "Latency",
       cell: ({ row }) => {
-        const value: number | undefined = row.getValue("latency");
-        return value !== undefined ? (
-          <span>{formatInterval(value)}</span>
+        const latency: number | undefined = row.getValue("latency");
+        return latency !== undefined ? (
+          <span>{formatIntervalSeconds(latency)}</span>
         ) : undefined;
       },
       enableHiding: true,
       enableSorting: true,
     },
     {
-      accessorKey: "cost",
-      header: "Cost",
+      accessorKey: "timePerOutputToken",
+      id: "timePerOutputToken",
+      header: "Time per Output Token",
       cell: ({ row }) => {
-        const value: number | undefined = row.getValue("cost");
+        const latency: number | undefined = row.getValue("latency");
+        const usage: {
+          promptTokens: number;
+          completionTokens: number;
+          totalTokens: number;
+        } = row.getValue("usage");
+        return latency !== undefined &&
+          (usage.completionTokens !== 0 || usage.totalTokens !== 0) ? (
+          <span>
+            {usage.completionTokens
+              ? formatIntervalSeconds(latency / usage.completionTokens)
+              : formatIntervalSeconds(latency / usage.totalTokens)}
+          </span>
+        ) : undefined;
+      },
+      defaultHidden: true,
+      enableHiding: true,
+    },
+    {
+      accessorKey: "inputCost",
+      header: "Input Cost",
+      cell: ({ row }) => {
+        const value: Decimal | undefined = row.getValue("inputCost");
 
         return value !== undefined ? (
-          <span>{usdFormatter(value)}</span>
+          <span>{usdFormatter(value.toNumber())}</span>
+        ) : undefined;
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "outputCost",
+      header: "Output Cost",
+      cell: ({ row }) => {
+        const value: Decimal | undefined = row.getValue("outputCost");
+
+        return value !== undefined ? (
+          <span>{usdFormatter(value.toNumber())}</span>
+        ) : undefined;
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "totalCost",
+      header: "Total Cost",
+      cell: ({ row }) => {
+        const value: Decimal | undefined = row.getValue("totalCost");
+
+        return value !== undefined ? (
+          <span>{usdFormatter(value.toNumber())}</span>
         ) : undefined;
       },
       enableHiding: true,
@@ -306,8 +412,11 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       accessorKey: "input",
       header: "Input",
       cell: ({ row }) => {
-        const value: unknown = row.getValue("input");
-        return <JSONView json={value} className="w-[500px]" />;
+        const observationId: string = row.getValue("id");
+        const traceId: string = row.getValue("traceId");
+        return (
+          <IOCell observationId={observationId} traceId={traceId} io="input" />
+        );
       },
       enableHiding: true,
       defaultHidden: true,
@@ -316,8 +425,11 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       accessorKey: "output",
       header: "Output",
       cell: ({ row }) => {
-        const value: unknown = row.getValue("output");
-        return <JSONView json={value} className="w-[500px] bg-green-50" />;
+        const observationId: string = row.getValue("id");
+        const traceId: string = row.getValue("traceId");
+        return (
+          <IOCell observationId={observationId} traceId={traceId} io="output" />
+        );
       },
       enableHiding: true,
       defaultHidden: true,
@@ -339,23 +451,60 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "prompt",
+      id: "prompt",
+      header: "Prompt",
+      enableHiding: true,
+      enableSorting: false,
+      cell: ({ row }) => {
+        const promptName = row.original.promptName;
+        const promptVersion = row.original.promptVersion;
+        const value = `${promptName} (v${promptVersion})`;
+        return (
+          promptName &&
+          promptVersion && (
+            <TableLink
+              path={`/project/${projectId}/prompts/${encodeURIComponent(promptName)}?version=${promptVersion}`}
+              value={value}
+              truncateAt={40}
+            />
+          )
+        );
+      },
+    },
   ];
-  const [columnVisibility, setColumnVisibility] =
+  const [columnVisibility, setColumnVisibilityState] =
     useColumnVisibility<GenerationsTableRow>(
       "generationsColumnVisibility",
       columns,
     );
 
+  const smallTableRequired =
+    columnVisibility["input"] === true || columnVisibility["output"] === true;
+
+  if (smallTableRequired && paginationState.pageSize !== 10) {
+    setPaginationState((prev) => {
+      const currentPage = prev.pageIndex;
+      const currentPageSize = prev.pageSize;
+      const newPageIndex = Math.floor((currentPage * currentPageSize) / 10);
+      return { pageIndex: newPageIndex, pageSize: 10 };
+    });
+  }
+
   const rows: GenerationsTableRow[] = generations.isSuccess
     ? generations.data.generations.map((generation) => {
         return {
           id: generation.id,
-          traceId: generation.traceId,
-          traceName: generation.traceName,
-          startTime: generation.startTime.toLocaleString(),
+          traceId: generation.traceId ?? undefined,
+          traceName: generation.traceName ?? "",
+          startTime: generation.startTime,
           endTime: generation.endTime?.toLocaleString() ?? undefined,
-          latency: generation.latency === null ? undefined : generation.latency,
-          cost: generation.calculatedTotalCost,
+          timeToFirstToken: generation.completionStartTime ?? undefined,
+          latency: generation.latency ?? undefined,
+          totalCost: generation.calculatedTotalCost ?? undefined,
+          inputCost: generation.calculatedInputCost ?? undefined,
+          outputCost: generation.calculatedOutputCost ?? undefined,
           name: generation.name ?? undefined,
           version: generation.version ?? "",
           model: generation.model ?? "",
@@ -372,6 +521,9 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
             completionTokens: generation.completionTokens,
             totalTokens: generation.totalTokens,
           },
+          promptId: generation.promptId ?? undefined,
+          promptName: generation.promptName ?? undefined,
+          promptVersion: generation.promptVersion ?? undefined,
         };
       })
     : [];
@@ -391,7 +543,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
           currentQuery: searchQuery ?? undefined,
         }}
         columnVisibility={columnVisibility}
-        setColumnVisibility={setColumnVisibility}
+        setColumnVisibility={setColumnVisibilityState}
         actionButtons={
           <DropdownMenu>
             <DropdownMenuTrigger asChild>
@@ -445,12 +597,84 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
           pageCount: Math.ceil(totalCount / paginationState.pageSize),
           onChange: setPaginationState,
           state: paginationState,
+          // enforce a minimum page size of 10 if input or output columns are visible
+          options: smallTableRequired ? [10] : undefined,
         }}
         setOrderBy={setOrderByState}
         orderBy={orderByState}
         columnVisibility={columnVisibility}
-        onColumnVisibilityChange={setColumnVisibility}
+        onColumnVisibilityChange={setColumnVisibilityState}
       />
     </div>
   );
 }
+
+const IOCell = ({
+  traceId,
+  observationId,
+  io,
+}: {
+  traceId: string;
+  observationId: string;
+  io: "input" | "output";
+}) => {
+  const observation = api.observations.byId.useQuery(
+    {
+      observationId: observationId,
+      traceId: traceId,
+    },
+    {
+      enabled: typeof traceId === "string" && typeof observationId === "string",
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
+  return (
+    <>
+      {observation.isLoading || !observation.data ? (
+        <JsonSkeleton className="h-[250px] w-[500px] px-3 py-1" />
+      ) : (
+        <JSONView
+          json={
+            io === "output" ? observation.data.output : observation.data.input
+          }
+          className="h-[250px] w-[500px] overflow-y-auto"
+        />
+      )}
+    </>
+  );
+};
+
+export const JsonSkeleton = ({
+  className,
+  numRows = 10,
+}: {
+  numRows?: number;
+  className?: string;
+}) => {
+  const sizingOptions = [
+    "h-5 w-full",
+    "h-5 w-[400px]",
+    "h-5 w-[450px]",
+    "h-5 w-[475px]",
+  ];
+
+  const generateRandomSize = () =>
+    sizingOptions[randomIntFromInterval(0, sizingOptions.length - 1)];
+
+  return (
+    <div className={cn("w-[500px] rounded-md border", className)}>
+      <div className="flex flex-col gap-1">
+        {[...Array<number>(numRows)].map((_) => (
+          <>
+            <Skeleton className={generateRandomSize()} />
+          </>
+        ))}
+        <br />
+      </div>
+    </div>
+  );
+};

web/src/components/table/use-cases/scores.tsx

@@ -62,7 +62,7 @@ export default function ScoresTable({
     filter: filterState,
     orderBy: orderByState,
   });
-  const totalCount = scores.data?.slice(1)[0]?.totalCount ?? 0;
+  const totalCount = scores.data?.totalCount ?? 0;
 
   const filterOptions = api.scores.filterOptions.useQuery({
     projectId,
@@ -161,7 +161,7 @@ export default function ScoresTable({
               : {
                   isLoading: false,
                   isError: false,
-                  data: scores.data.map((t) => convertToTableRow(t)),
+                  data: scores.data.scores.map((t) => convertToTableRow(t)),
                 }
         }
         pagination={{

web/src/components/table/use-cases/sessions.tsx

@@ -3,6 +3,7 @@ import { DataTable } from "@/src/components/table/data-table";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { type FilterState } from "@/src/features/filters/types";
@@ -10,9 +11,10 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsViewCols } from "@/src/server/api/definitions/sessionsView";
 import { api } from "@/src/utils/api";
-import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
+import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import { usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
+import type Decimal from "decimal.js";
 import { useEffect } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
 
@@ -23,7 +25,12 @@ export type SessionTableRow = {
   countTraces: number;
   bookmarked: boolean;
   sessionDuration: number | null;
-  totalCost: number;
+  inputCost: Decimal;
+  outputCost: Decimal;
+  totalCost: Decimal;
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
 };
 
 export type SessionTableProps = {
@@ -100,7 +107,12 @@ export default function SessionsTable({
       countTraces: session.countTraces,
       bookmarked: session.bookmarked,
       sessionDuration: session.sessionDuration,
+      inputCost: session.inputCost,
+      outputCost: session.outputCost,
       totalCost: session.totalCost,
+      inputTokens: session.promptTokens,
+      outputTokens: session.completionTokens,
+      totalTokens: session.totalTokens,
     };
   };
 
@@ -156,7 +168,7 @@ export default function SessionsTable({
       cell: ({ row }) => {
         const value = row.getValue("sessionDuration");
         return value && typeof value === "number"
-          ? formatInterval(value)
+          ? formatIntervalSeconds(value)
           : undefined;
       },
       enableSorting: true,
@@ -189,6 +201,33 @@ export default function SessionsTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "inputCost",
+      id: "inputCost",
+      header: "Input Cost",
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const value: Decimal | null | undefined = row.getValue("inputCost");
+        return value ? (
+          <span>{usdFormatter(value.toNumber())}</span>
+        ) : undefined;
+      },
+    },
+    {
+      accessorKey: "outputCost",
+      id: "outputCost",
+      header: "Output Cost",
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const value: Decimal | null | undefined = row.getValue("outputCost");
+
+        return value ? (
+          <span>{usdFormatter(value.toNumber())}</span>
+        ) : undefined;
+      },
+    },
     {
       accessorKey: "totalCost",
       id: "totalCost",
@@ -196,13 +235,67 @@ export default function SessionsTable({
       enableHiding: true,
       enableSorting: true,
       cell: ({ row }) => {
-        const value: number | undefined = row.getValue("totalCost");
+        const value: Decimal | null | undefined = row.getValue("totalCost");
 
-        return value !== undefined ? (
-          <span>{usdFormatter(value, 2, 2)}</span>
+        return value ? (
+          <span>{usdFormatter(value.toNumber())}</span>
         ) : undefined;
       },
     },
+    {
+      accessorKey: "inputTokens",
+      id: "inputTokens",
+      header: "Input Tokens",
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const value: number | undefined = row.getValue("inputTokens");
+
+        return value ? <span>{Number(value)}</span> : undefined;
+      },
+    },
+    {
+      accessorKey: "outputTokens",
+      id: "outputTokens",
+      header: "Output Tokens",
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const value = row.getValue("outputTokens");
+
+        return value ? <span>{Number(value)}</span> : undefined;
+      },
+    },
+    {
+      accessorKey: "totalTokens",
+      id: "totalTokens",
+      header: "Total Tokens",
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const value = row.getValue("totalTokens");
+        return value ? <span>{Number(value)}</span> : undefined;
+      },
+    },
+    {
+      accessorKey: "usage",
+      id: "usage",
+      header: "Usage",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const promptTokens = row.getValue("inputTokens");
+        const completionTokens = row.getValue("outputTokens");
+        const totalTokens = row.getValue("totalTokens");
+        return (
+          <TokenUsageBadge
+            promptTokens={Number(promptTokens)}
+            completionTokens={Number(completionTokens)}
+            totalTokens={Number(totalTokens)}
+            inline
+          />
+        );
+      },
+    },
   ];
 
   const [columnVisibility, setColumnVisibility] =

web/src/components/table/use-cases/traces.tsx

@@ -1,4 +1,3 @@
-import { DeleteTrace } from "@/src/components/delete-trace";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { StarTraceToggle } from "@/src/components/star-toggle";
 import { DataTable } from "@/src/components/table/data-table";
@@ -17,9 +16,9 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { tracesTableColsWithOptions } from "@/src/server/api/definitions/tracesTable";
 import { api } from "@/src/utils/api";
-import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
+import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
-import { type Score } from "@prisma/client";
+import { type ObservationLevel, type Score } from "@prisma/client";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
 import {
@@ -31,6 +30,9 @@ import {
 } from "use-query-params";
 import type Decimal from "decimal.js";
 import { usdFormatter } from "@/src/utils/numbers";
+import { DeleteButton } from "@/src/components/deleteButton";
+import { LevelColors } from "@/src/components/level-colors";
+import { cn } from "@/src/utils/tailwind";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -39,6 +41,7 @@ export type TracesTableRow = {
   name: string;
   userId: string;
   metadata?: string;
+  level: ObservationLevel;
   latency?: number;
   release?: string;
   version?: string;
@@ -52,7 +55,9 @@ export type TracesTableRow = {
     completionTokens: number;
     totalTokens: number;
   };
-  cost?: Decimal;
+  inputCost?: Decimal;
+  outputCost?: Decimal;
+  totalCost?: Decimal;
 };
 
 export type TracesTableProps = {
@@ -68,6 +73,7 @@ export default function TracesTable({
   userId,
   omittedFilter = [],
 }: TracesTableProps) {
+  const utils = api.useUtils();
   const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
   const { setDetailPageList } = useDetailPageLists();
   const [searchQuery, setSearchQuery] = useQueryParam(
@@ -147,6 +153,7 @@ export default function TracesTable({
       id: trace.id,
       timestamp: trace.timestamp.toLocaleString(),
       name: trace.name ?? "",
+      level: trace.level,
       metadata: JSON.stringify(trace.metadata),
       release: trace.release ?? undefined,
       version: trace.version ?? undefined,
@@ -162,7 +169,9 @@ export default function TracesTable({
         completionTokens: trace.completionTokens,
         totalTokens: trace.totalTokens,
       },
-      cost: trace.calculatedTotalCost ?? undefined,
+      inputCost: trace.calculatedInputCost ?? undefined,
+      outputCost: trace.calculatedOutputCost ?? undefined,
+      totalCost: trace.calculatedTotalCost ?? undefined,
     };
   };
 
@@ -285,11 +294,56 @@ export default function TracesTable({
       // add seconds to the end of the latency
       cell: ({ row }) => {
         const value: number | undefined = row.getValue("latency");
-        return value !== undefined ? formatInterval(value) : undefined;
+        return value !== undefined ? formatIntervalSeconds(value) : undefined;
       },
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "inputTokens",
+      id: "inputTokens",
+      header: "Input Tokens",
+      cell: ({ row }) => {
+        const value: {
+          promptTokens: number;
+          completionTokens: number;
+          totalTokens: number;
+        } = row.getValue("usage");
+        return <span>{value.promptTokens}</span>;
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "outputTokens",
+      id: "outputTokens",
+      header: "Output Tokens",
+      cell: ({ row }) => {
+        const value: {
+          promptTokens: number;
+          completionTokens: number;
+          totalTokens: number;
+        } = row.getValue("usage");
+        return <span>{value.completionTokens}</span>;
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "totalTokens",
+      id: "totalTokens",
+      header: "Total Tokens",
+      cell: ({ row }) => {
+        const value: {
+          promptTokens: number;
+          completionTokens: number;
+          totalTokens: number;
+        } = row.getValue("usage");
+        return <span>{value.totalTokens}</span>;
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
     {
       // TODO: Enable Ordering By Usage (not covered by API yet)
       accessorKey: "usage",
@@ -312,17 +366,55 @@ export default function TracesTable({
       enableHiding: true,
     },
     {
-      accessorKey: "cost",
-      id: "cost",
-      header: "Cost",
+      accessorKey: "inputCost",
+      id: "inputCost",
+      header: "Input Cost",
       cell: ({ row }) => {
-        const cost: Decimal | undefined = row.getValue("cost");
+        const cost: Decimal | undefined = row.getValue("inputCost");
         return (
           <div>
             {cost ? (
               <span>{usdFormatter(cost.toNumber())}</span>
             ) : (
-              <span>Not Available</span>
+              <span>-</span>
+            )}
+          </div>
+        );
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "outputCost",
+      id: "outputCost",
+      header: "Output Cost",
+      cell: ({ row }) => {
+        const cost: Decimal | undefined = row.getValue("outputCost");
+        return (
+          <div>
+            {cost ? (
+              <span>{usdFormatter(cost.toNumber())}</span>
+            ) : (
+              <span>-</span>
+            )}
+          </div>
+        );
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
+    {
+      accessorKey: "totalCost",
+      id: "totalCost",
+      header: "Total Cost",
+      cell: ({ row }) => {
+        const cost: Decimal | undefined = row.getValue("totalCost");
+        return (
+          <div>
+            {cost ? (
+              <span>{usdFormatter(cost.toNumber())}</span>
+            ) : (
+              <span>-</span>
             )}
           </div>
         );
@@ -370,6 +462,26 @@ export default function TracesTable({
       },
       enableHiding: true,
     },
+    {
+      accessorKey: "level",
+      header: "Level",
+      cell: ({ row }) => {
+        const value: ObservationLevel = row.getValue("level");
+        return (
+          <span
+            className={cn(
+              "rounded-sm p-0.5 text-xs",
+              LevelColors[value].bg,
+              LevelColors[value].text,
+            )}
+          >
+            {value}
+          </span>
+        );
+      },
+      enableHiding: true,
+      defaultHidden: true,
+    },
     {
       accessorKey: "version",
       id: "version",
@@ -411,10 +523,13 @@ export default function TracesTable({
       cell: ({ row }) => {
         const traceId = row.getValue("id");
         return traceId && typeof traceId === "string" ? (
-          <DeleteTrace
-            traceId={traceId}
-            isTableAction={true}
+          <DeleteButton
+            itemId={traceId}
             projectId={projectId}
+            scope="traces:delete"
+            invalidateFunc={() => void utils.traces.invalidate()}
+            type="trace"
+            isTableAction={true}
           />
         ) : undefined;
       },

web/src/components/trace/IOPreview.tsx

@@ -30,38 +30,34 @@ export const IOPreview: React.FC<{
     ? outLegacyCompletionSchemaParsed.data
     : props.output ?? null;
 
-  // OpenAI messages
-  let inOpenAiMessageArray = OpenAiMessageArraySchema.safeParse(input);
-  if (!inOpenAiMessageArray.success) {
-    // check if input is an array of length 1 including an array of OpenAiMessageSchema
+  // ChatML format
+  let inChatMlArray = ChatMlArraySchema.safeParse(input);
+  if (!inChatMlArray.success) {
+    // check if input is an array of length 1 including an array of ChatMlMessageSchema
     // this is the case for some integrations
-    // e.g. [[OpenAiMessageSchema, ...]]
-    const inputArray = z.array(OpenAiMessageArraySchema).safeParse(input);
+    // e.g. [[ChatMlMessageSchema, ...]]
+    const inputArray = z.array(ChatMlArraySchema).safeParse(input);
     if (inputArray.success && inputArray.data.length === 1) {
-      inOpenAiMessageArray = OpenAiMessageArraySchema.safeParse(
-        inputArray.data[0],
-      );
+      inChatMlArray = ChatMlArraySchema.safeParse(inputArray.data[0]);
     } else {
       // check if input is an object with a messages key
       // this is the case for some integrations
-      // e.g. { messages: [OpenAiMessageSchema, ...] }
+      // e.g. { messages: [ChatMlMessageSchema, ...] }
       const inputObject = z
         .object({
-          messages: OpenAiMessageArraySchema,
+          messages: ChatMlArraySchema,
         })
         .safeParse(input);
 
       if (inputObject.success) {
-        inOpenAiMessageArray = OpenAiMessageArraySchema.safeParse(
-          inputObject.data.messages,
-        );
+        inChatMlArray = ChatMlArraySchema.safeParse(inputObject.data.messages);
       }
     }
   }
-  const outOpenAiMessage = OpenAiMessageSchema.safeParse(output);
+  const outChatMlMessage = ChatMlMessageSchema.safeParse(output);
 
   // Pretty view available
-  const isPrettyViewAvailable = inOpenAiMessageArray.success;
+  const isPrettyViewAvailable = inChatMlArray.success;
 
   // default I/O
   return (
@@ -79,16 +75,16 @@ export const IOPreview: React.FC<{
       ) : null}
       {isPrettyViewAvailable && currentView === "pretty" ? (
         <OpenAiMessageView
-          messages={inOpenAiMessageArray.data.concat(
-            outOpenAiMessage.success
+          messages={inChatMlArray.data.concat(
+            outChatMlMessage.success
               ? {
-                  ...outOpenAiMessage.data,
-                  role: outOpenAiMessage.data.role ?? "assistant",
+                  ...outChatMlMessage.data,
+                  role: outChatMlMessage.data.role ?? "assistant",
                 }
-              : {
+              : ChatMlMessageSchema.parse({
                   role: "assistant",
                   content: outputClean ? JSON.stringify(outputClean) : null,
-                },
+                }),
           )}
         />
       ) : null}
@@ -116,34 +112,54 @@ export const IOPreview: React.FC<{
   );
 };
 
-const OpenAiMessageSchema = z
+const ChatMlMessageSchema = z
   .object({
-    role: z.enum(["system", "user", "assistant", "function"]).optional(),
-    name: z.string().optional(),
-    content: z.union([z.record(z.any()).array(), z.string()]).nullable(),
-    function_call: z
-      .object({
-        name: z.string(),
-        arguments: z.record(z.any()),
-      })
+    role: z
+      .enum(["system", "user", "assistant", "function", "tool"])
       .optional(),
+    name: z.string().optional(),
+    content: z
+      .union([z.record(z.any()), z.record(z.any()).array(), z.string()])
+      .nullish(),
+    additional_kwargs: z.record(z.any()).optional(),
   })
-  .strict() // no additional properties
-  .refine((value) => value.content !== null || value.role !== undefined);
+  .passthrough()
 
-const OpenAiMessageArraySchema = z.array(OpenAiMessageSchema).min(1);
+  .refine((value) => value.content !== null || value.role !== undefined)
+  .transform(({ additional_kwargs, ...other }) => ({
+    ...other,
+    ...additional_kwargs,
+  }))
+  .transform(({ role, name, content, ...other }) => ({
+    role,
+    name,
+    content,
+    json: Object.keys(other).length === 0 ? undefined : other,
+  }));
+const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
 
 const OpenAiMessageView: React.FC<{
-  messages: z.infer<typeof OpenAiMessageArraySchema>;
+  messages: z.infer<typeof ChatMlArraySchema>;
 }> = ({ messages }) => {
   const COLLAPSE_THRESHOLD = 3;
   const [isCollapsed, setCollapsed] = useState(
     messages.length > COLLAPSE_THRESHOLD ? true : null,
   );
 
+  const transformedMessages = messages;
+  console.log(transformedMessages);
+  // const transformedMessages = messages.map(
+  //   ({ role, name, content, ...rest }) => ({
+  //     role,
+  //     name,
+  //     content,
+  //     json: rest,
+  //   }),
+  // );
+
   return (
     <div className="flex flex-col gap-2 rounded-md border p-3">
-      {messages
+      {transformedMessages
         .filter(
           (_, i) =>
             // show all if not collapsed or null; show first and last n if collapsed
@@ -151,14 +167,36 @@ const OpenAiMessageView: React.FC<{
         )
         .map((message, index) => (
           <Fragment key={index}>
-            <JSONView
-              title={message.name ?? message.role}
-              json={message.function_call ?? message.content}
-              className={cn(
-                message.role === "system" && "bg-gray-100",
-                message.role === "assistant" && "bg-green-50",
+            <div>
+              {!!message.content && (
+                <JSONView
+                  title={message.name ?? message.role}
+                  json={message.content}
+                  className={cn(
+                    "bg-gray-100",
+                    message.role === "system" && "bg-gray-100",
+                    message.role === "assistant" && "bg-green-50",
+                    message.role === "user" && "bg-white",
+                    !!message.json && "rounded-b-none",
+                  )}
+                />
               )}
-            />
+              {!!message.json && (
+                <JSONView
+                  title={
+                    message.content ? undefined : message.name ?? message.role
+                  }
+                  json={message.json}
+                  className={cn(
+                    "bg-gray-100",
+                    message.role === "system" && "bg-gray-100",
+                    message.role === "assistant" && "bg-green-50",
+                    message.role === "user" && "bg-white",
+                    !!message.content && "rounded-t-none border-t-0",
+                  )}
+                />
+              )}
+            </div>
             {isCollapsed !== null && index === 0 ? (
               <Button
                 variant="ghost"

web/src/components/trace/ObservationPreview.tsx

@@ -21,7 +21,7 @@ import { NewDatasetItemFromObservationButton } from "@/src/features/datasets/com
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
 import { IOPreview } from "@/src/components/trace/IOPreview";
-import { formatInterval } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 import Link from "next/link";
 import { usdFormatter } from "@/src/utils/numbers";
 import { calculateDisplayTotalCost } from "@/src/components/trace";
@@ -70,7 +70,7 @@ export const ObservationPreview = (props: {
             {preloadedObservation.completionStartTime ? (
               <Badge variant="outline">
                 Time to first token:{" "}
-                {formatInterval(
+                {formatIntervalSeconds(
                   (preloadedObservation.completionStartTime.getTime() -
                     preloadedObservation.startTime.getTime()) /
                     1000,
@@ -80,7 +80,7 @@ export const ObservationPreview = (props: {
             {preloadedObservation.endTime ? (
               <Badge variant="outline">
                 Latency:{" "}
-                {formatInterval(
+                {formatIntervalSeconds(
                   (preloadedObservation.endTime.getTime() -
                     preloadedObservation.startTime.getTime()) /
                     1000,

web/src/components/trace/ObservationTree.tsx

@@ -5,7 +5,7 @@ import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { LevelColors } from "@/src/components/level-colors";
-import { formatInterval } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 import { MinusCircle, MinusIcon, PlusCircleIcon, PlusIcon } from "lucide-react";
 import { Toggle } from "@/src/components/ui/toggle";
 import { Button } from "@/src/components/ui/button";
@@ -96,7 +96,7 @@ const ObservationTreeTraceNode = (props: {
     {props.showMetrics && props.trace.latency ? (
       <div className="flex gap-2">
         <span className="text-xs text-gray-500">
-          {formatInterval(props.trace.latency)}
+          {formatIntervalSeconds(props.trace.latency)}
         </span>
       </div>
     ) : null}
@@ -179,7 +179,7 @@ const ObservationTreeNode = (props: {
                     <div className="flex gap-2">
                       {observation.endTime ? (
                         <span className="text-xs text-gray-500">
-                          {formatInterval(
+                          {formatIntervalSeconds(
                             (observation.endTime.getTime() -
                               observation.startTime.getTime()) /
                               1000,

web/src/components/trace/TracePreview.tsx

@@ -21,7 +21,7 @@ import { ManualScoreButton } from "@/src/features/manual-scoring/components/Manu
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
-import { formatInterval } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 
 export const TracePreview = ({
   trace,
@@ -45,7 +45,9 @@ export const TracePreview = ({
           <CardDescription>{trace.timestamp.toLocaleString()}</CardDescription>
           <div className="flex flex-wrap gap-2">
             {!!trace.latency && (
-              <Badge variant="outline">{formatInterval(trace.latency)}</Badge>
+              <Badge variant="outline">
+                {formatIntervalSeconds(trace.latency)}
+              </Badge>
             )}
             <TraceAggUsageBadge observations={observations} />
             {!!trace.release && (

web/src/components/trace/index.tsx

@@ -12,7 +12,6 @@ import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNa
 import { useRouter } from "next/router";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
-import { DeleteTrace } from "@/src/components/delete-trace";
 import { StarTraceDetailsToggle } from "@/src/components/star-toggle";
 import Link from "next/link";
 import { NoAccessError } from "@/src/components/no-access";
@@ -23,6 +22,7 @@ import { Award, ChevronsDownUp, ChevronsUpDown } from "lucide-react";
 import { usdFormatter } from "@/src/utils/numbers";
 import Decimal from "decimal.js";
 import { useCallback, useState } from "react";
+import { DeleteButton } from "@/src/components/deleteButton";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
@@ -159,6 +159,7 @@ export function Trace(props: {
 
 export function TracePage({ traceId }: { traceId: string }) {
   const router = useRouter();
+  const utils = api.useUtils();
   const trace = api.traces.byId.useQuery(
     { traceId },
     {
@@ -220,9 +221,13 @@ export function TracePage({ traceId }: { traceId: string }) {
               }
               listKey="traces"
             />
-            <DeleteTrace
-              traceId={trace.data.id}
+            <DeleteButton
+              itemId={traceId}
               projectId={trace.data.projectId}
+              scope="traces:delete"
+              invalidateFunc={() => void utils.traces.invalidate()}
+              type="trace"
+              redirectUrl={`/project/${router.query.projectId as string}/traces`}
             />
           </>
         }

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.7.2";
+export const VERSION = "v2.11.3";

web/src/env.mjs

@@ -42,11 +42,14 @@ export const env = createEnv({
     // AUTH
     AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
     AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
+    AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_GITHUB_CLIENT_ID: z.string().optional(),
     AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),
+    AUTH_GITHUB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_AZURE_AD_CLIENT_ID: z.string().optional(),
     AUTH_AZURE_AD_CLIENT_SECRET: z.string().optional(),
     AUTH_AZURE_AD_TENANT_ID: z.string().optional(),
+    AUTH_AZURE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     // EMAIL
@@ -105,11 +108,17 @@ export const env = createEnv({
     // AUTH
     AUTH_GOOGLE_CLIENT_ID: process.env.AUTH_GOOGLE_CLIENT_ID,
     AUTH_GOOGLE_CLIENT_SECRET: process.env.AUTH_GOOGLE_CLIENT_SECRET,
+    AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING,
     AUTH_GITHUB_CLIENT_ID: process.env.AUTH_GITHUB_CLIENT_ID,
     AUTH_GITHUB_CLIENT_SECRET: process.env.AUTH_GITHUB_CLIENT_SECRET,
+    AUTH_GITHUB_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GITHUB_ALLOW_ACCOUNT_LINKING,
     AUTH_AZURE_AD_CLIENT_ID: process.env.AUTH_AZURE_AD_CLIENT_ID,
     AUTH_AZURE_AD_CLIENT_SECRET: process.env.AUTH_AZURE_AD_CLIENT_SECRET,
     AUTH_AZURE_AD_TENANT_ID: process.env.AUTH_AZURE_AD_TENANT_ID,
+    AUTH_AZURE_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_AZURE_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,

web/src/features/dashboard/components/BaseTimeSeriesChart.tsx

@@ -1,7 +1,7 @@
 import { type DateTimeAggregationOption } from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { cn } from "@/src/utils/tailwind";
-import { AreaChart } from "@tremor/react";
+import { AreaChart, LineChart } from "@tremor/react";
 
 export type TimeSeriesChartDataPoint = {
   ts: number;
@@ -15,6 +15,7 @@ export function BaseTimeSeriesChart(props: {
   showLegend?: boolean;
   connectNulls?: boolean;
   valueFormatter?: (value: number) => string;
+  chartType?: "line" | "area";
 }) {
   const labels = new Set(
     props.data.flatMap((d) => d.values.map((v) => v.label)),
@@ -54,8 +55,10 @@ export function BaseTimeSeriesChart(props: {
       day: "numeric",
     });
   };
+
+  const ChartComponent = props.chartType === "area" ? AreaChart : LineChart;
   return (
-    <AreaChart
+    <ChartComponent
       className={cn("mt-4", props.className)}
       data={transformArray(props.data)}
       index="timestamp"

web/src/features/dashboard/components/LatencyChart.tsx

@@ -16,7 +16,7 @@ import { TabComponent } from "@/src/features/dashboard/components/TabsComponent"
 import { numberFormatter } from "@/src/utils/numbers";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 
-export const LatencyChart = ({
+export const GenerationLatencyChart = ({
   className,
   projectId,
   globalFilterState,

web/src/features/dashboard/components/LatencyTables.tsx

@@ -0,0 +1,208 @@
+import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
+import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
+import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
+import { type FilterState } from "@/src/features/filters/types";
+import { api } from "@/src/utils/api";
+
+import { type DatabaseRow } from "@/src/server/api/services/query-builder";
+import { formatIntervalSeconds } from "@/src/utils/dates";
+import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
+import { truncate } from "@/src/utils/string";
+import { Popup } from "@/src/components/layouts/doc-popup";
+
+export const LatencyTables = ({
+  projectId,
+  globalFilterState,
+}: {
+  projectId: string;
+  globalFilterState: FilterState;
+}) => {
+  const generationsLatencies = api.dashboard.chart.useQuery(
+    {
+      projectId,
+      from: "traces_observations",
+      select: [
+        { column: "duration", agg: "50thPercentile" },
+        { column: "duration", agg: "90thPercentile" },
+        { column: "duration", agg: "95thPercentile" },
+        { column: "duration", agg: "99thPercentile" },
+        { column: "name" },
+      ],
+      filter: [
+        ...globalFilterState,
+        {
+          type: "string",
+          column: "type",
+          operator: "=",
+          value: "GENERATION",
+        },
+      ],
+      groupBy: [{ type: "string", column: "name" }],
+      orderBy: [
+        { column: "duration", agg: "95thPercentile", direction: "DESC" },
+      ],
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
+
+  const spansLatencies = api.dashboard.chart.useQuery(
+    {
+      projectId,
+      from: "traces_observations",
+      select: [
+        { column: "duration", agg: "50thPercentile" },
+        { column: "duration", agg: "90thPercentile" },
+        { column: "duration", agg: "95thPercentile" },
+        { column: "duration", agg: "99thPercentile" },
+        { column: "name" },
+      ],
+      filter: [
+        ...globalFilterState,
+        {
+          type: "string",
+          column: "type",
+          operator: "=",
+          value: "SPAN",
+        },
+      ],
+      groupBy: [{ type: "string", column: "name" }],
+      orderBy: [
+        { column: "duration", agg: "95thPercentile", direction: "DESC" },
+      ],
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
+
+  const tracesLatencies = api.dashboard.chart.useQuery(
+    {
+      projectId,
+      from: "traces_metrics",
+      select: [
+        { column: "duration", agg: "50thPercentile" },
+        { column: "duration", agg: "90thPercentile" },
+        { column: "duration", agg: "95thPercentile" },
+        { column: "duration", agg: "99thPercentile" },
+        { column: "traceName" },
+      ],
+      filter: [...createTracesTimeFilter(globalFilterState)],
+      groupBy: [{ type: "string", column: "traceName" }],
+      orderBy: [
+        { column: "duration", agg: "95thPercentile", direction: "DESC" },
+      ],
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+    },
+  );
+
+  const generateLatencyData = (data?: DatabaseRow[]) => {
+    return data
+      ? data
+          .filter((item) => item.name !== null)
+          .map((item, i) => [
+            <div key={`${item.name as string}-${i}`}>
+              <Popup
+                triggerContent={truncate(item.name as string)}
+                description={item.name as string}
+              />
+            </div>,
+            ...[
+              "percentile50Duration",
+              "percentile90Duration",
+              "percentile95Duration",
+              "percentile99Duration",
+            ].map((percentile) => (
+              <RightAlignedCell key={`${i}-${percentile}`}>
+                {item[percentile]
+                  ? formatIntervalSeconds(item[percentile] as number, 3)
+                  : "-"}
+              </RightAlignedCell>
+            )),
+          ])
+      : [];
+  };
+
+  return (
+    <>
+      <DashboardCard
+        className="col-span-1 xl:col-span-2"
+        title="Trace latencies"
+        isLoading={tracesLatencies.isLoading}
+      >
+        <DashboardTable
+          headers={[
+            "Trace Name",
+            <RightAlignedCell key="50th">50th</RightAlignedCell>,
+            <RightAlignedCell key="90th">90th</RightAlignedCell>,
+            <RightAlignedCell key="95th">
+              95th<span className="ml-1">▼</span>
+            </RightAlignedCell>,
+            <RightAlignedCell key="99th">99th</RightAlignedCell>,
+          ]}
+          rows={generateLatencyData(
+            tracesLatencies.data
+              ?.filter((item) => item.traceName !== null)
+              .map((item) => {
+                return { ...item, name: item.traceName as string };
+              }),
+          )}
+          collapse={{ collapsed: 5, expanded: 20 }}
+        />
+      </DashboardCard>
+      <DashboardCard
+        className="col-span-1 xl:col-span-2"
+        title="Generation latencies"
+        isLoading={generationsLatencies.isLoading}
+      >
+        <DashboardTable
+          headers={[
+            "Generation Name",
+            <RightAlignedCell key="50th">50th</RightAlignedCell>,
+            <RightAlignedCell key="90th">90th</RightAlignedCell>,
+            <RightAlignedCell key="95th">
+              95th<span className="ml-1">▼</span>
+            </RightAlignedCell>,
+            <RightAlignedCell key="99th">99th</RightAlignedCell>,
+          ]}
+          rows={generateLatencyData(generationsLatencies.data)}
+          collapse={{ collapsed: 5, expanded: 20 }}
+        />
+      </DashboardCard>
+      <DashboardCard
+        className="col-span-1 xl:col-span-2"
+        title="Span latencies"
+        isLoading={spansLatencies.isLoading}
+      >
+        <DashboardTable
+          headers={[
+            "Span Name",
+            <RightAlignedCell key="50th">50th</RightAlignedCell>,
+            <RightAlignedCell key="90th">90th</RightAlignedCell>,
+            <RightAlignedCell key="95th">
+              95th<span className="ml-1">▼</span>
+            </RightAlignedCell>,
+            <RightAlignedCell key="99th">99th</RightAlignedCell>,
+          ]}
+          rows={generateLatencyData(spansLatencies.data)}
+          collapse={{ collapsed: 5, expanded: 20 }}
+        />
+      </DashboardCard>
+    </>
+  );
+};

web/src/features/dashboard/components/MetricTable.tsx

@@ -4,8 +4,9 @@ import { DashboardCard } from "@/src/features/dashboard/components/cards/Dashboa
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
 import { type FilterState } from "@/src/features/filters/types";
 import { api } from "@/src/utils/api";
-import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";
+import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
 export const MetricTable = ({
   className,
@@ -19,13 +20,21 @@ export const MetricTable = ({
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observations",
+      from: "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "totalTokens", agg: "SUM" },
         { column: "model" },
       ],
-      filter: globalFilterState,
+      filter: [
+        ...globalFilterState,
+        {
+          type: "string",
+          column: "type",
+          operator: "=",
+          value: "GENERATION",
+        },
+      ],
       groupBy: [{ type: "string", column: "model" }],
       orderBy: [
         { column: "calculatedTotalCost", direction: "DESC", agg: "SUM" },
@@ -61,7 +70,9 @@ export const MetricTable = ({
           </RightAlignedCell>,
           <RightAlignedCell key={`${i}-cost`}>
             {item.sumCalculatedTotalCost
-              ? usdFormatter(item.sumCalculatedTotalCost as number, 2, 2)
+              ? totalCostDashboardFormatted(
+                  item.sumCalculatedTotalCost as number,
+                )
               : "$0"}
           </RightAlignedCell>,
         ])
@@ -83,7 +94,7 @@ export const MetricTable = ({
         collapse={{ collapsed: 5, expanded: 20 }}
       >
         <TotalMetric
-          metric={totalTokenCost ? usdFormatter(totalTokenCost, 2, 2) : "$0"}
+          metric={totalCostDashboardFormatted(totalTokenCost)}
           description="Total cost"
         >
           <DocPopup

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -13,11 +13,12 @@ import {
   isEmptyTimeSeries,
 } from "@/src/features/dashboard/components/hooks";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
-import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TabComponent } from "@/src/features/dashboard/components/TabsComponent";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
 import { NoData } from "@/src/features/dashboard/components/NoData";
+import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 
 export const ModelUsageChart = ({
   className,
@@ -33,13 +34,16 @@ export const ModelUsageChart = ({
   const tokens = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observations",
+      from: "traces_observationsview",
       select: [
         { column: "totalTokens", agg: "SUM" },
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "model" },
       ],
-      filter: globalFilterState,
+      filter: [
+        ...globalFilterState,
+        { type: "string", column: "type", operator: "=", value: "GENERATION" },
+      ],
       groupBy: [
         {
           type: "datetime",
@@ -102,14 +106,14 @@ export const ModelUsageChart = ({
   // had to add this function as tremor under the hodd adds more variables
   // to the function call which would break usdFormatter.
   const oneValueUsdFormatter = (value: number) => {
-    return usdFormatter(value, 2, 2);
+    return totalCostDashboardFormatted(value);
   };
 
   const data = [
     {
       tabTitle: "Total cost",
       data: transformedModelCost,
-      totalMetric: totalCost ? usdFormatter(totalCost, 2, 2) : usdFormatter(0),
+      totalMetric: totalCostDashboardFormatted(totalCost),
       metricDescription: `Token cost`,
       formatter: oneValueUsdFormatter,
     },

web/src/features/dashboard/components/TracesTimeSeriesChart.tsx

@@ -88,6 +88,7 @@ export const TracesTimeSeriesChart = ({
           agg={agg}
           data={transformedTraces}
           connectNulls={true}
+          chartType="area"
         />
       ) : (
         <NoData noDataText="No data available">

web/src/features/dashboard/components/UserChart.tsx

@@ -2,7 +2,7 @@ import { api } from "@/src/utils/api";
 import { type DateTimeAggregationOption } from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { type FilterState } from "@/src/features/filters/types";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
-import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
+import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TabComponent } from "@/src/features/dashboard/components/TabsComponent";
 import { BarList } from "@tremor/react";
 import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
@@ -10,7 +10,10 @@ import { ExpandListButton } from "@/src/features/dashboard/components/cards/Chev
 import { useState } from "react";
 import DocPopup from "@/src/components/layouts/doc-popup";
 import { NoData } from "@/src/features/dashboard/components/NoData";
-import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
+import {
+  createTracesTimeFilter,
+  totalCostDashboardFormatted,
+} from "@/src/features/dashboard/lib/dashboard-utils";
 
 type BarChartDataPoint = {
   name: string;
@@ -31,13 +34,21 @@ export const UserChart = ({
   const user = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "traces_observations",
+      from: "traces_observationsview",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "user" },
         { column: "traceId", agg: "COUNT" },
       ],
-      filter: globalFilterState,
+      filter: [
+        ...globalFilterState,
+        {
+          type: "string",
+          column: "type",
+          operator: "=",
+          value: "GENERATION",
+        },
+      ],
       groupBy: [
         {
           type: "string",
@@ -116,7 +127,8 @@ export const UserChart = ({
 
   const maxNumberOfEntries = { collapsed: 5, expanded: 20 } as const;
 
-  const localUsdFormatter = (value: number) => usdFormatter(value, 2, 2);
+  const localUsdFormatter = (value: number) =>
+    totalCostDashboardFormatted(value);
 
   const data = [
     {
@@ -124,7 +136,7 @@ export const UserChart = ({
       data: isExpanded
         ? transformedCost.slice(0, maxNumberOfEntries.expanded)
         : transformedCost.slice(0, maxNumberOfEntries.collapsed),
-      totalMetric: totalCost ? usdFormatter(totalCost, 2, 2) : usdFormatter(0),
+      totalMetric: totalCostDashboardFormatted(totalCost),
       metricDescription: "Total cost",
       formatter: localUsdFormatter,
     },

web/src/features/dashboard/lib/dashboard-utils.ts

@@ -1,4 +1,5 @@
 import { type FilterState } from "@/src/features/filters/types";
+import { usdFormatter } from "@/src/utils/numbers";
 
 // traces do not have a startTime or endTime column, so we need to map these to the timestamp column
 export const createTracesTimeFilter = (filters: FilterState) => {
@@ -13,3 +14,11 @@ export const createTracesTimeFilter = (filters: FilterState) => {
     }
   });
 };
+
+export const totalCostDashboardFormatted = (totalCost?: number) => {
+  return totalCost
+    ? totalCost < 5
+      ? usdFormatter(totalCost, 2, 4)
+      : usdFormatter(totalCost, 2, 2)
+    : usdFormatter(0);
+};

web/src/features/datasets/components/DatasetActionButton.tsx

@@ -0,0 +1,140 @@
+import { Button } from "@/src/components/ui/button";
+import { Edit, LockIcon, PlusIcon, Trash } from "lucide-react";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+} from "@/src/components/ui/dialog";
+import { useState } from "react";
+import { DialogTrigger } from "@radix-ui/react-dialog";
+import { DatasetForm } from "@/src/features/datasets/components/DatasetForm";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+
+interface BaseDatasetButtonProps {
+  mode: "create" | "rename" | "delete";
+  projectId: string;
+  className?: string;
+  onFormSuccess?: () => void;
+}
+
+interface CreateDatasetButtonProps extends BaseDatasetButtonProps {
+  mode: "create";
+}
+
+interface DeleteDatasetButtonProps extends BaseDatasetButtonProps {
+  mode: "delete";
+  datasetId: string;
+}
+
+interface RenameDatasetButtonProps extends BaseDatasetButtonProps {
+  mode: "rename";
+  datasetId: string;
+  datasetName: string;
+  icon?: boolean;
+}
+
+type DatasetActionButtonProps =
+  | CreateDatasetButtonProps
+  | RenameDatasetButtonProps
+  | DeleteDatasetButtonProps;
+
+export const DatasetActionButton = (props: DatasetActionButtonProps) => {
+  const [open, setOpen] = useState(false);
+  const hasAccess = useHasAccess({
+    projectId: props.projectId,
+    scope: "datasets:CUD",
+  });
+
+  return (
+    <Dialog open={hasAccess && open} onOpenChange={setOpen}>
+      <DialogTrigger asChild>
+        {props.mode === "rename" ? (
+          props.icon ? (
+            <Button
+              variant="outline"
+              size={"icon"}
+              className={props.className}
+              disabled={!hasAccess}
+            >
+              <Edit className="h-4 w-4" />
+            </Button>
+          ) : (
+            <div
+              className="relative flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none transition-colors hover:bg-accent hover:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50"
+              onClick={() => setOpen(true)}
+            >
+              {hasAccess ? (
+                <Edit className="mr-2 h-4 w-4" />
+              ) : (
+                <LockIcon className="mr-2 h-4 w-4" aria-hidden="true" />
+              )}
+              Rename
+            </div>
+          )
+        ) : props.mode === "delete" ? (
+          <div
+            className="relative flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none transition-colors hover:bg-accent hover:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50"
+            onClick={() => setOpen(true)}
+          >
+            <Trash className="mr-2 h-4 w-4" />
+            Delete
+          </div>
+        ) : (
+          <Button
+            variant="secondary"
+            className={props.className}
+            disabled={!hasAccess}
+          >
+            {hasAccess ? (
+              <PlusIcon className="-ml-0.5 mr-1.5" aria-hidden="true" />
+            ) : (
+              <LockIcon className="-ml-0.5 mr-1.5 h-3 w-3" aria-hidden="true" />
+            )}
+            New dataset
+          </Button>
+        )}
+      </DialogTrigger>
+      <DialogContent className="sm:max-w-xl">
+        <DialogHeader>
+          <DialogTitle className="mb-4">
+            {props.mode === "create"
+              ? "Create new dataset"
+              : props.mode === "delete"
+                ? "Please confirm"
+                : "Rename dataset"}
+          </DialogTitle>
+          {props.mode === "delete" && (
+            <DialogDescription className="text-md p-0">
+              This action cannot be undone and removes all the data associated
+              with this dataset.
+            </DialogDescription>
+          )}
+        </DialogHeader>
+        {props.mode === "create" ? (
+          <DatasetForm
+            mode="create"
+            projectId={props.projectId}
+            onFormSuccess={() => setOpen(false)}
+          />
+        ) : props.mode === "delete" ? (
+          <DatasetForm
+            mode="delete"
+            projectId={props.projectId}
+            onFormSuccess={() => setOpen(false)}
+            datasetId={props.datasetId}
+          />
+        ) : (
+          <DatasetForm
+            mode="rename"
+            projectId={props.projectId}
+            onFormSuccess={() => setOpen(false)}
+            datasetId={props.datasetId}
+            datasetName={props.datasetName}
+          />
+        )}
+      </DialogContent>
+    </Dialog>
+  );
+};

web/src/features/datasets/components/DatasetForm.tsx

@@ -0,0 +1,180 @@
+import { Button } from "@/src/components/ui/button";
+import * as z from "zod";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { api } from "@/src/utils/api";
+import { useState } from "react";
+import { usePostHog } from "posthog-js/react";
+import { Input } from "@/src/components/ui/input";
+
+interface BaseDatasetFormProps {
+  mode: "create" | "rename" | "delete";
+  projectId: string;
+  onFormSuccess?: () => void;
+  className?: string;
+}
+
+interface CreateDatasetFormProps extends BaseDatasetFormProps {
+  mode: "create";
+}
+
+interface DeleteDatasetFormProps extends BaseDatasetFormProps {
+  mode: "delete";
+  datasetId: string;
+}
+
+interface RenameDatasetFormProps extends BaseDatasetFormProps {
+  mode: "rename";
+  datasetId: string;
+  datasetName: string;
+}
+
+type DatasetFormProps =
+  | CreateDatasetFormProps
+  | RenameDatasetFormProps
+  | DeleteDatasetFormProps;
+
+const formSchema = z.object({
+  name: z
+    .string()
+    .min(1, { message: "Input is required" })
+    .refine((name) => name.trim().length > 0, {
+      message: "Input should not be only whitespace",
+    }),
+});
+
+export const DatasetForm = (props: DatasetFormProps) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const posthog = usePostHog();
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    defaultValues: {
+      name: "",
+    },
+  });
+
+  const utils = api.useUtils();
+  const createMutation = api.datasets.createDataset.useMutation();
+  const renameMutation = api.datasets.updateDataset.useMutation();
+  const deleteMutation = api.datasets.deleteDataset.useMutation();
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    const trimmedValues = {
+      ...values,
+      name: values.name.trim(),
+    };
+    if (props.mode === "create") {
+      posthog.capture("datasets:new_dataset_form_submit");
+      createMutation
+        .mutateAsync({
+          ...trimmedValues,
+          projectId: props.projectId,
+        })
+        .then(() => {
+          void utils.datasets.invalidate();
+          props.onFormSuccess?.();
+          form.reset();
+        })
+        .catch((error: Error) => {
+          setFormError(error.message);
+          console.error(error);
+        });
+    } else if (props.mode === "rename") {
+      posthog.capture("datasets:rename_dataset_form_submit");
+      renameMutation
+        .mutateAsync({
+          ...trimmedValues,
+          projectId: props.projectId,
+          datasetId: props.datasetId,
+        })
+        .then(() => {
+          void utils.datasets.invalidate();
+          props.onFormSuccess?.();
+          form.reset();
+        })
+        .catch((error: Error) => {
+          setFormError(error.message);
+          console.error(error);
+        });
+    }
+  }
+
+  const handleDelete = () => {
+    if (props.mode !== "delete") return;
+    posthog.capture("datasets:delete_dataset_form_submit");
+    deleteMutation
+      .mutateAsync({
+        projectId: props.projectId,
+        datasetId: props.datasetId,
+      })
+      .then(() => {
+        void utils.datasets.invalidate();
+        props.onFormSuccess?.();
+        form.reset();
+      })
+      .catch((error: Error) => {
+        setFormError(error.message);
+        console.error(error);
+      });
+  };
+
+  return (
+    <div>
+      <Form {...form}>
+        <form
+          // eslint-disable-next-line @typescript-eslint/no-misused-promises
+          onSubmit={
+            props.mode === "delete" ? handleDelete : form.handleSubmit(onSubmit)
+          }
+          className="space-y-8"
+        >
+          {props.mode !== "delete" && (
+            <FormField
+              control={form.control}
+              name="name"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Name</FormLabel>
+                  <FormControl>
+                    <Input
+                      {...field}
+                      placeholder={
+                        props.mode === "rename" ? props.datasetName : ""
+                      }
+                    />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          )}
+          <Button
+            type="submit"
+            variant={props.mode === "delete" ? "destructive" : "default"}
+            loading={props.mode === "create" && createMutation.isLoading}
+            className="w-full"
+          >
+            {props.mode === "create"
+              ? "Create dataset"
+              : props.mode === "delete"
+                ? "Delete Dataset"
+                : "Rename dataset"}
+          </Button>
+        </form>
+      </Form>
+      {formError && (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      )}
+    </div>
+  );
+};

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -3,9 +3,10 @@ import { DataTable } from "@/src/components/table/data-table";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { api } from "@/src/utils/api";
-import { formatInterval, intervalInSeconds } from "@/src/utils/dates";
+import { formatIntervalSeconds, intervalInSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { type Score } from "@prisma/client";
+import { usdFormatter } from "../../../utils/numbers";
 
 type RowData = {
   id: string;
@@ -14,6 +15,7 @@ type RowData = {
   observation: { id: string; traceId: string };
   scores: Score[];
   latency: number;
+  totalCost: string;
 };
 
 export function DatasetRunItemsTable(
@@ -69,7 +71,15 @@ export function DatasetRunItemsTable(
       header: "Latency",
       cell: ({ row }) => {
         const latency: RowData["latency"] = row.getValue("latency");
-        return <>{formatInterval(latency)}</>;
+        return <>{formatIntervalSeconds(latency)}</>;
+      },
+    },
+    {
+      accessorKey: "totalCost",
+      header: "Total Cost",
+      cell: ({ row }) => {
+        const totalCost: RowData["totalCost"] = row.getValue("totalCost");
+        return <>{totalCost}</>;
       },
     },
     {
@@ -94,6 +104,9 @@ export function DatasetRunItemsTable(
         traceId: item.observation.traceId ?? "", // never actually null, just not enforced by db
       },
       scores: item.observation.scores,
+      totalCost: usdFormatter(
+        item.observation.calculatedTotalCost?.toNumber() ?? 0,
+      ),
       latency: intervalInSeconds(
         item.observation.startTime,
         item.observation.endTime,

web/src/features/datasets/components/DatasetRunsTable.tsx

@@ -4,9 +4,10 @@ import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
-import { formatInterval } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { useEffect } from "react";
+import { usdFormatter } from "../../../utils/numbers";
 
 type RowData = {
   key: {
@@ -16,6 +17,7 @@ type RowData = {
   createdAt: string;
   countRunItems: string;
   avgLatency: number;
+  avgTotalCost: string;
   scores: RouterOutput["datasets"]["runsByDatasetId"][number]["scores"];
 };
 
@@ -65,7 +67,16 @@ export function DatasetRunsTable(props: {
       header: "Latency (avg)",
       cell: ({ row }) => {
         const avgLatency: RowData["avgLatency"] = row.getValue("avgLatency");
-        return <>{formatInterval(avgLatency)}</>;
+        return <>{formatIntervalSeconds(avgLatency)}</>;
+      },
+    },
+    {
+      accessorKey: "avgTotalCost",
+      header: "Total Cost (avg)",
+      cell: ({ row }) => {
+        const avgTotalCost: RowData["avgTotalCost"] =
+          row.getValue("avgTotalCost");
+        return <>{avgTotalCost}</>;
       },
     },
     {
@@ -94,6 +105,7 @@ export function DatasetRunsTable(props: {
       createdAt: item.createdAt.toISOString(),
       countRunItems: item.countRunItems.toString(),
       avgLatency: item.avgLatency,
+      avgTotalCost: usdFormatter(item.avgTotalCost.toNumber()),
       scores: item.scores,
     };
   };

web/src/features/datasets/components/DatasetsTable.tsx

@@ -5,15 +5,14 @@ import { Button } from "@/src/components/ui/button";
 import {
   DropdownMenu,
   DropdownMenuContent,
-  DropdownMenuItem,
   DropdownMenuLabel,
   DropdownMenuTrigger,
 } from "@/src/components/ui/dropdown-menu";
-import { NewDatasetButton } from "@/src/features/datasets/components/NewDatasetButton";
+import { DatasetActionButton } from "@/src/features/datasets/components/DatasetActionButton";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
 import { type RouterOutput } from "@/src/utils/types";
-import { MoreVertical, Trash } from "lucide-react";
+import { MoreVertical } from "lucide-react";
 import { useEffect } from "react";
 
 type RowData = {
@@ -29,14 +28,11 @@ type RowData = {
 
 export function DatasetsTable(props: { projectId: string }) {
   const { setDetailPageList } = useDetailPageLists();
-  const utils = api.useUtils();
   const datasets = api.datasets.allDatasets.useQuery({
     projectId: props.projectId,
   });
-  const mutDelete = api.datasets.deleteDataset.useMutation({
-    onSuccess: () => utils.datasets.invalidate(),
-  });
-    useEffect(() => {
+
+  useEffect(() => {
     if (datasets.isSuccess) {
       setDetailPageList(
         "datasets",
@@ -91,17 +87,17 @@ export function DatasetsTable(props: { projectId: string }) {
             </DropdownMenuTrigger>
             <DropdownMenuContent align="end">
               <DropdownMenuLabel>Actions</DropdownMenuLabel>
-              <DropdownMenuItem
-                onClick={() =>
-                  mutDelete.mutate({
-                    projectId: props.projectId,
-                    datasetId: key.id,
-                  })
-                }
-              >
-                <Trash className="mr-2 h-4 w-4" />
-                Delete permanently
-              </DropdownMenuItem>
+              <DatasetActionButton
+                mode="rename"
+                projectId={props.projectId}
+                datasetId={key.id}
+                datasetName={key.name}
+              />
+              <DatasetActionButton
+                mode="delete"
+                projectId={props.projectId}
+                datasetId={key.id}
+              />
             </DropdownMenuContent>
           </DropdownMenu>
         );
@@ -141,7 +137,11 @@ export function DatasetsTable(props: { projectId: string }) {
                 }
         }
       />
-      <NewDatasetButton projectId={props.projectId} className="mt-4" />
+      <DatasetActionButton
+        projectId={props.projectId}
+        className="mt-4"
+        mode="create"
+      />
     </div>
   );
 }

web/src/features/datasets/components/NewDatasetButton.tsx

@@ -1,52 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import { LockIcon, PlusIcon } from "lucide-react";
-import {
-  Dialog,
-  DialogContent,
-  DialogHeader,
-  DialogTitle,
-} from "@/src/components/ui/dialog";
-import { useState } from "react";
-import { DialogTrigger } from "@radix-ui/react-dialog";
-import { NewDatasetForm } from "@/src/features/datasets/components/NewDatasetForm";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-
-export const NewDatasetButton = (props: {
-  projectId: string;
-  datasetId?: string;
-  className?: string;
-}) => {
-  const [open, setOpen] = useState(false);
-  const hasAccess = useHasAccess({
-    projectId: props.projectId,
-    scope: "datasets:CUD",
-  });
-
-  return (
-    <Dialog open={hasAccess && open} onOpenChange={setOpen}>
-      <DialogTrigger asChild>
-        <Button
-          variant="secondary"
-          className={props.className}
-          disabled={!hasAccess}
-        >
-          {hasAccess ? (
-            <PlusIcon className="-ml-0.5 mr-1.5" aria-hidden="true" />
-          ) : (
-            <LockIcon className="-ml-0.5 mr-1.5 h-3 w-3" aria-hidden="true" />
-          )}
-          New dataset
-        </Button>
-      </DialogTrigger>
-      <DialogContent className="sm:max-w-3xl">
-        <DialogHeader>
-          <DialogTitle className="mb-5">Create new dataset</DialogTitle>
-        </DialogHeader>
-        <NewDatasetForm
-          projectId={props.projectId}
-          onFormSuccess={() => setOpen(false)}
-        />
-      </DialogContent>
-    </Dialog>
-  );
-};

web/src/features/datasets/components/NewDatasetForm.tsx

@@ -1,94 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import * as z from "zod";
-import { zodResolver } from "@hookform/resolvers/zod";
-import { useForm } from "react-hook-form";
-import {
-  Form,
-  FormControl,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from "@/src/components/ui/form";
-import { api } from "@/src/utils/api";
-import { useState } from "react";
-import { usePostHog } from "posthog-js/react";
-import { Input } from "@/src/components/ui/input";
-
-const formSchema = z.object({
-  name: z.string(),
-});
-
-export const NewDatasetForm = (props: {
-  projectId: string;
-  onFormSuccess?: () => void;
-}) => {
-  const [formError, setFormError] = useState<string | null>(null);
-  const posthog = usePostHog();
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
-    defaultValues: {
-      name: "",
-    },
-  });
-
-  const utils = api.useUtils();
-  const createDatasetMutation = api.datasets.createDataset.useMutation({
-    onSuccess: () => utils.datasets.invalidate(),
-    onError: (error) => setFormError(error.message),
-  });
-
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    posthog.capture("datasets:new_dataset_form_submit");
-    createDatasetMutation
-      .mutateAsync({
-        ...values,
-        projectId: props.projectId,
-      })
-      .then(() => {
-        props.onFormSuccess?.();
-        form.reset();
-      })
-      .catch((error) => {
-        console.error(error);
-      });
-  }
-
-  return (
-    <div>
-      <Form {...form}>
-        <form
-          // eslint-disable-next-line @typescript-eslint/no-misused-promises
-          onSubmit={form.handleSubmit(onSubmit)}
-          className="space-y-8"
-        >
-          <FormField
-            control={form.control}
-            name="name"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Name</FormLabel>
-                <FormControl>
-                  <Input {...field} />
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <Button
-            type="submit"
-            loading={createDatasetMutation.isLoading}
-            className="w-full"
-          >
-            New dataset
-          </Button>
-        </form>
-      </Form>
-      {formError ? (
-        <p className="text-red text-center">
-          <span className="font-bold">Error:</span> {formError}
-        </p>
-      ) : null}
-    </div>
-  );
-};

web/src/features/datasets/server/dataset-router.ts

@@ -79,6 +79,7 @@ export const datasetRouter = createTRPCRouter({
             countRunItems: number;
             scores: Record<string, number>;
             avgLatency: number;
+            avgTotalCost: Prisma.Decimal;
           }
         >
       >(Prisma.sql`
@@ -111,13 +112,14 @@ export const datasetRouter = createTRPCRouter({
           ORDER BY
             run_id
         ),
-        latency_by_run_id AS (
+        latency_and_total_cost_by_run_id AS (
           SELECT
             ri.dataset_run_id run_id,
-            AVG(CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END)  AS "avgLatency"
+            AVG(CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END)  AS "avgLatency",
+            AVG(COALESCE(o.calculated_total_cost, 0)) AS "avgTotalCost"
           FROM
             dataset_run_items ri
-            JOIN observations o ON o.id = ri.observation_id
+            JOIN observations_view o ON o.id = ri.observation_id
           WHERE o.project_id = ${input.projectId}
           group by 1
         )
@@ -128,14 +130,15 @@ export const datasetRouter = createTRPCRouter({
           runs.created_at "createdAt",
           runs.updated_at "updatedAt",
           COALESCE(avg_scores.scores, '[]'::jsonb) scores,
-          COALESCE(latency."avgLatency", 0) "avgLatency",
+          COALESCE(latency_and_total_cost."avgLatency", 0) "avgLatency",
+          COALESCE(latency_and_total_cost."avgTotalCost", 0) "avgTotalCost",  
           count(DISTINCT ri.id)::int "countRunItems"
         FROM
           dataset_runs runs
           JOIN datasets ON datasets.id = runs.dataset_id
           LEFT JOIN dataset_run_items ri ON ri.dataset_run_id = runs.id
           LEFT JOIN json_avg_scores_by_run_id avg_scores ON avg_scores.run_id = runs.id
-          LEFT JOIN latency_by_run_id latency ON latency.run_id = runs.id
+          LEFT JOIN latency_and_total_cost_by_run_id latency_and_total_cost ON latency_and_total_cost.run_id = runs.id
         WHERE runs.dataset_id = ${input.datasetId}
           AND datasets.project_id = ${input.projectId}
         GROUP BY
@@ -144,7 +147,8 @@ export const datasetRouter = createTRPCRouter({
           3,
           4,
           5,
-          6
+          6,
+          7
         ORDER BY
           runs.created_at DESC
       `);
@@ -270,6 +274,41 @@ export const datasetRouter = createTRPCRouter({
         after: dataset,
       });
 
+      return dataset;
+    }),
+  updateDataset: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        datasetId: z.string(),
+        name: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "datasets:CUD",
+      });
+      const dataset = await ctx.prisma.dataset.update({
+        where: {
+          id: input.datasetId,
+          projectId: input.projectId,
+        },
+        data: {
+          name: input.name,
+        },
+      });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "dataset",
+        resourceId: dataset.id,
+        projectId: input.projectId,
+        action: "update",
+        before: { name: input.name },
+        after: dataset,
+      });
+
       return dataset;
     }),
   deleteDataset: protectedProjectProcedure
@@ -360,7 +399,7 @@ export const datasetRouter = createTRPCRouter({
         ),
     )
     .query(async ({ input, ctx }) => {
-      return ctx.prisma.datasetRunItems.findMany({
+      const runItems = await ctx.prisma.datasetRunItems.findMany({
         where: {
           datasetRunId: input.datasetRunId,
           datasetItemId: input.datasetItemId,
@@ -382,6 +421,31 @@ export const datasetRouter = createTRPCRouter({
           createdAt: "desc",
         },
       });
+
+      const observationIds = runItems.map((ri) => ri.observationId);
+      const observations = await ctx.prisma.observationView.findMany({
+        select: {
+          id: true,
+          calculatedTotalCost: true,
+        },
+        where: {
+          id: {
+            in: observationIds,
+          },
+        },
+      });
+
+      return runItems.map((ri) => {
+        return {
+          id: ri.id,
+          createdAt: ri.createdAt,
+          datasetItemId: ri.datasetItemId,
+          observation: {
+            ...ri.observation,
+            ...observations.find((o) => o.id === ri.observationId),
+          },
+        };
+      });
     }),
   observationInDatasets: protectedProjectProcedure
     .input(

web/src/features/feedback/server/feedbackHandler.ts

@@ -19,6 +19,6 @@ export default async function feedbackApiHandler(
     }
   } catch (error) {
     console.error(error);
-    res.status(400).json({ status: "Error" });
+    res.status(500).json({ status: "Error" });
   }
 }

web/src/features/filters/components/filter-builder.tsx

@@ -26,6 +26,7 @@ import {
   filterOperators,
   singleFilter,
 } from "@/src/server/api/interfaces/filters";
+import { NonEmptyString } from "@/src/utils/zod";
 
 // Has WipFilterState, passes all valid filters to parent onChange
 export function FilterBuilder({
@@ -185,6 +186,7 @@ function FilterBuilderForm({
               <tr key={i}>
                 <td className="p-1 text-sm">{i === 0 ? "Where" : "And"}</td>
                 <td className="flex gap-2 p-1">
+                  {/* selector of the column to be filtered */}
                   <Select
                     value={filter.column ?? ""}
                     onValueChange={(value) =>
@@ -217,6 +219,7 @@ function FilterBuilderForm({
                   (column?.type === "numberObject" ||
                     column?.type === "stringObject") ? (
                     column.keyOptions ? (
+                      // selector of the key of the object to be filtered
                       <Select
                         disabled={!filter.column}
                         onValueChange={(value) => {
@@ -228,11 +231,13 @@ function FilterBuilderForm({
                           <SelectValue placeholder="" />
                         </SelectTrigger>
                         <SelectContent>
-                          {column.keyOptions.map((option) => (
-                            <SelectItem key={option} value={option}>
-                              {option}
-                            </SelectItem>
-                          ))}
+                          {column.keyOptions
+                            .filter((o) => NonEmptyString.safeParse(o).success)
+                            .map((option) => (
+                              <SelectItem key={option} value={option}>
+                                {option}
+                              </SelectItem>
+                            ))}
                         </SelectContent>
                       </Select>
                     ) : (

web/src/features/filters/server/filterToPrisma.ts

@@ -21,6 +21,9 @@ const arrayOperatorReplacements = {
   "none of": "&&",
 };
 
+/**
+ * SECURITY: This function must only be used, when all its inputs were verified with zod.
+ */
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
@@ -33,6 +36,10 @@ export function tableColumnsToSqlFilterAndPrefix(
   return Prisma.join([Prisma.raw("AND "), sql], "");
 }
 
+/**
+ * SECURITY: This function must only be used, when all its inputs were verified with zod.
+ * Converts filter state and table columns to a Prisma SQL filter.
+ */
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
@@ -138,7 +145,9 @@ export function tableColumnsToSqlFilter(
   if (statements.length === 0) {
     return Prisma.empty;
   }
-
+  // FOR SECURITY: We join the statements with " AND " to prevent SQL injection.
+  // IF WE EVER CHANGE THIS, WE MUST ENSURE THAT USERS ONLY ACCESS THE DATA THEY ARE ALLOWED TO.
+  // Example: Or condition on charts API on projectId would break this.
   return Prisma.join(statements, " AND ");
 }
 
@@ -149,6 +158,7 @@ const castValueToPostgresTypes = (
   return column.name === "type" &&
     (table === "observations" ||
       table === "traces_observations" ||
+      table === "traces_observationsview" ||
       table === "traces_parent_observation_scores")
     ? Prisma.sql`::"ObservationType"`
     : Prisma.empty;

web/src/features/manual-scoring/components/ManualScoreButton.tsx

@@ -69,9 +69,15 @@ export function ManualScoreButton({
       utils.sessions.invalidate(),
     ]);
   };
-  const mutCreateScore = api.scores.create.useMutation({ onSuccess });
-  const mutUpdateScore = api.scores.update.useMutation({ onSuccess });
-  const mutDeleteScore = api.scores.delete.useMutation({ onSuccess });
+  const mutCreateScore = api.scores.createReviewScore.useMutation({
+    onSuccess,
+  });
+  const mutUpdateScore = api.scores.updateReviewScore.useMutation({
+    onSuccess,
+  });
+  const mutDeleteScore = api.scores.deleteReviewScore.useMutation({
+    onSuccess,
+  });
 
   const handleDelete = async () => {
     if (score) {

web/src/features/projects/components/HostNameProject.tsx

@@ -1,10 +1,7 @@
 import { Card } from "@tremor/react";
 import { CodeView } from "@/src/components/ui/code";
-import { env } from "@/src/env.mjs";
 
 export function HostNameProject() {
-  if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU") return null;
-
   return (
     <div>
       <h2 className="mb-5 text-base font-semibold leading-6 text-gray-900">

web/src/features/prompts/components/new-prompt-button.tsx

@@ -178,7 +178,7 @@ export const NewPromptForm = (props: {
         form.reset();
         // go to the following page after creating the prompt
         void router.push(
-          `/project/${props.projectId}/prompts/${newPrompt.name}`,
+          `/project/${props.projectId}/prompts/${encodeURIComponent(newPrompt.name)}`,
         );
       })
       .catch((error) => {

web/src/features/prompts/components/prompt-detail.tsx

@@ -56,7 +56,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
               },
               {
                 name: prompt.name,
-                href: `/project/${props.projectId}/prompts/${prompt.name}`,
+                href: `/project/${props.projectId}/prompts/${props.promptName}`,
               },
               { name: `Version ${prompt.version}` },
             ]}
@@ -89,7 +89,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
                 />
                 <DetailPageNav
                   key="nav"
-                  currentId={prompt.name}
+                  currentId={props.promptName}
                   path={(name) => `/project/${props.projectId}/prompts/${name}`}
                   listKey="prompts"
                 />