chore: release v2.25.0

perf: load trace i/o separately from rest of session (#1706 )
* refactor: io table cell * wip * fix imports * perf: load trace i/o within session separately from the rest of the session
2024-04-14 00:02:12 +02:00 · 2024-04-13 22:01:09 +00:00 · 2024-04-13 23:53:39 +02:00 · 2024-04-13 20:44:23 +00:00 · 2024-04-13 20:36:07 +00:00 · 2024-04-13 19:18:56 +00:00
283 changed files with 11545 additions and 8948 deletions
@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""

 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
@@ -16,4 +16,11 @@ SALT="salt"

 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
@@ -20,6 +20,9 @@ NEXTAUTH_URL="http://localhost:3000"
 NEXTAUTH_SECRET="secret"
 SALT="salt"

+# Use CSP headers to enforce HTTPS, optional
+LANGFUSE_CSP_ENFORCE_HTTPS="true"
+
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
@@ -34,6 +37,7 @@ SALT="salt"
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
+# AUTH_DISABLE_SIGNUP=true

 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -46,6 +50,14 @@ SALT="salt"
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
 # AUTH_AZURE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_OKTA_CLIENT_ID=
+# AUTH_OKTA_CLIENT_SECRET=
+# AUTH_OKTA_ISSUER=
+# AUTH_OKTA_ALLOW_ACCOUNT_LINKING=false
+# AUTH_AUTH0_CLIENT_ID=
+# AUTH_AUTH0_CLIENT_SECRET=
+# AUTH_AUTH0_ISSUER=
+# AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false

 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -65,11 +77,6 @@ SALT="salt"
 # DB_EXPORT_PAGE_SIZE=1000


-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
@@ -94,6 +101,10 @@ SALT="salt"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=

+# Prompt playground
+# OPENAI_API_KEY=""
+# ANTHROPIC_API_KEY=""
+
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=

@@ -0,0 +1,2 @@
+# Currently inactive
+# * @langfuse/maintainers
@@ -6,37 +6,11 @@
 version: 2
 updates:
  - package-ecosystem: npm
-    directory: "/worker" # Location of package manifests
+    directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
    schedule:
      interval: "daily"
-    rebase-strategy: "disabled" # use dependabot-rebase-stale
-    commit-message:
-      prefix: chore
-      prefix-development: chore
-      include: scope
-    ignore:
-      - dependency-name: "@types/node"
-      - dependency-name: "@trpc/*"
-    groups:
-      sentry:
-        patterns:
-          - "@sentry/*"
-      prisma:
-        patterns:
-          - "prisma"
-          - "@prisma/*"
-      next:
-        patterns:
-          - "eslint-config-next"
-          - "next"
-      patches:
-        update-types:
-          - "patch"
-  - package-ecosystem: npm
-    directory: "/web" # Location of package manifests
-    schedule:
-      interval: "daily"
-    rebase-strategy: "disabled" # use dependabot-rebase-stale
+    versioning-strategy: "increase"
    commit-message:
      prefix: chore
      prefix-development: chore
@@ -25,7 +25,7 @@ jobs:
        uses: actions/checkout@v3

      - name: Start containers
-        run: docker-compose -f "docker-compose.yml" up -d --build
+        run: docker compose -f "docker-compose.yml" up -d --build

      - name: Setup pnpm
        uses: pnpm/action-setup@v2.2.4
@@ -54,7 +54,7 @@ jobs:
          file: ./worker/Dockerfile
          push: false

-  tests:
+  tests-web:
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -87,7 +87,7 @@ jobs:

      - name: Run + migrate
        run: |
-          docker-compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev.yml up -d
          sleep 5 # Wait for PostgreSQL to accept connections

      - name: Seed DB
@@ -102,7 +102,56 @@ jobs:
        run: (pnpm run start&)

      - name: run tests
-        run: pnpm run test
+        run: pnpm --filter=web run test
+
+  tests-worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 8
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+
+      - name: Build
+        run: pnpm --filter=worker... run build
+
+      - name: run tests
+        run: pnpm --filter=worker run test

  e2e-tests:
    runs-on: ubuntu-latest
@@ -128,7 +177,7 @@ jobs:

      - name: Run + migrate
        run: |
-          docker-compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev.yml up -d
          sleep 5 # Wait for PostgreSQL to accept connections

      - name: Seed DB
@@ -140,9 +189,7 @@ jobs:
        run: pnpm run build

      - name: Install playwright
-        run: |
-          npm add --global playwright
-          pnpm exec playwright install
+        run: pnpm --filter=web exec playwright install --with-deps

      - name: Run e2e tests
        run: pnpm --filter=web run test:e2e
@@ -150,7 +197,7 @@ jobs:
  all-ci-passed:
    # This allows us to have a branch protection rule for tests and deploys with matrix
    runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests, test-docker-build]
+    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
    if: always()
    steps:
      - name: Successful deploy
@@ -167,9 +214,6 @@ jobs:
    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
    environment: "protected branches"
    runs-on: ubuntu-latest
-    env:
-      REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
    permissions:
      packages: write
      contents: read
@@ -187,32 +231,61 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v3

-      - name: Log in to the Container registry
+      - name: Log in to the GitHub Container registry
        uses: docker/login-action@v2
        with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
        uses: docker/metadata-action@v4
        with:
          images: |
-            ${{ env.REGISTRY }}/${{ github.repository }}
+            ghcr.io/langfuse/langfuse # GitHub
+            langfuse/langfuse # Docker Hub
          tags: |
            type=ref,event=branch
            type=ref,event=pr
-            type=sha
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}

-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./web/Dockerfile
          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/langfuse/langfuse-worker # GitHub
+            langfuse/langfuse-worker # Docker Hub
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}
@@ -55,8 +55,10 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 flowchart TB
   subgraph s4["Clients"]
      subgraph s2["langfuse/langfuse-python"]
-         Python["Python SDK"]
+         Python["Python low-level SDK"]
+         Decorator["observe() decorator"] -->|extends| Python
         OAI["OpenAI drop-in replacement"] -->|extends| Python
+         Llamaindex["LlamaIndex Integration"] -->|extends| Python
         LCPYTHON["Langchain Python Integration"] -->|extends| Python
         Langflow -->|uses| LCPYTHON
         LiteLLM -->|uses| Python
@@ -190,7 +192,7 @@ Requirements
  To run migrations, you can execute the following command.

  ```bash
-  pnpm --filter=shared run db:migrate
+  pnpm run db:migrate -- --name <name of the migration>
  ```

 > [!NOTE]
@@ -1,25 +1,19 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 <div align="center">
-   <a href="https://langfuse.com">
-      <h1>🪢 Langfuse</h1>
-   </a>
- <div>
-      <h3> <a href="https://cloud.langfuse.com">
-         <strong>Sign up</strong>
-       </a> · 
-      <a href="https://langfuse.com/docs/deployment/self-host">
-         <strong>Self Host Langfuse</strong>
-      </a> · 
-      <a href="https://langfuse.com/demo">
-         <strong>Demo Project (live data)</strong>
-      </a>
+   <div>
+      <h3>
+         <a href="https://cloud.langfuse.com">
+            <strong>Sign up</strong>
+         </a> · 
+         <a href="https://langfuse.com/docs/deployment/self-host">
+            <strong>Self Host</strong>
+         </a> · 
+         <a href="https://langfuse.com/demo">
+            <strong>Demo (live data)</strong>
+         </a>
      </h3>
-   <h3>
-      Langfuse is the open source LLM engineering platform.   
-   </h3>
-    <div>
-     Debug, analyze and iterate - together
   </div>
-   </br>
   <div>
      <a href="https://langfuse.com/docs"><strong>Docs</strong></a> ·
      <a href="https://langfuse.com/issues"><strong>Report Bug</strong></a> ·
@@ -28,7 +22,10 @@
      <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
      <a href="https://langfuse.com/discord"><strong>Discord</strong></a> 
   </div>
-   </br>
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <br/>
+   <br/>
+   <div>
      <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
      <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
      <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
@@ -60,12 +57,6 @@
  - Datasets let you test expected in and output pairs and benchmark performance before deploying ([Datasets](https://langfuse.com/docs/datasets))
  - Track versions and releases in your application ([Experimentation](https://langfuse.com/docs/experimentation), [Prompt Management](https://langfuse.com/docs/prompts))

-### Video: Langfuse in two minutes
-
-https://github.com/langfuse/langfuse/assets/2834609/6041347a-b517-4a11-8737-93ef8f8af49f
-
-_Muted by default, enable sound for voice-over_
-
 ## Get started

 ### Langfuse Cloud
@@ -98,15 +89,15 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe

 ### API Keys

-You require a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
+You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.

 ### Ingesting Data · Instrumenting Your Application

-Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
+Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.

 You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.

-See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get started in integrating Langfuse.
+See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.

 ### Integrations

@@ -114,8 +105,8 @@ See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get start
 | ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
 | [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.      |
 | [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python        | Automated instrumentation via drop-in replacement of OpenAI SDK. |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.        |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python       | Automated instrumentation via LlamaIndex callback system.        |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                         |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.        |
 | [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.            |

 External projects/packages that integrate with Langfuse:
@@ -140,8 +131,8 @@ External projects/packages that integrate with Langfuse:
 In order of preference the best way to communicate with us:

 - [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
- [Discord](https://langfuse.com/discord): For community support and to chat directly with maintainers
- Privately: Email contact at langfuse dot com
+- [Discord](https://langfuse.com/discord): community support
+- Privately: contact at langfuse dot com

 ## Contributing to Langfuse

@@ -16,11 +16,9 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  worker:
@@ -30,7 +28,6 @@ services:
    depends_on:
      - db
      - redis
-      - server
    ports:
      - "3030:3030"
    environment:
@@ -39,9 +36,10 @@ services:
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
      - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
      - REDIS_PORT=${REDIS_PORT:-6379}
      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  redis:
@@ -13,7 +13,6 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

  db:
@@ -122,39 +122,59 @@ types:
      observationId: optional<string>
      timestamp: datetime
      comment: optional<string>
-  Dataset:
+  DatasetCore:
    properties:
      id: string
      name: string
+      description: optional<string>
      projectId: string
      createdAt: datetime
      updatedAt: datetime
+  DatasetWithReferences: # GET datasets
+    extends: DatasetCore
+    properties:
+      items:
+        type: list<string>
+        docs: list of dataset item ids
+      runs:
+        type: list<string>
+        docs: list of dataset run names
+  Dataset: # GET datasets/[name]
+    docs: Dataset including all items
+    extends: DatasetCore
+    properties:
      items: list<DatasetItem>
      runs: list<string>
  DatasetItem:
    properties:
      id: string
      status: DatasetStatus
-      input: unknown
+      input: optional<unknown>
      expectedOutput: optional<unknown>
+      sourceTraceId: optional<string>
      sourceObservationId: optional<string>
      datasetId: string
+      datasetName: string
      createdAt: datetime
      updatedAt: datetime
  DatasetRunItem:
    properties:
      id: string
      datasetRunId: string
+      datasetRunName: string
      datasetItemId: string
-      observationId: string
+      traceId: string
+      observationId: optional<string>
      createdAt: datetime
      updatedAt: datetime
  DatasetRun:
    properties:
      id: string
      name: string
+      description: optional<string>
      metadata: optional<unknown>
      datasetId: string
+      datasetName: string
      createdAt: datetime
      updatedAt: datetime
      datasetRunItems: list<DatasetRunItem>
@@ -24,7 +24,7 @@ types:
  CreateDatasetItemRequest:
    properties:
      datasetName: string
-      input: unknown
+      input: optional<unknown>
      expectedOutput: optional<unknown>
      id:
        type: optional<string>
@@ -15,8 +15,14 @@ types:
  CreateDatasetRunItemRequest:
    properties:
      runName: string
+      runDescription:
+        type: optional<string>
+        docs: Description of the run. If run exists, description will be updated.
      metadata:
        type: optional<unknown>
        docs: Metadata of the dataset run, updates run if run already exists
      datasetItemId: string
-      observationId: string
+      observationId: optional<string>
+      traceId:
+        type: optional<string>
+        docs: traceId should always be provided. For compatibility with older SDK versions it can also be inferred from the provided observationId.
@@ -1,10 +1,21 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
  auth: true
  base-path: /api/public
  endpoints:
+    list:
+      method: GET
+      docs: Get all datasets
+      path: /datasets
+      request:
+        name: GetDatasetsRequest
+        query-parameters:
+          page: optional<integer>
+          limit: optional<integer>
+      response: PaginatedDatasets
    get:
      method: GET
      docs: Get a dataset and its items
@@ -28,6 +39,11 @@ service:
      response: commons.DatasetRun

 types:
+  PaginatedDatasets:
+    properties:
+      data: list<commons.DatasetWithReferences>
+      meta: pagination.MetaResponse
  CreateDatasetRequest:
    properties:
      name: string
+      description: optional<string>
@@ -13,8 +13,12 @@ service:
      request:
        name: GetDailyMetricsRequest
        query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
          traceName:
            type: optional<string>
            docs: Optional filter by the name of the trace
@@ -22,13 +22,20 @@ service:
      request:
        name: GetObservationsRequest
        query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
          name: optional<string>
          userId: optional<string>
          type: optional<string>
          traceId: optional<string>
          parentObservationId: optional<string>
+          fromStartTime:
+            type: optional<datetime>
+            docs: Retrieve only observations with a start_time greater than this timestamp.
      response: ObservationsViews

 types:
@@ -25,6 +25,20 @@ service:

 types:
  CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
    properties:
      name: string
      isActive:
@@ -34,8 +48,29 @@ types:
      config: optional<unknown>

  Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
    properties:
      name: string
      version: integer
-      prompt: string
      config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>
@@ -19,10 +19,17 @@ service:
      request:
        name: GetScoresRequest
        query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
          userId: optional<string>
          name: optional<string>
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Retrieve only scores newer than this timestamp.
      response: Scores
    get-by-id:
      docs: Get a score
@@ -22,10 +22,17 @@ service:
      request:
        name: GetTracesRequest
        query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
          userId: optional<string>
          name: optional<string>
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Retrieve only traces newer than this timestamp.
          orderBy:
            type: optional<string>
            docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"
@@ -11,14 +11,16 @@ groups:
          namespaceExport: Langfuse
          allowCustomFetcher: true
      - name: fernapi/fern-python-sdk
-        version: 0.7.3
+        version: 1.1.0-rc2
        output:
          location: local-file-system
          path: ../../../generated/python
        config:
          client_class_name: FernLangfuse
+          improved_imports: false
          pydantic_config:
            require_optional_fields: false
+            use_str_enums: false
      - name: fernapi/fern-typescript-node-sdk
        version: 0.7.1
        output:
@@ -1,4 +1,4 @@
 {
  "organization": "finto",
-  "version": "0.16.36"
+  "version": "0.19.30"
 }
@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer
@@ -137,7 +137,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"runName\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\"\n}",
+              "raw": "{\n    \"runName\": \"example\",\n    \"runDescription\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\",\n    \"traceId\": \"example\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -154,6 +154,42 @@
      "description": null,
      "name": "Datasets",
      "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all datasets",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/datasets?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "datasets"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": null
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
        {
          "_type": "endpoint",
          "name": "Get",
@@ -209,7 +245,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"name\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -357,12 +393,12 @@
                {
                  "key": "page",
                  "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                },
                {
                  "key": "limit",
                  "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                },
                {
                  "key": "traceName",
@@ -434,7 +470,7 @@
          "request": {
            "description": "Get a list of observations",
            "url": {
-              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=",
+              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -447,12 +483,12 @@
                {
                  "key": "page",
                  "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                },
                {
                  "key": "limit",
                  "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                },
                {
                  "key": "name",
@@ -478,6 +514,11 @@
                  "key": "parentObservationId",
                  "value": "",
                  "description": null
+                },
+                {
+                  "key": "fromStartTime",
+                  "value": "",
+                  "description": "Retrieve only observations with a start_time greater than this timestamp."
                }
              ],
              "variable": []
@@ -587,7 +628,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -643,7 +684,7 @@
          "request": {
            "description": "Get a list of scores",
            "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -656,12 +697,12 @@
                {
                  "key": "page",
                  "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                },
                {
                  "key": "limit",
                  "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                },
                {
                  "key": "userId",
@@ -672,6 +713,11 @@
                  "key": "name",
                  "value": "",
                  "description": null
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Retrieve only scores newer than this timestamp."
                }
              ],
              "variable": []
@@ -831,7 +877,7 @@
          "request": {
            "description": "Get list of traces",
            "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -844,12 +890,12 @@
                {
                  "key": "page",
                  "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                },
                {
                  "key": "limit",
                  "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                },
                {
                  "key": "userId",
@@ -861,6 +907,11 @@
                  "value": "",
                  "description": null
                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Retrieve only traces newer than this timestamp."
+                },
                {
                  "key": "orderBy",
                  "value": "",
@@ -1,31 +1,72 @@
 {
  "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.25.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
+  "private": true,
  "engines": {
    "node": ">=20"
  },
  "scripts": {
    "preinstall": "npx only-allow pnpm",
-    "infra:dev:up": "docker-compose -f ./docker-compose.dev.yml up -d",
-    "infra:dev:down": "docker-compose -f ./docker-compose.dev.yml down",
+    "infra:dev:up": "docker compose -f ./docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker compose -f ./docker-compose.dev.yml down",
    "db:generate": "turbo run db:generate",
    "db:migrate": "turbo run db:migrate",
    "db:seed": "turbo run db:seed",
    "db:seed:examples": "turbo run db:seed:examples",
    "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=worker run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
    "build": "turbo run build",
    "start": "turbo run start",
    "dev": "turbo run dev",
    "lint": "turbo run lint",
    "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate"
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
  },
  "devDependencies": {
-    "dotenv-cli": "^7.3.0",
+    "@release-it/bumper": "^6.0.1",
+    "dotenv-cli": "^7.4.1",
    "prettier": "^3.2.5",
-    "turbo": "^1.13.0"
+    "release-it": "^17.1.1",
+    "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
  }
 }
@@ -8,12 +8,13 @@
    "next.js"
  ],
  "devDependencies": {
-    "@vercel/style-guide": "^5.2.0",
-    "eslint-config-turbo": "^1.12.4",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-only-warn": "^1.1.0",
-    "@typescript-eslint/parser": "^7.1.0",
    "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "typescript": "^5.3.3"
+    "@typescript-eslint/parser": "^7.6.0",
+    "@vercel/style-guide": "^6.0.0",
+    "eslint-config-next": "^14.2.1",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-turbo": "^1.13.2",
+    "eslint-plugin-only-warn": "^1.1.0",
+    "typescript": "^5.4.5"
  }
 }
@@ -5,12 +5,23 @@
  "private": true,
  "main": "./dist/src/index.js",
  "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
  "engines": {
    "node": ">=20"
  },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    }
+  },
  "scripts": {
    "build": "tsc",
    "dev": "tsc --watch",
@@ -28,26 +39,26 @@
    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
  },
  "dependencies": {
-    "@langchain/anthropic": "^0.1.8",
-    "@langchain/core": "^0.1.49",
-    "@langchain/openai": "^0.0.23",
-    "@prisma/client": "^5.11.0",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.57",
+    "@langchain/openai": "^0.0.28",
+    "@prisma/client": "^5.12.1",
    "@types/bcryptjs": "^2.4.6",
    "bcryptjs": "^2.4.3",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.28",
+    "langchain": "^0.1.33",
    "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
  },
  "devDependencies": {
    "@repo/eslint-config": "*",
    "@repo/typescript-config": "*",
    "@types/lodash": "^4.14.202",
    "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.2",
+    "@types/pg": "^8.11.5",
    "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.2.0",
+    "@typescript-eslint/parser": "^7.6.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
@@ -55,12 +66,12 @@
    "kysely-codegen": "^0.11.0",
    "nodemon": "^3.0.3",
    "prettier": "^3.2.5",
-    "prisma": "^5.11.0",
+    "prisma": "^5.12.1",
    "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
    "ts-node": "^10.9.2",
-    "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
-    "typescript": "^5.3.3",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5",
    "vitest": "^1.3.1"
  }
 }
@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
    API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,22 @@ export const DatasetStatus = {
    ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobConfigState = {
+    ACTIVE: "ACTIVE",
+    INACTIVE: "INACTIVE"
+} as const;
+export type JobConfigState = (typeof JobConfigState)[keyof typeof JobConfigState];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
    id: string;
    user_id: string;
@@ -95,6 +112,7 @@ export type CronJobs = {
 export type Dataset = {
    id: string;
    name: string;
+    description: string | null;
    project_id: string;
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
@@ -102,8 +120,9 @@ export type Dataset = {
 export type DatasetItem = {
    id: string;
    status: Generated<DatasetStatus>;
-    input: unknown;
+    input: unknown | null;
    expected_output: unknown | null;
+    source_trace_id: string | null;
    source_observation_id: string | null;
    dataset_id: string;
    created_at: Generated<Timestamp>;
@@ -113,18 +132,33 @@ export type DatasetRunItems = {
    id: string;
    dataset_run_id: string;
    dataset_item_id: string;
-    observation_id: string;
+    trace_id: string;
+    observation_id: string | null;
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
 };
 export type DatasetRuns = {
    id: string;
    name: string;
+    description: string | null;
    metadata: unknown | null;
    dataset_id: string;
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
    id: string;
    created_at: Generated<Timestamp>;
@@ -135,6 +169,34 @@ export type Events = {
    url: string | null;
    method: string | null;
 };
+export type JobConfiguration = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    status: Generated<JobConfigState>;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
+};
 export type Membership = {
    project_id: string;
    user_id: string;
@@ -249,11 +311,13 @@ export type Prompt = {
    updated_at: Generated<Timestamp>;
    project_id: string;
    created_by: string;
-    prompt: string;
+    prompt: unknown;
    name: string;
    version: number;
+    type: Generated<string>;
    is_active: boolean;
    config: Generated<unknown>;
+    tags: Generated<string[]>;
 };
 export type Score = {
    id: string;
@@ -340,7 +404,10 @@ export type DB = {
    dataset_run_items: DatasetRunItems;
    dataset_runs: DatasetRuns;
    datasets: Dataset;
+    eval_templates: EvalTemplate;
    events: Events;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
    membership_invitations: MembershipInvitation;
    memberships: Membership;
    models: Model;
@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "dataset_run_items" ADD COLUMN     "trace_id" TEXT,
+ALTER COLUMN "observation_id" DROP NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_trace_id_fkey" FOREIGN KEY ("trace_id") REFERENCES "traces"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,5 @@
+-- Backfill trace_id for existing run_items based on the linked observation
+UPDATE dataset_run_items
+SET trace_id = observations.trace_id
+FROM observations 
+WHERE dataset_run_items.observation_id = observations.id;
@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Made the column `trace_id` on table `dataset_run_items` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "dataset_run_items" ALTER COLUMN "trace_id" SET NOT NULL;
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_trace_id_idx" ON "dataset_run_items"("trace_id");
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ALTER COLUMN "input" DROP NOT NULL;
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "source_trace_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "dataset_items" ADD CONSTRAINT "dataset_items_source_trace_id_fkey" FOREIGN KEY ("source_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_runs" ADD COLUMN     "description" TEXT;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "description" TEXT;
@@ -0,0 +1,7 @@
+-- Backfill source_trace_id for existing dataset_items based on the linked source_observation_id
+UPDATE dataset_items
+SET source_trace_id = observations.trace_id
+FROM observations
+WHERE dataset_items.source_observation_id = observations.id
+  AND dataset_items.source_observation_id IS NOT NULL
+  AND dataset_items.source_trace_id IS NULL
@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;
@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_tags_idx" ON "prompts" USING GIN ("tags" array_ops);
@@ -0,0 +1,30 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://ai.google.dev/models/gemini#model-variations
+  -- https://cloud.google.com/vertex-ai/generative-ai/pricing
+  ('cluv2sjeo000008ih0fv23hi0', NULL, 'gemini-1.0-pro-latest', '(?i)^(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  -- stable versions priced differently
+  -- https://developers.googleblog.com/2024/02/gemini-15-available-for-private-preview-in-google-ai-studio.html
+  ('cluv2subq000108ih2mlrga6a', NULL, 'gemini-1.0-pro', '(?i)^(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2sx04000208ihbek75lsz', NULL, 'gemini-1.0-pro-001', '(?i)^(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2szw0000308ihch3n79x7', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+
+  ('cluv2t2x0000408ihfytl45l1', NULL, 'gemini-1.5-pro-latest', '(?i)^(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
+
+
+  
@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "JobConfigState" AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- AlterTable
+ALTER TABLE "job_configurations" ADD COLUMN     "status" "JobConfigState" NOT NULL DEFAULT 'ACTIVE';
@@ -0,0 +1,16 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluvpl4ls000008l6h2gx3i07', NULL, 'gpt-4-turbo', '(?i)^(gpt-4-turbo)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
@@ -92,22 +92,25 @@ model VerificationToken {
 }

 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id               String                 @id @default(cuid())
+    createdAt        DateTime               @default(now()) @map("created_at")
+    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
+    name             String
+    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members          Membership[]
+    traces           Trace[]
+    observations     Observation[]
+    apiKeys          ApiKey[]
+    dataset          Dataset[]
+    RawEvents        Events[]
+    invitations      MembershipInvitation[]
+    sessions         TraceSession[]
+    Prompt           Prompt[]
+    Model            Model[]
+    AuditLog         AuditLog[]
+    EvalTemplate     EvalTemplate[]
+    JobConfiguration JobConfiguration[]
+    JobExecution     JobExecution[]

    @@map("projects")
 }
@@ -206,7 +209,10 @@ model Trace {
    sessionId  String?       @map("session_id")
    session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])

-    scores Score[]
+    scores          Score[]
+    DatasetRunItems DatasetRunItems[]
+    DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]

    @@index([projectId])
    @@index([sessionId])
@@ -360,16 +366,17 @@ enum ObservationLevel {
 }

 model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
    name          String
    value         Float
    source        ScoreSource
    comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]

    @@unique([id, traceId]) // used for upsert
    @@index(timestamp)
@@ -383,6 +390,7 @@ model Score {
 enum ScoreSource {
    API
    REVIEW
+    EVAL
 }

 enum PricingUnit {
@@ -420,6 +428,7 @@ model CronJobs {
 model Dataset {
    id           String        @id @default(cuid())
    name         String
+    description  String?
    projectId    String        @map("project_id")
    project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
    createdAt    DateTime      @default(now()) @map("created_at")
@@ -435,8 +444,10 @@ model Dataset {
 model DatasetItem {
    id                  String            @id @default(cuid())
    status              DatasetStatus     @default(ACTIVE)
-    input               Json
+    input               Json?
    expectedOutput      Json?             @map("expected_output")
+    sourceTraceId       String?           @map("source_trace_id")
+    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
    sourceObservationId String?           @map("source_observation_id")
    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
    datasetId           String            @map("dataset_id")
@@ -458,6 +469,7 @@ enum DatasetStatus {
 model DatasetRuns {
    id              String            @id @default(cuid())
    name            String
+    description     String?
    metadata        Json?
    datasetId       String            @map("dataset_id")
    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
@@ -471,19 +483,22 @@ model DatasetRuns {
 }

 model DatasetRunItems {
-    id            String      @id @default(cuid())
-    datasetRunId  String      @map("dataset_run_id")
-    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String      @map("dataset_item_id")
-    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    observationId String      @map("observation_id")
-    observation   Observation @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime    @default(now()) @map("created_at")
-    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
+    id            String       @id @default(cuid())
+    datasetRunId  String       @map("dataset_run_id")
+    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String       @map("dataset_item_id")
+    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String       @map("trace_id")
+    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?      @map("observation_id")
+    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
+    createdAt     DateTime     @default(now()) @map("created_at")
+    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")

    @@index([datasetRunId], type: Hash)
    @@index([datasetItemId], type: Hash)
    @@index([observationId], type: Hash)
+    @@index([traceId])
    @@map("dataset_run_items")
 }

@@ -512,17 +527,20 @@ model Prompt {

    createdBy String @map("created_by")

-    prompt      String
+    prompt      Json
    name        String
    version     Int
+    type        String        @default("text")
    isActive    Boolean       @map("is_active")
    config      Json          @default("{}")
+    tags        String[]      @default([])
    Observation Observation[]

    @@unique([projectId, name, version])
    @@index([projectId, name, version])
    @@index([projectId, id])
    @@index([projectId])
+    @@index([tags(ops: ArrayOps)], type: Gin)
    @@map("prompts")
 }

@@ -570,3 +588,95 @@ model AuditLog {
    @@index([createdAt])
    @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+enum JobConfigState {
+    ACTIVE
+    INACTIVE
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType        @map("job_type")
+    status         JobConfigState @default(ACTIVE)
+    evalTemplateId String?        @map("eval_template_id")
+    evalTemplate   EvalTemplate?  @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}
@@ -4,13 +4,14 @@ import {
  type Prisma,
  ObservationType,
  ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";

 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";

 const LOAD_TRACE_VOLUME = 10_000;

@@ -149,15 +150,14 @@ async function main() {

    const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;

-    const { generationIds, traces, observations, scores, sessions, events } =
-      createObjects(
-        traceVolume,
-        envTags,
-        colorTags,
-        project1,
-        project2,
-        promptIds
-      );
+    const { traces, observations, scores, sessions, events } = createObjects(
+      traceVolume,
+      envTags,
+      colorTags,
+      project1,
+      project2,
+      promptIds
+    );

    console.log(
      `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
@@ -165,32 +165,109 @@ async function main() {

    await uploadObjects(traces, observations, scores, sessions, events);

+    // add eval objects
+    const evalTemplate = await prisma.evalTemplate.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project1.id,
+          name: "toxicity-template",
+          version: 1,
+        },
+      },
+      create: {
+        projectId: project1.id,
+        name: "toxicity-template",
+        version: 1,
+        prompt:
+          "Please evaluate the toxicity of the following text {{input}} {{output}}",
+        model: "gpt-3.5-turbo",
+        vars: ["input", "output"],
+        outputSchema: {
+          score: "provide a score between 0 and 1",
+          reasoning: "one sentence reasoning for the score",
+        },
+        modelParams: {
+          temperature: 0.7,
+          outputTokenLimit: 100,
+          topP: 0.9,
+        },
+      },
+      update: {},
+    });
+
+    await prisma.jobConfiguration.upsert({
+      where: {
+        id: "toxicity-job",
+      },
+      create: {
+        id: "toxicity-job",
+        evalTemplateId: evalTemplate.id,
+        projectId: project1.id,
+        jobType: "EVAL",
+        status: "ACTIVE",
+        scoreName: "toxicity",
+        filter: [
+          {
+            type: "string",
+            value: "user",
+            column: "User ID",
+            operator: "contains",
+          },
+        ],
+        variableMapping: [
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "input",
+            templateVariable: "input",
+          },
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "metadata",
+            templateVariable: "output",
+          },
+        ],
+        targetObject: "trace",
+        sampling: 1,
+        delay: 5_000,
+      },
+      update: {},
+    });
+
    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
      const dataset = await prisma.dataset.create({
        data: {
          name: `demo-dataset-${datasetNumber}`,
+          description:
+            datasetNumber === 0 ? "Dataset test description" : undefined,
          projectId: project2.id,
        },
      });

      const datasetItemIds = [];
      for (let i = 0; i < 18; i++) {
-        const sourceObservationId =
-          Math.random() > 0.5
-            ? generationIds[Math.floor(Math.random() * generationIds.length)]
+        const sourceObservation =
+          Math.random() > 0.3
+            ? observations[Math.floor(Math.random() * observations.length)]
            : undefined;
        const datasetItem = await prisma.datasetItem.create({
          data: {
            datasetId: dataset.id,
-            sourceObservationId: sourceObservationId,
-            input: [
-              {
-                role: "user",
-                content: "How can i create a React component?",
-              },
-            ],
+            sourceTraceId: sourceObservation?.traceId,
+            sourceObservationId:
+              Math.random() > 0.5 ? sourceObservation?.id : undefined,
+            input:
+              Math.random() > 0.3
+                ? [
+                    {
+                      role: "user",
+                      content: "How can i create a React component?",
+                    },
+                  ]
+                : undefined,
            expectedOutput:
-              "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.",
+              Math.random() > 0.3
+                ? "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both."
+                : undefined,
          },
        });
        datasetItemIds.push(datasetItem.id);
@@ -200,6 +277,7 @@ async function main() {
        const datasetRun = await prisma.datasetRuns.create({
          data: {
            name: `demo-dataset-run-${datasetRunNumber}`,
+            description: Math.random() > 0.5 ? "Dataset run description" : "",
            datasetId: dataset.id,
            metadata: [
              undefined,
@@ -212,13 +290,19 @@ async function main() {
        });

        for (const datasetItemId of datasetItemIds) {
-          const runObservationId =
-            generationIds[Math.floor(Math.random() * generationIds.length)];
+          const relevantObservations = observations.filter(
+            (o) => o.projectId === project2.id
+          );
+          const observation =
+            relevantObservations[
+              Math.floor(Math.random() * relevantObservations.length)
+            ];

          await prisma.datasetRunItems.create({
            data: {
              datasetItemId,
-              observationId: runObservationId,
+              traceId: observation.traceId as string,
+              observationId: Math.random() > 0.5 ? observation.id : undefined,
              datasetRunId: datasetRun.id,
            },
          });
@@ -354,7 +438,6 @@ function createObjects(
  const observations: Prisma.ObservationCreateManyInput[] = [];
  const scores: Prisma.ScoreCreateManyInput[] = [];
  const sessions: Prisma.TraceSessionCreateManyInput[] = [];
-  const generationIds: string[] = [];
  const events: Prisma.ObservationCreateManyInput[] = [];

  for (let i = 0; i < traceVolume; i++) {
@@ -618,8 +701,6 @@ function createObjects(
            source: ScoreSource.API,
          });

-        generationIds.push(generation.id);
-
        for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
          // random start time within span
          const eventTs = new Date(
@@ -651,7 +732,6 @@ function createObjects(
  ).map((session) => JSON.parse(session) as Prisma.TraceSessionCreateManyInput);

  return {
-    generationIds,
    traces,
    observations,
    scores,
@@ -702,6 +782,16 @@ async function generatePrompts(project: Project) {
      version: 1,
      isActive: true,
    },
+    {
+      id: `prompt-${v4()}`,
+      projectId: project.id,
+      createdBy: "user-1",
+      prompt: "Prompt 4 content",
+      name: "Prompt 4",
+      version: 1,
+      isActive: true,
+      tags: ["tag1", "tag2"],
+    },
  ];

  for (const prompt of prompts) {
@@ -721,6 +811,7 @@ async function generatePrompts(project: Project) {
        name: prompt.name,
        version: prompt.version,
        isActive: prompt.isActive,
+        tags: prompt.tags,
      },
      update: {
        id: prompt.id,
@@ -3,11 +3,20 @@

 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";

 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices

 const prismaClientSingleton = () => {
+  console.log("Creating new PrismaClient");
  return new PrismaClient({
    log:
      env.NODE_ENV === "development"
@@ -16,12 +25,33 @@ const prismaClientSingleton = () => {
  });
 };

+const kyselySingleton = (prismaClient: PrismaClient) => {
+  console.log("Creating new Kysely client");
+  return prismaClient.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    })
+  );
+};
 declare global {
  // eslint-disable-next-line no-var
  var prisma: undefined | ReturnType<typeof prismaClientSingleton>;
+  var kyselyPrisma: undefined | ReturnType<typeof kyselySingleton>;
 }

 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+export const kyselyPrisma = globalThis.kyselyPrisma ?? kyselySingleton(prisma);
+
 export * from "@prisma/client";

 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;
@@ -0,0 +1,31 @@
+import z from "zod";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(),
+    objectName: z.string().nullish(), // can be null as this is only required for langfuseObjects other than trace
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});
@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
  type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";

 const operatorReplacements = {
  "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
  if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const internalFilters = filters.map((filter) => {
    // Get column definition to map column to internal name, e.g. "t.id"
    const col = tableColumns.find(
      (c) =>
        // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
    );
    if (!col) {
      console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
        ? Prisma.raw(
            arrayOperatorReplacements[
              filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
          )
        : filter.operator in operatorReplacements
          ? Prisma.raw(
              operatorReplacements[
                filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
            )
          : Prisma.raw(filter.operator); //checked by zod

@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
        break;
      case "stringOptions":
        valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
        )})`;
        break;
      case "arrayOptions":
        valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
          filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
        )}] `;
        break;

@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
      filter.type === "string" || filter.type === "stringObject"
        ? [
            ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw("'%' || ")
              : Prisma.empty,
            ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw(" || '%'")
              : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(

 const castValueToPostgresTypes = (
  column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
  return column.name === "type" &&
    (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
  safeColumn: string,
  operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
  if (!dateOperators.includes(operator)) {
    throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
  }

  return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
  )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };
@@ -1,9 +1,19 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
-
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./features/evals/types";
+export * from "./observationsTable";
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
+export * from "./server/evals/types";

-export { ModelUsageUnit };
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";
@@ -20,7 +20,7 @@ export const filterOperators = {
 export const timeFilter = z.object({
  column: z.string(),
  operator: z.enum(filterOperators.datetime),
-  value: z.date(),
+  value: z.coerce.date(), // coerce required to parse stringified dates from the db in evals
  type: z.literal("datetime"),
 });
 export const stringFilter = z.object({
@@ -41,6 +41,7 @@ export const tableNames = [
  "traces_metrics",
  "traces_parent_observation_scores",
  "sessions",
+  "prompts",
 ] as const;

 export type TableNames = (typeof tableNames)[number];
@@ -1,8 +1,8 @@
+import { ObservationLevel } from "@prisma/client";
 import {
  type OptionsDefinition,
  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";

 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -164,7 +164,7 @@ export type ObservationOptions = {
 };

 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
  return observationsTableCols.map((col) => {
    if (col.id === "model") {
@@ -0,0 +1,41 @@
+import { z } from "zod";
+
+export const QueueEnvelope = z.object({
+  timestamp: z.string().datetime({ offset: true }),
+  id: z.string(),
+});
+
+export const TraceUpsertEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    traceId: z.string(),
+  }),
+});
+
+export const EvalExecutionEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    jobExecutionId: z.string(),
+  }),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};
@@ -0,0 +1,80 @@
+import z from "zod";
+import { ModelProvider, observationsTableCols, tracesTableCols } from "../..";
+
+export const evalObjects = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];
+
+export const evalModels = [
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-3.5-turbo",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-4-turbo-preview",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+] as const;
+
+export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;
@@ -14,13 +14,17 @@ import { ChatOpenAI } from "@langchain/openai";
 import {
  ChatMessage,
  ChatMessageRole,
+  LLMFunctionCall,
  ModelParams,
  ModelProvider,
 } from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";

 type LLMCompletionParams = {
  messages: ChatMessage[];
  modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
 };

 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -30,18 +34,27 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
  params: LLMCompletionParams & {
    streaming: true;
+    functionCall: undefined;
  }
 ): Promise<IterableReadableStream<Uint8Array>>;

 export async function fetchLLMCompletion(
  params: LLMCompletionParams & {
    streaming: false;
+    functionCall: undefined;
  }
 ): Promise<string>;

+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array>> {
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
  const { messages, modelParams, streaming } = params;
  const finalMessages = messages.map((message) => {
    if (message.role === ChatMessageRole.User)
@@ -71,6 +84,22 @@ export async function fetchLLMCompletion(
    });
  }

+  console.log("Making LLM call with params: ", modelParams);
+
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
  if (streaming) {
    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
  }
@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
  role: ChatMessageRole;
  content: string;
@@ -23,11 +26,13 @@ export type UIModelParams = Required<
 };

 // Generic config
-export type ModelConfig = {
-  max_tokens?: number;
-  temperature?: number;
-  top_p?: number;
-};
+export type ModelConfig = z.infer<typeof ZodModelConfig>;
+
+export const ZodModelConfig = z.object({
+  max_tokens: z.number().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+});

 // OpenAI
 export type OpenAIModelParams = {
@@ -72,3 +77,9 @@ export const supportedModels = {
  [ModelProvider.Anthropic]: anthropicModels,
  [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};
@@ -1,8 +1,5 @@
-import {
-  type OptionsDefinition,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
 import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";

 export const tracesTableCols: ColumnDefinition[] = [
  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
@@ -122,7 +119,7 @@ export type TraceOptions = {
 };

 export function tracesTableColsWithOptions(
-  options?: TraceOptions,
+  options?: TraceOptions
 ): ColumnDefinition[] {
  return tracesTableCols.map((col) => {
    if (col.id === "scores_avg") {
@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";

 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;
@@ -29,4 +29,5 @@ export type TableName =
  | "generations"
  | "sessions"
  | "scores"
+  | "prompts"
  | "dashboard";
@@ -6,7 +6,8 @@
    "lib": ["ES2020"],
    "outDir": "./dist",
    "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
  },
  "include": ["."],
  "exclude": ["node_modules", "dist"]
@@ -152,25 +152,32 @@ paths:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateDatasetRunItemRequest'
-  /api/public/datasets/{datasetName}:
+  /api/public/datasets:
    get:
-      description: Get a dataset and its items
-      operationId: datasets_get
+      description: Get all datasets
+      operationId: datasets_list
      tags:
        - Datasets
      parameters:
-        - name: datasetName
-          in: path
-          required: true
+        - name: page
+          in: query
+          required: false
          schema:
-            type: string
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
      responses:
        '200':
          description: ''
          content:
            application/json:
              schema:
-                $ref: '#/components/schemas/Dataset'
+                $ref: '#/components/schemas/PaginatedDatasets'
        '400':
          description: ''
          content:
@@ -197,7 +204,6 @@ paths:
            application/json:
              schema: {}
      security: *ref_0
-  /api/public/datasets:
    post:
      description: Create a dataset
      operationId: datasets_create
@@ -243,6 +249,51 @@ paths:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateDatasetRequest'
+  /api/public/datasets/{datasetName}:
+    get:
+      description: Get a dataset and its items
+      operationId: datasets_get
+      tags:
+        - Datasets
+      parameters:
+        - name: datasetName
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Dataset'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
  /api/public/datasets/{datasetName}/runs/{runName}:
    get:
      description: Get a dataset run and its items
@@ -396,12 +447,14 @@ paths:
      parameters:
        - name: page
          in: query
+          description: page number, starts at 1
          required: false
          schema:
            type: integer
            nullable: true
        - name: limit
          in: query
+          description: limit of items per page
          required: false
          schema:
            type: integer
@@ -519,12 +572,14 @@ paths:
      parameters:
        - name: page
          in: query
+          description: page number, starts at 1
          required: false
          schema:
            type: integer
            nullable: true
        - name: limit
          in: query
+          description: limit of items per page
          required: false
          schema:
            type: integer
@@ -559,6 +614,16 @@ paths:
          schema:
            type: string
            nullable: true
+        - name: fromStartTime
+          in: query
+          description: >-
+            Retrieve only observations with a start_time greater than this
+            timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
      responses:
        '200':
          description: ''
@@ -782,12 +847,14 @@ paths:
      parameters:
        - name: page
          in: query
+          description: page number, starts at 1
          required: false
          schema:
            type: integer
            nullable: true
        - name: limit
          in: query
+          description: limit of items per page
          required: false
          schema:
            type: integer
@@ -804,6 +871,14 @@ paths:
          schema:
            type: string
            nullable: true
+        - name: fromTimestamp
+          in: query
+          description: Retrieve only scores newer than this timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
      responses:
        '200':
          description: ''
@@ -1025,12 +1100,14 @@ paths:
      parameters:
        - name: page
          in: query
+          description: page number, starts at 1
          required: false
          schema:
            type: integer
            nullable: true
        - name: limit
          in: query
+          description: limit of items per page
          required: false
          schema:
            type: integer
@@ -1047,6 +1124,14 @@ paths:
          schema:
            type: string
            nullable: true
+        - name: fromTimestamp
+          in: query
+          description: Retrieve only traces newer than this timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
        - name: orderBy
          in: query
          description: >-
@@ -1394,14 +1479,17 @@ components:
        - value
        - source
        - timestamp
-    Dataset:
-      title: Dataset
+    DatasetCore:
+      title: DatasetCore
      type: object
      properties:
        id:
          type: string
        name:
          type: string
+        description:
+          type: string
+          nullable: true
        projectId:
          type: string
        createdAt:
@@ -1410,6 +1498,36 @@ components:
        updatedAt:
          type: string
          format: date-time
+      required:
+        - id
+        - name
+        - projectId
+        - createdAt
+        - updatedAt
+    DatasetWithReferences:
+      title: DatasetWithReferences
+      type: object
+      properties:
+        items:
+          type: array
+          items:
+            type: string
+          description: list of dataset item ids
+        runs:
+          type: array
+          items:
+            type: string
+          description: list of dataset run names
+      required:
+        - items
+        - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
+    Dataset:
+      title: Dataset
+      type: object
+      description: Dataset including all items
+      properties:
        items:
          type: array
          items:
@@ -1419,13 +1537,10 @@ components:
          items:
            type: string
      required:
-        - id
-        - name
-        - projectId
-        - createdAt
-        - updatedAt
        - items
        - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
    DatasetItem:
      title: DatasetItem
      type: object
@@ -1434,14 +1549,20 @@ components:
          type: string
        status:
          $ref: '#/components/schemas/DatasetStatus'
-        input: {}
+        input:
+          nullable: true
        expectedOutput:
          nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
        sourceObservationId:
          type: string
          nullable: true
        datasetId:
          type: string
+        datasetName:
+          type: string
        createdAt:
          type: string
          format: date-time
@@ -1451,8 +1572,8 @@ components:
      required:
        - id
        - status
-        - input
        - datasetId
+        - datasetName
        - createdAt
        - updatedAt
    DatasetRunItem:
@@ -1463,10 +1584,15 @@ components:
          type: string
        datasetRunId:
          type: string
+        datasetRunName:
+          type: string
        datasetItemId:
          type: string
+        traceId:
+          type: string
        observationId:
          type: string
+          nullable: true
        createdAt:
          type: string
          format: date-time
@@ -1476,8 +1602,9 @@ components:
      required:
        - id
        - datasetRunId
+        - datasetRunName
        - datasetItemId
-        - observationId
+        - traceId
        - createdAt
        - updatedAt
    DatasetRun:
@@ -1488,10 +1615,15 @@ components:
          type: string
        name:
          type: string
+        description:
+          type: string
+          nullable: true
        metadata:
          nullable: true
        datasetId:
          type: string
+        datasetName:
+          type: string
        createdAt:
          type: string
          format: date-time
@@ -1506,6 +1638,7 @@ components:
        - id
        - name
        - datasetId
+        - datasetName
        - createdAt
        - updatedAt
        - datasetRunItems
@@ -1558,7 +1691,8 @@ components:
      properties:
        datasetName:
          type: string
-        input: {}
+        input:
+          nullable: true
        expectedOutput:
          nullable: true
        id:
@@ -1567,13 +1701,16 @@ components:
          description: Dataset items are upserted on their id
      required:
        - datasetName
-        - input
    CreateDatasetRunItemRequest:
      title: CreateDatasetRunItemRequest
      type: object
      properties:
        runName:
          type: string
+        runDescription:
+          type: string
+          nullable: true
+          description: Description of the run. If run exists, description will be updated.
        metadata:
          nullable: true
          description: Metadata of the dataset run, updates run if run already exists
@@ -1581,16 +1718,38 @@ components:
          type: string
        observationId:
          type: string
+          nullable: true
+        traceId:
+          type: string
+          nullable: true
+          description: >-
+            traceId should always be provided. For compatibility with older SDK
+            versions it can also be inferred from the provided observationId.
      required:
        - runName
        - datasetItemId
-        - observationId
+    PaginatedDatasets:
+      title: PaginatedDatasets
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetWithReferences'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
    CreateDatasetRequest:
      title: CreateDatasetRequest
      type: object
      properties:
        name:
          type: string
+        description:
+          type: string
+          nullable: true
      required:
        - name
    HealthResponse:
@@ -2259,6 +2418,50 @@ components:
        - name
    CreatePromptRequest:
      title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
      type: object
      properties:
        name:
@@ -2276,20 +2479,75 @@ components:
        - prompt
    Prompt:
      title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
      type: object
      properties:
        name:
          type: string
        version:
          type: integer
-        prompt:
-          type: string
        config: {}
      required:
        - name
        - version
-        - prompt
        - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
    CreateScoreRequest:
      title: CreateScoreRequest
      type: object
@@ -6,6 +6,25 @@ await import("./src/env.mjs");
 import { withSentryConfig } from "@sentry/nextjs";
 import { env } from "./src/env.mjs";

+/**
+ * CSP headers
+ * img-src https to allow loading images from SSO providers
+ */
+const cspHeader = `
+  default-src 'self' https://ph.langfuse.com https://*.posthog.com wss://*.crisp.chat https://*.crisp.chat;
+  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  style-src 'self' 'unsafe-inline' https://*.crisp.chat;
+  img-src 'self' https: blob: data:;
+  font-src 'self' https://*.crisp.chat;
+  frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com;
+  worker-src 'self' blob:;
+  object-src 'none';
+  base-uri 'self';
+  form-action 'self';
+  frame-ancestors 'none';
+  ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
+`;
+
 /** @type {import("next").NextConfig} */
 const nextConfig = {
  transpilePackages: ["@langfuse/shared"],
@@ -32,27 +51,48 @@ const nextConfig = {
            key: "x-frame-options",
            value: "SAMEORIGIN",
          },
+          {
+            key: "X-Content-Type-Options",
+            value: "nosniff",
+          },
+          {
+            key: "Referrer-Policy",
+            value: "strict-origin-when-cross-origin",
+          },
+          {
+            key: "Permissions-Policy",
+            value: "autoplay=*, fullscreen=*, microphone=*",
+          },
+        ],
+      },
+      {
+        source: "/:path((?!api).*)*",
+        headers: [
+          {
+            key: "Content-Security-Policy",
+            value: cspHeader.replace(/\n/g, ""),
+          },
        ],
      },
      // Required to check authentication status from langfuse.com
      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
        ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
        : []),
    ];
  },
@@ -1,14 +1,14 @@
 {
  "name": "web",
-  "version": "2.14.2",
+  "version": "2.25.0",
  "private": true,
  "license": "MIT",
  "engines": {
    "node": ">=20"
  },
  "scripts": {
-    "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
-    "build": "dotenv -e ../.env -- next build",
+    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
+    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
    "dev": "dotenv -e ../.env -- next dev",
    "lint": "dotenv -e ../.env -- next lint",
    "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -18,22 +18,22 @@
    "test": "dotenv -e ../.env -- jest --runInBand",
    "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
    "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.515.0",
-    "@aws-sdk/s3-request-presigner": "^3.507.0",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@codemirror/lang-json": "^6.0.1",
    "@headlessui/react": "^1.7.18",
-    "@heroicons/react": "^2.1.1",
+    "@heroicons/react": "^2.1.3",
    "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.8",
-    "@langchain/core": "^0.1.49",
-    "@langchain/openai": "^0.0.23",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.57",
+    "@langchain/openai": "^0.0.28",
    "@langfuse/shared": "workspace:*",
-    "@marsidev/react-turnstile": "^0.5.3",
+    "@marsidev/react-turnstile": "^0.5.4",
    "@next-auth/prisma-adapter": "^1.0.7",
    "@radix-ui/react-accordion": "^1.1.2",
    "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -55,14 +55,14 @@
    "@radix-ui/react-tabs": "^1.0.4",
    "@radix-ui/react-toggle": "^1.0.3",
    "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.14",
+    "@react-email/components": "^0.0.16",
    "@react-email/render": "^0.0.12",
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.102.1",
-    "@sentry/node": "^7.107.0",
-    "@sentry/profiling-node": "^7.102.1",
-    "@sentry/types": "^7.88.0",
+    "@sentry/nextjs": "^7.110.0",
+    "@sentry/node": "^7.110.0",
+    "@sentry/profiling-node": "^7.110.0",
+    "@sentry/types": "^7.110.0",
    "@t3-oss/env-nextjs": "^0.8.0",
    "@tailwindcss/forms": "^0.5.7",
    "@tanstack/react-query": "^4.36.1",
@@ -72,36 +72,38 @@
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
-    "ai": "^3.0.11",
+    "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/react-codemirror": "^4.21.25",
+    "ai": "^3.0.21",
    "bcryptjs": "^2.4.3",
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.0",
-    "cmdk": "^0.2.1",
-    "core-js": "^3.36.0",
+    "cmdk": "^1.0.0",
+    "core-js": "^3.36.1",
    "cors": "^2.8.5",
    "date-fns": "^3.3.1",
    "decimal.js": "^10.4.3",
    "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.10",
+    "js-tiktoken": "^1.0.11",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.28",
+    "langchain": "^0.1.33",
    "lodash": "^4.17.21",
-    "lucide-react": "^0.330.0",
-    "next": "^14.1.0",
+    "lucide-react": "^0.364.0",
+    "next": "^14.2.1",
    "next-auth": "^4.24.7",
    "next-query-params": "^5.0.0",
-    "nodemailer": "^6.9.9",
+    "nodemailer": "^6.9.13",
    "posthog-js": "^1.105.9",
    "posthog-node": "^3.6.3",
-    "prisma": "^5.11.0",
+    "prisma": "^5.12.1",
    "react": "18.2.0",
    "react-day-picker": "^8.10.0",
    "react-dom": "18.2.0",
-    "react-hook-form": "^7.50.1",
+    "react-hook-form": "^7.51.3",
    "react-icons": "^5.0.1",
    "react-responsive": "^9.0.2",
-    "react18-json-view": "^0.2.8-canary.1",
-    "sonner": "^1.4.0",
+    "react18-json-view": "^0.2.8-canary.6",
+    "sonner": "^1.4.41",
    "superjson": "2.2.1",
    "tailwind-merge": "^2.2.1",
    "tailwindcss-animate": "^1.0.7",
@@ -112,68 +114,41 @@
  "devDependencies": {
    "@jedmao/location": "^3.0.0",
    "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.0",
    "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^14.2.1",
+    "@testing-library/react": "^14.2.2",
    "@types/bcryptjs": "^2.4.6",
    "@types/cors": "^2.8.17",
-    "@types/eslint": "^8.56.2",
+    "@types/eslint": "^8.56.7",
    "@types/jest": "^29.5.12",
    "@types/lodash": "^4.14.202",
    "@types/node": "20.10.5",
    "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.55",
-    "@types/react-dom": "^18.2.19",
+    "@types/react": "^18.2.77",
+    "@types/react-dom": "^18.2.25",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^6.21.0",
-    "autoprefixer": "^10.4.17",
+    "@typescript-eslint/parser": "^7.6.0",
+    "autoprefixer": "^10.4.19",
    "dotenv-cli": "^7.3.0",
    "eslint": "^8.56.0",
-    "eslint-config-next": "^14.1.0",
+    "eslint-config-next": "^14.2.1",
    "jest": "^29.7.0",
    "jest-environment-jsdom": "^29.7.0",
    "node-mocks-http": "^1.14.1",
-    "postcss": "^8.4.35",
+    "postcss": "^8.4.38",
    "prettier": "^3.2.5",
-    "prettier-plugin-tailwindcss": "^0.5.11",
-    "release-it": "^17.1.1",
-    "tailwindcss": "^3.4.1",
+    "prettier-plugin-tailwindcss": "^0.5.13",
+    "tailwindcss": "^3.4.3",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.7.1",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.2",
+    "typescript": "^5.4.5"
  },
  "ct3aMetadata": {
    "initVersion": "7.13.0"
  },
  "optionalDependencies": {
    "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
  }
 }
@@ -0,0 +1 @@
+openapi*.yml
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>
@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer
@@ -9,7 +9,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
    // Set tracesSampleRate to 1.0 to capture 100%
    // of transactions for performance monitoring.
    // We recommend adjusting this value in production
-    tracesSampleRate: 0.3,
+    tracesSampleRate: 0.2,

    // Capture Replay for 100% of all sessions,
    // plus for 100% of sessions with an error
@@ -7,7 +7,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
    // Set tracesSampleRate to 1.0 to capture 100%
    // of transactions for performance monitoring.
    // We recommend adjusting this value in production
-    tracesSampleRate: 0.3,
+    tracesSampleRate: 0.2,

    // ...

@@ -12,7 +12,6 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
    // Set tracesSampleRate to 1.0 to capture 100%
    // of transactions for performance monitoring.
    // We recommend adjusting this value in production
-    tracesSampleRate: 1.0,
    tracesSampler: (samplingContext: SamplingContext) => {
      if (
        samplingContext.request &&
@@ -20,9 +19,9 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
        samplingContext.request.url &&
        samplingContext.request.url.includes("api/trpc")
      ) {
-        return 1.0;
+        return 0.9;
      }
-      return 0.2;
+      return 0.15;
    },

    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
@@ -1,4 +1,7 @@
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
+import {
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server/auth";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { prisma } from "@langfuse/shared/src/db";

@@ -11,6 +11,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
  it("should create and get a dataset", async () => {
    await makeAPICall("POST", "/api/public/datasets", {
      name: "dataset-name",
+      description: "dataset-description",
    });

    const dbDataset = await prisma.dataset.findMany({
@@ -29,6 +30,101 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
    expect(getDataset.status).toBe(200);
    expect(getDataset.body).toMatchObject({
      name: "dataset-name",
+      description: "dataset-description",
+    });
+  });
+
+  it("GET datasets", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-1",
+    });
+
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-2",
+    });
+
+    const datasetItemId = v4();
+
+    const createItemRes = await makeAPICall<{
+      datasetName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name-2",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+      id: datasetItemId,
+    });
+
+    expect(createItemRes.status).toBe(200);
+    expect(createItemRes.body).toMatchObject({
+      datasetName: "dataset-name-2", // not included in db table
+    });
+
+    const traceId = v4();
+    const observationId = v4();
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+        {
+          id: v4(),
+          type: "observation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: observationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            startTime: "2021-01-01T00:00:00.000Z",
+            endTime: "2021-01-01T00:00:00.000Z",
+            modelParameters: { key: "value" },
+            input: { key: "value" },
+            metadata: { key: "value" },
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(response.status).toBe(207);
+
+    await makeAPICall("POST", "/api/public/dataset-run-items", {
+      datasetItemId: datasetItemId,
+      observationId: observationId,
+      runName: "test-run",
+      metadata: { key: "value" },
+    });
+
+    const getDatasets = await makeAPICall("GET", `/api/public/datasets`);
+
+    expect(getDatasets.status).toBe(200);
+    expect(getDatasets.body).toMatchObject({
+      data: expect.arrayContaining([
+        expect.objectContaining({
+          name: "dataset-name-1",
+          items: [],
+          runs: [],
+        }),
+        expect.objectContaining({
+          name: "dataset-name-2",
+          items: [datasetItemId],
+          runs: ["test-run"],
+        }),
+      ]),
+      meta: expect.objectContaining({
+        totalItems: 2,
+        page: 1,
+      }),
    });
  });

@@ -63,6 +159,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
          id: dbDatasetItem!.id,
          input: { key: "value" },
          expectedOutput: { key: "value" },
+          datasetName: "dataset-name", // not included in db table
        },
      ],
    });
@@ -76,6 +173,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      id: dbDatasetItem!.id,
      input: { key: "value" },
      expectedOutput: { key: "value" },
+      datasetName: "dataset-name", // not included in db table
    });
  });

@@ -168,25 +266,128 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
    });
    expect(response.status).toBe(207);

-    const runItem = await makeAPICall("POST", "/api/public/dataset-run-items", {
-      datasetItemId: "dataset-item-id",
-      observationId: observationId,
-      runName: "run-name",
-      metadata: { key: "value" },
-    });
-    const dbRun = await prisma.datasetRuns.findFirst({
+    const runItemObservation = await makeAPICall(
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        observationId: observationId,
+        runName: "run-only-observation",
+        runDescription: "run-description",
+        metadata: { key: "value" },
+      },
+    );
+    const dbRunObservation = await prisma.datasetRuns.findFirst({
      where: {
-        name: "run-name",
+        name: "run-only-observation",
+      },
+      include: {
+        datasetRunItems: true,
      },
    });
-    expect(dbRun).not.toBeNull();
-    expect(dbRun?.datasetId).toBe(dataset.body.id);
-    expect(dbRun?.metadata).toMatchObject({ key: "value" });
-    expect(runItem.status).toBe(200);
-    expect(runItem.body).toMatchObject({
+    expect(dbRunObservation).not.toBeNull();
+    expect(dbRunObservation?.datasetId).toBe(dataset.body.id);
+    expect(dbRunObservation?.metadata).toMatchObject({ key: "value" });
+    expect(dbRunObservation?.description).toBe("run-description");
+    expect(runItemObservation.status).toBe(200);
+    expect(dbRunObservation?.datasetRunItems[0]).toMatchObject({
      datasetItemId: "dataset-item-id",
      observationId: observationId,
-      datasetRunId: dbRun?.id,
+      traceId: traceId,
+    });
+
+    const getRunAPI = await makeAPICall(
+      "GET",
+      `/api/public/datasets/dataset-name/runs/run-only-observation`,
+    );
+    expect(getRunAPI.status).toBe(200);
+    expect(getRunAPI.body).toMatchObject({
+      name: "run-only-observation",
+      description: "run-description",
+      metadata: { key: "value" },
+      datasetId: dataset.body.id,
+      datasetName: "dataset-name",
+      datasetRunItems: expect.arrayContaining([
+        expect.objectContaining({
+          datasetItemId: "dataset-item-id",
+          observationId: observationId,
+          traceId: traceId,
+          datasetRunName: "run-only-observation",
+        }),
+      ]),
+    });
+
+    const runItemTrace = await makeAPICall<{
+      datasetRunName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-run-items", {
+      datasetItemId: "dataset-item-id",
+      traceId: traceId,
+      runName: "run-only-trace",
+      metadata: { key: "value" },
+    });
+
+    expect(runItemTrace.status).toBe(200);
+    expect(runItemTrace.body).toMatchObject({
+      datasetRunName: "run-only-trace", // not included in db table
+    });
+
+    const dbRunTrace = await prisma.datasetRuns.findFirst({
+      where: {
+        name: "run-only-trace",
+      },
+      include: {
+        datasetRunItems: true,
+      },
+    });
+    expect(dbRunTrace).not.toBeNull();
+    expect(dbRunTrace?.datasetId).toBe(dataset.body.id);
+    expect(dbRunTrace?.metadata).toMatchObject({ key: "value" });
+    expect(runItemTrace.status).toBe(200);
+    expect(dbRunTrace?.datasetRunItems[0]).toMatchObject({
+      datasetItemId: "dataset-item-id",
+      traceId: traceId,
+      observationId: null,
+    });
+
+    const runItemBoth = await makeAPICall(
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        observationId: observationId,
+        traceId: traceId,
+        runName: "run-name-both",
+        metadata: { key: "value" },
+      },
+    );
+    const dbRunBoth = await prisma.datasetRuns.findFirst({
+      where: {
+        name: "run-name-both",
+      },
+      include: {
+        datasetRunItems: true,
+      },
+    });
+    expect(dbRunBoth).not.toBeNull();
+    expect(dbRunBoth?.datasetId).toBe(dataset.body.id);
+    expect(dbRunBoth?.metadata).toMatchObject({ key: "value" });
+    expect(runItemBoth.status).toBe(200);
+    expect(dbRunBoth?.datasetRunItems[0]).toMatchObject({
+      datasetItemId: "dataset-item-id",
+      observationId: observationId,
+      traceId: traceId,
    });
  });
+
+  it("dataset-run-items should fail when neither trace nor observation provided", async () => {
+    const response = await makeAPICall(
+      "POST",
+      "/api/public/dataset-run-items",
+      {
+        datasetItemId: "dataset-item-id",
+        runName: "run-fail",
+      },
+    );
+    expect(response.status).toBe(400);
+  });
 });
@@ -520,8 +520,6 @@ describe("/api/public/ingestion API Endpoint", () => {

      expect(response.status).toBe(207);

-      console.log("response body", response.body);
-
      const dbGeneration = await prisma.observation.findUnique({
        where: {
          id: generationId,
@@ -851,7 +849,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -901,7 +898,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -1167,8 +1163,6 @@ describe("/api/public/ingestion API Endpoint", () => {
    });
    expect(responseOne.status).toBe(207);

-    console.log(responseOne.body);
-
    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
    const errors = responseOne.body.errors;

@@ -1419,7 +1413,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
      input,
    )} ${JSON.stringify(expected)}`, () => {
      const cleanedEvent = cleanEvent(input);
-      console.log(cleanedEvent);
      expect(cleanedEvent).toStrictEqual(expected);
    });
  });
@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared/src/db";
+import { type ObservationView } from "@langfuse/shared";

 describe("/api/public/observations API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -41,6 +41,19 @@ describe("/api/public/observations API Endpoint", () => {
      },
    });

+    const prompt = await prisma.prompt.create({
+      data: {
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
    await prisma.observation.create({
      data: {
        id: uuidv4(),
@@ -62,6 +75,9 @@ describe("/api/public/observations API Endpoint", () => {
        },
        internalModel: "gpt-3.5-turbo",
        unit: ModelUsageUnit.Tokens,
+        prompt: {
+          connect: { id: prompt.id },
+        },
      },
    });

@@ -95,6 +111,7 @@ describe("/api/public/observations API Endpoint", () => {
    expect(
      fetchedObservations.body.data[0]?.calculatedTotalCost,
    ).toBeGreaterThan(0);
+    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
  });
  it("should fetch all observations, filtered by generations", async () => {
    await pruneDatabase();
@@ -1,5 +1,5 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+import { tracesTableCols } from "@langfuse/shared";

 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
@@ -3,7 +3,12 @@
 import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
-import { type Prompt } from "@langfuse/shared/src/db";
+import { type Prompt } from "@langfuse/shared";
+import {
+  PromptSchema,
+  PromptType,
+  type ValidatedPrompt,
+} from "@/src/features/prompts/server/validation";

 describe("/api/public/prompts API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -43,10 +48,12 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptId);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
+    expect(fetchedObservations.body.tags).toEqual([]);
  });

  it("should fetch active prompt only if no prompt version is given", async () => {
@@ -190,6 +197,7 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptIdTwo);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(2);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -221,6 +229,7 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
@@ -342,8 +351,6 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(response.status).toBe(207);

-    console.log("response body", response.body);
-
    const dbGeneration = await prisma.observation.findUnique({
      where: {
        id: generationId,
@@ -375,11 +382,185 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
    expect(fetchedObservations.body.config).toEqual({});
  });
+
+  it("should create and fetch a chat prompt", async () => {
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(201);
+
+    const { body: fetchedPrompt } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+
+    const validatedPrompt = validatePrompt(fetchedPrompt);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+  });
+
+  it("should fail if chat prompt has string prompt", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if chat prompt has incorrect messages format", async () => {
+    const promptName = "prompt-name";
+    const incorrectChatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", message: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: incorrectChatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+  it("should fail if text prompt has message format", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: [{ role: "system", content: "You are a bot" }],
+      type: "text",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if previous versions have different prompt type", async () => {
+    // Create a chat prompt
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const postResponse1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse1.status).toBe(201);
+
+    // Try creating a text prompt with the same name
+    const postResponse2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "text",
+      isActive: true,
+      version: 2,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse2.status).toBe(400);
+    expect(postResponse2.body).toEqual({
+      error: "ValidationError",
+      message:
+        "Previous versions have different prompt type. Create a new prompt with a different name.",
+    });
+
+    // Check if the prompt is still the chat prompt
+    const getResponse1 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(getResponse1.status).toBe(200);
+
+    const validatedPrompt = validatePrompt(getResponse1.body);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+
+    // Check that the text prompt has not been created
+    const getResponse2 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}&version=2`,
+      undefined,
+    );
+    expect(getResponse2.status).toBe(404);
+    expect(getResponse2.body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
 });

 const isPrompt = (x: unknown): x is Prompt => {
@@ -393,6 +574,18 @@ const isPrompt = (x: unknown): x is Prompt => {
    typeof prompt.isActive === "boolean" &&
    typeof prompt.projectId === "string" &&
    typeof prompt.createdBy === "string" &&
-    typeof prompt.config === "object"
+    typeof prompt.config === "object" &&
+    Object.values(PromptType).includes(prompt.type as PromptType)
  );
 };
+
+const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+  Object.keys(obj).forEach((key) => {
+    obj[key] =
+      key === "createdAt" || key === "updatedAt"
+        ? new Date(obj[key] as string)
+        : obj[key];
+  });
+
+  return PromptSchema.parse(obj);
+};
@@ -40,11 +40,13 @@ export function DatePicker({
  onChange,
  clearable = false,
  className,
+  disabled,
 }: {
  date?: Date | undefined;
  onChange: (date: Date | undefined) => void;
  clearable?: boolean;
  className?: string;
+  disabled?: boolean;
 }) {
  return (
    <div className="flex flex-row gap-2 align-middle">
@@ -52,6 +54,7 @@ export function DatePicker({
        <PopoverTrigger asChild>
          <Button
            variant={"outline"}
+            disabled={disabled}
            className={cn(
              "justify-start text-left font-normal",
              !date && "text-muted-foreground",
@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
              <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
                <MessageCircle size={12} />
              </HoverCardTrigger>
-              <HoverCardContent>
+              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                <p>{s.comment}</p>
              </HoverCardContent>
            </HoverCard>
@@ -0,0 +1,33 @@
+import CodeMirror, { EditorView } from "@uiw/react-codemirror";
+import { githubLight } from "@uiw/codemirror-theme-github";
+import { json } from "@codemirror/lang-json";
+
+// todo: add json linting
+
+export function JsonEditor({
+  defaultValue,
+  onChange,
+  editable = true,
+  lineWrapping = true,
+}: {
+  defaultValue: string;
+  onChange?: (value: string) => void;
+  editable?: boolean;
+  lineWrapping?: boolean;
+}) {
+  return (
+    <CodeMirror
+      value={defaultValue}
+      theme={githubLight}
+      basicSetup={{
+        foldGutter: true,
+      }}
+      lang={"json"}
+      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      defaultValue={defaultValue}
+      onChange={onChange}
+      className="overflow-hidden rounded-md border"
+      editable={editable}
+    />
+  );
+}
@@ -40,7 +40,7 @@ export default function Header({
    [...props.breadcrumb.map((i) => i.href).filter(Boolean)].pop();

  return (
-    <div className={cn(level === "h2" ? "mb-4" : "mb-1")}>
+    <div className={cn(level === "h2" ? "mb-4" : "mb-2")}>
      <div>
        {backHref ? (
          <nav className="sm:hidden" aria-label="Back">
@@ -245,21 +245,19 @@ export default function Layout(props: PropsWithChildren) {
                      )}
                    />
                    <nav className="flex flex-1 flex-col">
-                      <ul role="list" className="flex flex-1 flex-col">
+                      <ul role="list">
                        <MainNavigation nav={navigation} />
-                        <li>
-                          <div className="mb-2 flex flex-row place-content-between items-center">
-                            <div className="text-xs font-semibold leading-6 text-gray-400">
-                              Project
-                            </div>
-                            <NewProjectButton size="xs" />
-                          </div>
-                          <ProjectNavigation
-                            currentProjectId={projectId ?? ""}
-                            projects={projects}
-                          />
-                        </li>
                      </ul>
+                      <div className="mb-2 flex flex-row place-content-between items-center">
+                        <div className="text-xs font-semibold leading-6 text-gray-400">
+                          Project
+                        </div>
+                        <NewProjectButton size="xs" />
+                      </div>
+                      <ProjectNavigation
+                        currentProjectId={projectId ?? ""}
+                        projects={projects}
+                      />
                    </nav>
                  </div>
                </Dialog.Panel>
@@ -283,38 +281,36 @@ export default function Layout(props: PropsWithChildren) {
            <nav className="flex h-full flex-1 flex-col overflow-y-auto px-6 pb-3">
              <ul role="list" className="flex h-full flex-col">
                <MainNavigation nav={topNavigation} />
-                <li className="mt-auto">
-                  <MainNavigation nav={bottomNavigation} />
-                  <FeedbackButtonWrapper
-                    className="space-y-1"
-                    title="Provide feedback"
-                    description="What do you think about this project? What can be improved?"
-                    type="feedback"
-                  >
-                    <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
-                      <MessageSquarePlus
-                        className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
-                        aria-hidden="true"
-                      />
-                      Feedback
-                    </li>
-                  </FeedbackButtonWrapper>
-                  <div className="mb-2 flex flex-row place-content-between items-center">
-                    <div className="text-xs font-semibold leading-6 text-gray-400">
-                      Project
-                    </div>
-                    <NewProjectButton size="xs" />
+                <MainNavigation nav={bottomNavigation} className="mt-auto" />
+                <FeedbackButtonWrapper
+                  className="space-y-1"
+                  title="Provide feedback"
+                  description="What do you think about this project? What can be improved?"
+                  type="feedback"
+                >
+                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
+                    <MessageSquarePlus
+                      className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                      aria-hidden="true"
+                    />
+                    Feedback
+                  </li>
+                </FeedbackButtonWrapper>
+                <div className="mb-2 flex flex-row place-content-between items-center">
+                  <div className="text-xs font-semibold leading-6 text-gray-400">
+                    Project
                  </div>
-                  <ProjectNavigation
-                    currentProjectId={projectId ?? ""}
-                    projects={projects}
-                  />
-                </li>
+                  <NewProjectButton size="xs" />
+                </div>
+                <ProjectNavigation
+                  currentProjectId={projectId ?? ""}
+                  projects={projects}
+                />
              </ul>
            </nav>

-            <Menu as="div" className="relative left-1">
-              <Menu.Button className="flex w-full items-center gap-x-4 p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+            <Menu as="div" className="relative">
+              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
                <span className="sr-only">Open user menu</span>
                <Avatar className="h-8 w-8">
                  <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -346,7 +342,10 @@ export default function Layout(props: PropsWithChildren) {
                leaveFrom="transform opacity-100 scale-100"
                leaveTo="transform opacity-0 scale-95"
              >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                    {session.data?.user?.email}
+                  </span>
                  {userNavigation.map((item) => (
                    <Menu.Item key={item.name}>
                      {({ active }) => (
@@ -407,7 +406,10 @@ export default function Layout(props: PropsWithChildren) {
              leaveFrom="transform opacity-100 scale-100"
              leaveTo="transform opacity-0 scale-95"
            >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  {session.data?.user?.email}
+                </span>
                {userNavigation.map((item) => (
                  <Menu.Item key={item.name}>
                    {({ active }) => (
@@ -478,14 +480,15 @@ type NestedNavigationItem = Omit<Route, "children"> & {
 const MainNavigation: React.FC<{
  nav: NavigationItem[];
  onNavitemClick?: () => void;
-}> = ({ nav, onNavitemClick }) => {
+  className?: string;
+}> = ({ nav, onNavitemClick, className }) => {
  const [isOpen, setIsOpen] = useLocalStorage(
    "sidebar-tracing-default-open",
    false,
  );

  return (
-    <li>
+    <li className={className}>
      <ul role="list" className="-mx-2 space-y-1">
        {nav.map((item) => (
          <li key={item.name}>
@@ -499,6 +502,7 @@ const MainNavigation: React.FC<{
                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
                )}
                onClick={onNavitemClick}
+                target={item.newTab ? "_blank" : undefined}
              >
                {item.icon && (
                  <item.icon
@@ -577,6 +581,7 @@ const MainNavigation: React.FC<{
                                : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
                            )}
+                            target={subItem.newTab ? "_blank" : undefined}
                          >
                            {subItem.name}
                            {subItem.label && (
@@ -9,6 +9,7 @@ import {
  Settings,
  UsersIcon,
  PenSquareIcon,
+  LibraryBig,
  TerminalIcon,
 } from "lucide-react";

@@ -21,6 +22,7 @@ export type Route = {
  pathname?: string; // link, ignored if children
  children?: Array<Route>; // folder
  bottom?: boolean; // bottom of the sidebar, only for first level routes
+  newTab?: boolean; // open in new tab
 };

 export const ROUTES: Route[] = [
@@ -53,6 +55,16 @@ export const ROUTES: Route[] = [
        name: "Models",
        pathname: `/project/[projectId]/models`,
      },
+      {
+        name: "Templates",
+        pathname: `/project/[projectId]/evals/templates`,
+        featureFlag: "evals",
+      },
+      {
+        name: "Configs",
+        pathname: `/project/[projectId]/evals/configs`,
+        featureFlag: "evals",
+      },
    ],
  },
  {
@@ -83,6 +95,13 @@ export const ROUTES: Route[] = [
    icon: Settings,
    bottom: true,
  },
+  {
+    name: "Docs",
+    pathname: "https://langfuse.com/docs",
+    icon: LibraryBig,
+    bottom: true,
+    newTab: true,
+  },
  {
    name: "Support",
    pathname: "/project/[projectId]/support",
@@ -1,7 +1,7 @@
 import { cn } from "@/src/utils/tailwind";

 const statusCategories = {
-  active: ["production", "live"],
+  active: ["production", "live", "active"],
  inactive: ["disabled", "inactive"],
 };

@@ -7,7 +7,6 @@ import {
  SelectValue,
 } from "@/src/components/ui/select";
 import { cn } from "@/src/utils/tailwind";
-import { Code } from "lucide-react";
 import { useRouter } from "next/router";

 interface Project {
@@ -4,6 +4,7 @@ import { NoAccessError } from "@/src/components/no-access";
 import { PublishSessionSwitch } from "@/src/components/publish-object-switch";
 import { StarSessionToggle } from "@/src/components/star-toggle";
 import { IOPreview } from "@/src/components/trace/IOPreview";
+import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 import { Badge } from "@/src/components/ui/badge";
 import { Card } from "@/src/components/ui/card";
 import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
@@ -101,20 +102,7 @@ export const SessionPage: React.FC<{
            className="border-border-gray-150 group grid gap-3 p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
            key={trace.id}
          >
-            <div className="col-span-2 flex flex-col gap-2 p-0">
-              {trace.input || trace.output ? (
-                <IOPreview
-                  key={trace.id}
-                  input={trace.input}
-                  output={trace.output}
-                  hideIfNull
-                />
-              ) : (
-                <div className="p-2 text-xs text-gray-500">
-                  This trace has no input or output.
-                </div>
-              )}
-            </div>
+            <SessionIO traceId={trace.id} />
            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
              <Link
                href={`/project/${projectId}/traces/${trace.id}`}
@@ -142,3 +130,39 @@ export const SessionPage: React.FC<{
    </div>
  );
 };
+
+const SessionIO = ({ traceId }: { traceId: string }) => {
+  const trace = api.traces.byId.useQuery(
+    { traceId: traceId },
+    {
+      enabled: typeof traceId === "string",
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+  return (
+    <div className="col-span-2 flex flex-col gap-2 p-0">
+      {!trace.data ? (
+        <JsonSkeleton
+          className="h-full w-full overflow-hidden px-2 py-1"
+          numRows={4}
+        />
+      ) : trace.data.input || trace.data.output ? (
+        <IOPreview
+          key={traceId}
+          input={trace.data.input}
+          output={trace.data.output}
+          hideIfNull
+        />
+      ) : (
+        <div className="p-2 text-xs text-gray-500">
+          This trace has no input or output.
+        </div>
+      )}
+    </div>
+  );
+};
@@ -12,7 +12,7 @@ import {
  DropdownMenuCheckboxItem,
 } from "@/src/components/ui/dropdown-menu";
 import { type VisibilityState } from "@tanstack/react-table";
-import { ChevronDownIcon } from "lucide-react";
+import { ChevronDown, Columns } from "lucide-react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";

 interface DataTableColumnVisibilityFilterProps<TData, TValue> {
@@ -70,14 +70,16 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
        className="select-none"
        asChild
      >
-        <Button variant="outline" className="ml-auto">
-          Columns {count <= total ? `${count}/${total}` : ""}
-          <ChevronDownIcon className="ml-2 h-4 w-4" />
+        <Button variant="outline" title="Show/hide columns">
+          <Columns className="mr-2 h-4 w-4" />
+          <span className="text-xs text-gray-500">{`(${count}/${total})`}</span>
+          <ChevronDown className="ml-2 h-4 w-4" />
        </Button>
      </DropdownMenuTrigger>
      <DropdownMenuContent
        align="end"
        onPointerDownOutside={() => setIsOpen(false)}
+        className="max-h-96 overflow-y-auto"
      >
        {columns.map(
          (column, index) =>
@@ -46,11 +46,7 @@ export function TraceTableMultiSelectAction({
    <>
      <DropdownMenu>
        <DropdownMenuTrigger asChild>
-          <Button
-            variant="outline"
-            className="bg-white p-2 font-medium text-black"
-            disabled={selectedTraceIds.length < 1}
-          >
+          <Button disabled={selectedTraceIds.length < 1}>
            Actions ({selectedTraceIds.length} selected)
            <ChevronDown className="h-5 w-5" />
          </Button>
@@ -0,0 +1,59 @@
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import {
+  MdDensityLarge,
+  MdDensityMedium,
+  MdDensitySmall,
+} from "react-icons/md";
+
+const heightOptions = [
+  { id: "s", label: "Small", value: "h-6", icon: <MdDensitySmall /> },
+  { id: "m", label: "Medium", value: "h-24", icon: <MdDensityMedium /> },
+  { id: "l", label: "Large", value: "h-64", icon: <MdDensityLarge /> },
+] as const;
+
+export type RowHeight = (typeof heightOptions)[number]["id"];
+
+export const getRowHeightTailwindClass = (rowHeight: RowHeight | undefined) =>
+  heightOptions.find((h) => h.id === rowHeight)?.value;
+
+export function useRowHeightLocalStorage(
+  tableName: string,
+  defaultValue: RowHeight,
+) {
+  const [rowHeight, setRowHeight, clearRowHeight] = useLocalStorage<RowHeight>(
+    `${tableName}Height`,
+    defaultValue,
+  );
+
+  return [rowHeight, setRowHeight, clearRowHeight] as const;
+}
+
+export const DataTableRowHeightSwitch = ({
+  rowHeight,
+  setRowHeight,
+}: {
+  rowHeight: RowHeight;
+  setRowHeight: (e: RowHeight) => void;
+}) => (
+  <Tabs
+    //defaultValue={height}
+    value={rowHeight}
+    onValueChange={(e) => setRowHeight(e as any)}
+    key="height"
+  >
+    <TabsList className="gap-1 border bg-transparent px-2">
+      {heightOptions.map(({ id, label, icon }) => (
+        <TabsTrigger
+          key={id}
+          value={id}
+          className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
+        >
+          <span role="img" aria-label={`${label} size`}>
+            {icon}
+          </span>
+        </TabsTrigger>
+      ))}
+    </TabsList>
+  </Tabs>
+);
@@ -2,11 +2,16 @@ import { Button } from "@/src/components/ui/button";
 import React, { type Dispatch, type SetStateAction, useState } from "react";
 import { Input } from "@/src/components/ui/input";
 import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-table-column-visibility-filter";
-import { type FilterState } from "@/src/features/filters/types";
-import { FilterBuilder } from "@/src/features/filters/components/filter-builder";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import { type FilterState } from "@langfuse/shared";
+import { PopoverFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import { type ColumnDefinition } from "@langfuse/shared";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import {
+  DataTableRowHeightSwitch,
+  type RowHeight,
+} from "@/src/components/table/data-table-row-height-switch";
+import { Search } from "lucide-react";

 interface SearchConfig {
  placeholder: string;
@@ -16,13 +21,15 @@ interface SearchConfig {

 interface DataTableToolbarProps<TData, TValue> {
  columns: LangfuseColumnDef<TData, TValue>[];
-  filterColumnDefinition: ColumnDefinition[];
+  filterColumnDefinition?: ColumnDefinition[];
  searchConfig?: SearchConfig;
  actionButtons?: React.ReactNode;
-  filterState: FilterState;
-  setFilterState: Dispatch<SetStateAction<FilterState>>;
+  filterState?: FilterState;
+  setFilterState?: Dispatch<SetStateAction<FilterState>>;
  columnVisibility?: VisibilityState;
  setColumnVisibility?: Dispatch<SetStateAction<VisibilityState>>;
+  rowHeight?: RowHeight;
+  setRowHeight?: Dispatch<SetStateAction<RowHeight>>;
 }

 export function DataTableToolbar<TData, TValue>({
@@ -34,42 +41,46 @@ export function DataTableToolbar<TData, TValue>({
  setFilterState,
  columnVisibility,
  setColumnVisibility,
+  rowHeight,
+  setRowHeight,
 }: DataTableToolbarProps<TData, TValue>) {
  const [searchString, setSearchString] = useState(
    searchConfig?.currentQuery ?? "",
  );

  return (
-    <div className="my-2 flex max-w-full items-center justify-between overflow-x-auto">
-      <div className="flex flex-1 items-center space-x-2">
-        {searchConfig && (
-          <div className="flex max-w-md items-center space-x-2">
-            <Input
-              autoFocus
-              placeholder={searchConfig.placeholder}
-              value={searchString}
-              onChange={(event) => setSearchString(event.currentTarget.value)}
-              onKeyDown={(event) => {
-                if (event.key === "Enter") {
-                  searchConfig.updateQuery(searchString);
-                }
-              }}
-              className="h-10 w-[200px] lg:w-[350px]"
-            />
-            <Button
-              variant="outline"
-              onClick={() => searchConfig.updateQuery(searchString)}
-            >
-              Search
-            </Button>
-          </div>
-        )}
-        <FilterBuilder
+    <div className="my-2 flex flex-1 flex-wrap items-center gap-2">
+      {searchConfig && (
+        <div className="flex max-w-md items-center">
+          <Input
+            autoFocus
+            placeholder={searchConfig.placeholder}
+            value={searchString}
+            onChange={(event) => setSearchString(event.currentTarget.value)}
+            onKeyDown={(event) => {
+              if (event.key === "Enter") {
+                searchConfig.updateQuery(searchString);
+              }
+            }}
+            className="h-10 w-[200px] rounded-r-none lg:w-[250] 2xl:w-[350px]"
+          />
+          <Button
+            variant="outline"
+            onClick={() => searchConfig.updateQuery(searchString)}
+            className="rounded-l-none border-l-0 p-3"
+          >
+            <Search className="h-4 w-4" />
+          </Button>
+        </div>
+      )}
+      {!!filterColumnDefinition && !!filterState && !!setFilterState && (
+        <PopoverFilterBuilder
          columns={filterColumnDefinition}
          filterState={filterState}
          onChange={setFilterState}
        />
-        <div className="flex-1" />
+      )}
+      <div className="flex flex-row flex-wrap gap-2 lg:ml-auto">
        {!!columnVisibility && !!setColumnVisibility && (
          <DataTableColumnVisibilityFilter
            columns={columns}
@@ -77,6 +88,12 @@ export function DataTableToolbar<TData, TValue>({
            setColumnVisibility={setColumnVisibility}
          />
        )}
+        {!!rowHeight && !!setRowHeight && (
+          <DataTableRowHeightSwitch
+            rowHeight={rowHeight}
+            setRowHeight={setRowHeight}
+          />
+        )}
        {actionButtons}
      </div>
    </div>
@@ -2,6 +2,10 @@

 import DocPopup from "@/src/components/layouts/doc-popup";
 import { DataTablePagination } from "@/src/components/table/data-table-pagination";
+import {
+  type RowHeight,
+  getRowHeightTailwindClass,
+} from "@/src/components/table/data-table-row-height-switch";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { type ModelTableRow } from "@/src/components/table/use-cases/models";
 import {
@@ -43,6 +47,7 @@ interface DataTableProps<TData, TValue> {
  orderBy?: OrderByState;
  setOrderBy?: (s: OrderByState) => void;
  help?: { description: string; href: string };
+  rowHeight?: RowHeight;
 }

 export interface AsyncTableData<T> {
@@ -63,8 +68,10 @@ export function DataTable<TData extends object, TValue>({
  help,
  orderBy,
  setOrderBy,
+  rowHeight,
 }: DataTableProps<TData, TValue>) {
  const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
+  const rowheighttw = getRowHeightTailwindClass(rowHeight);

  const table = useReactTable({
    data: data.data ?? [],
@@ -185,10 +192,12 @@ export function DataTable<TData extends object, TValue>({
                        key={cell.id}
                        className="overflow-hidden whitespace-nowrap px-2 py-1 text-xs first:pl-2"
                      >
-                        {flexRender(
-                          cell.column.columnDef.cell,
-                          cell.getContext(),
-                        )}
+                        <div className={cn("flex items-center", rowheighttw)}>
+                          {flexRender(
+                            cell.column.columnDef.cell,
+                            cell.getContext(),
+                          )}
+                        </div>
                      </TableCell>
                    ))}
                  </TableRow>
@@ -4,22 +4,26 @@ export type TableLinkProps = {
  path: string;
  value: string;
  truncateAt?: number;
+  icon?: React.ReactNode;
 };

 export default function TableLink({
  path,
  value,
  truncateAt = 7,
+  icon,
 }: TableLinkProps) {
+  const truncatedValue =
+    value.length - truncateAt > 3
+      ? `...${value.substring(value.length - truncateAt)}`
+      : value;
  return (
    <Link
      className="inline-block rounded bg-indigo-50 px-2 py-1 text-xs font-semibold text-blue-600 shadow-sm hover:bg-indigo-100"
      href={path}
      title={value}
    >
-      {value.length - truncateAt > 3
-        ? `...${value.substring(value.length - truncateAt)}`
-        : value}
+      {icon ? icon : truncatedValue}
    </Link>
  );
 }
@@ -1,54 +0,0 @@
-import { JSONView } from "@/src/components/ui/code";
-import { cn } from "@/src/utils/tailwind";
-import { randomIntFromInterval } from "@/src/utils/numbers";
-import { Skeleton } from "@/src/components/ui/skeleton";
-import React from "react";
-
-export const IOCell = ({
-  isLoading,
-  data,
-}: {
-  isLoading: boolean;
-  data: unknown;
-}) => {
-  return (
-    <>
-      {isLoading  ? (
-        <JsonSkeleton className="h-[250px] w-[500px] px-3 py-1" />
-      ) : (
-        <JSONView json={data} className="h-[250px] w-[500px] overflow-y-auto" />
-      )}
-    </>
-  );
-};
-
-export const JsonSkeleton = ({
-  className,
-  numRows = 10,
-}: {
-  numRows?: number;
-  className?: string;
-}) => {
-  const sizingOptions = [
-    "h-5 w-full",
-    "h-5 w-[400px]",
-    "h-5 w-[450px]",
-    "h-5 w-[475px]",
-  ];
-
-  const generateRandomSize = () =>
-    sizingOptions[randomIntFromInterval(0, sizingOptions.length - 1)];
-
-  return (
-    <div className={cn("w-[500px] rounded-md border", className)}>
-      <div className="flex flex-col gap-1">
-        {[...Array<number>(numRows)].map((_) => (
-          <>
-            <Skeleton className={generateRandomSize()} />
-          </>
-        ))}
-        <br />
-      </div>
-    </div>
-  );
-};
@@ -22,7 +22,6 @@ import {
  withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
 import {
  formatIntervalSeconds,
  intervalInSeconds,
@@ -30,19 +29,20 @@ import {
 } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type ObservationLevel } from "@langfuse/shared/src/db";
+import { type Prisma, type ObservationLevel } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
 import { usdFormatter } from "@/src/utils/numbers";
 import {
  exportOptions,
  type ExportFileFormats,
-} from "@/src/server/api/interfaces/exportTypes";
+  observationsTableColsWithOptions,
+} from "@langfuse/shared";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
-import { IOCell } from "./IOCell";
-import { setSmallPaginationIfColumnsVisible } from "../../../features/column-visibility/hooks/setSmallPaginationIfColumnsVisible";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";

 export type GenerationsTableRow = {
  id: string;
@@ -62,7 +62,7 @@ export type GenerationsTableRow = {
  outputCost?: Decimal;
  totalCost?: Decimal;
  traceName?: string;
-  metadata?: string;
+  metadata?: Prisma.JsonValue;
  scores?: ScoreSimplified[];
  usage: {
    promptTokens: number;
@@ -91,6 +91,11 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
    pageSize: withDefault(NumberParam, 50),
  });

+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "generations",
+    "s",
+  );
+
  const [filterState, setFilterState] = useQueryFilterState(
    [
      {
@@ -510,8 +515,10 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
      accessorKey: "metadata",
      header: "Metadata",
      cell: ({ row }) => {
-        const values: string | undefined = row.getValue("metadata");
-        return <div className="flex flex-wrap gap-x-3 gap-y-1">{values}</div>;
+        const values = row.getValue(
+          "metadata",
+        ) as GenerationsTableRow["metadata"];
+        return !!values ? <IOTableCell data={values} /> : null;
      },
      enableHiding: true,
      defaultHidden: true,
@@ -552,13 +559,6 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
      columns,
    );

-  const smallTableRequired = setSmallPaginationIfColumnsVisible(
-    columnVisibility,
-    ["input", "output"],
-    paginationState,
-    setPaginationState,
-  );
-
  const rows: GenerationsTableRow[] = generations.isSuccess
    ? generations.data.generations.map((generation) => {
        return {
@@ -577,9 +577,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
          model: generation.model ?? "",
          scores: generation.scores,
          level: generation.level,
-          metadata: generation.metadata
-            ? JSON.stringify(generation.metadata)
-            : undefined,
+          metadata: generation.metadata,
          statusMessage: generation.statusMessage ?? undefined,
          usage: {
            promptTokens: generation.promptTokens,
@@ -609,14 +607,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
        }}
        columnVisibility={columnVisibility}
        setColumnVisibility={setColumnVisibilityState}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
        actionButtons={
          <DropdownMenu>
            <DropdownMenuTrigger asChild>
-              <Button
-                variant="outline"
-                className="ml-auto whitespace-nowrap"
-                size="sm"
-              >
+              <Button variant="outline" className="ml-auto whitespace-nowrap">
                {filterState.length > 0 || searchQuery
                  ? "Export selection"
                  : "Export all"}{" "}
@@ -662,13 +658,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
          pageCount: Math.ceil(totalCount / paginationState.pageSize),
          onChange: setPaginationState,
          state: paginationState,
-          // enforce a minimum page size of 10 if input or output columns are visible
-          options: smallTableRequired ? [10] : undefined,
        }}
        setOrderBy={setOrderByState}
        orderBy={orderByState}
        columnVisibility={columnVisibility}
        onColumnVisibilityChange={setColumnVisibilityState}
+        rowHeight={rowHeight}
      />
    </div>
  );
@@ -695,14 +690,16 @@ const GenerationsIOCell = ({
          skipBatch: true,
        },
      },
+      refetchOnMount: false, // prevents refetching loops
    },
  );
  return (
-    <IOCell
+    <IOTableCell
      isLoading={observation.isLoading}
      data={
        io === "output" ? observation.data?.output : observation.data?.input
      }
+      className={cn(io === "output" && "bg-green-50")}
    />
  );
 };
@@ -1,16 +1,18 @@
 import { DataTable } from "@/src/components/table/data-table";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
-  ScoreOptions,
+  type ScoreOptions,
  scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { RouterOutput, type RouterInput } from "@/src/utils/types";
+import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";

 export type ScoresTableRow = {
@@ -21,7 +23,7 @@ export type ScoresTableRow = {
  value: number;
  comment?: string;
  observationId?: string;
-  traceName: string;
+  traceName?: string;
  userId?: string;
 };

@@ -44,6 +46,8 @@ export default function ScoresTable({
    pageSize: withDefault(NumberParam, 50),
  });

+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage("scores", "s");
+
  const [userFilterState, setUserFilterState] = useQueryFilterState(
    [],
    "scores",
@@ -102,15 +106,16 @@ export default function ScoresTable({
      header: "Observation ID",
      enableSorting: true,
      cell: ({ row }) => {
-        const observationId = row.getValue("observationId");
-        const traceId = row.getValue("traceId");
-        return typeof observationId === "string" &&
-          typeof traceId === "string" ? (
+        const observationId = row.getValue(
+          "observationId",
+        ) as ScoresTableRow["observationId"];
+        const traceId = row.getValue("traceId") as ScoresTableRow["traceId"];
+        return traceId && observationId ? (
          <TableLink
            path={`/project/${projectId}/traces/${traceId}?observation=${observationId}`}
            value={observationId}
          />
-        ) : null;
+        ) : undefined;
      },
    },
    {
@@ -120,17 +125,17 @@ export default function ScoresTable({
      enableHiding: true,
      enableSorting: true,
      cell: ({ row }) => {
-        const value: string = row.getValue("traceName");
+        const value = row.getValue("traceName") as ScoresTableRow["traceName"];
        const filter = encodeURIComponent(
          `name;stringOptions;;any of;${value}`,
        );
-        return (
+        return value ? (
          <TableLink
            path={`/project/${projectId}/traces?filter=${value ? filter : ""}`}
            value={value}
            truncateAt={40}
          />
-        );
+        ) : undefined;
      },
    },
    {
@@ -162,6 +167,10 @@ export default function ScoresTable({
      accessorKey: "userId",
      header: "User ID",
      id: "userId",
+      headerTooltip: {
+        description: "The user ID associated with the trace.",
+        href: "https://langfuse.com/docs/tracing/users",
+      },
      enableHiding: true,
      enableSorting: true,
      cell: ({ row }) => {
@@ -182,6 +191,10 @@ export default function ScoresTable({
      header: "Comment",
      id: "comment",
      enableHiding: true,
+      cell: ({ row }) => {
+        const value = row.getValue("comment") as ScoresTableRow["comment"];
+        return value !== undefined && <IOTableCell data={value} />;
+      },
    },
  ];

@@ -199,7 +212,7 @@ export default function ScoresTable({
      comment: score.comment ?? undefined,
      observationId: score.observationId ?? undefined,
      traceId: score.traceId,
-      traceName: score.traceName,
+      traceName: score.traceName ?? undefined,
      userId: score.userId ?? undefined,
    };
  };
@@ -221,6 +234,8 @@ export default function ScoresTable({
        setFilterState={setUserFilterState}
        columnVisibility={columnVisibility}
        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
      />
      <DataTable
        columns={columns}
@@ -248,6 +263,7 @@ export default function ScoresTable({
        setOrderBy={setOrderByState}
        columnVisibility={columnVisibility}
        onColumnVisibilityChange={setColumnVisibility}
+        rowHeight={rowHeight}
      />
    </div>
  );
@@ -6,7 +6,7 @@ import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsTableColsWithOptions } from "@/src/server/api/definitions/sessionsView";
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>