chore: release v2.29.2

fix: do not load NODE_ENV for posthog in frontend
chore: release v2.29.1
2024-04-17 17:08:09 +02:00 · 2024-04-17 16:58:55 +02:00 · 2024-04-17 15:09:29 +02:00 · 2024-04-17 15:09:15 +02:00 · 2024-04-17 14:24:13 +02:00 · 2024-04-17 14:22:20 +02:00
295 changed files with 12061 additions and 3549 deletions
@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""

 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
@@ -16,4 +16,11 @@ SALT="salt"

 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
@@ -37,6 +37,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
+# AUTH_DISABLE_SIGNUP=true

 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -76,11 +77,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # DB_EXPORT_PAGE_SIZE=1000


-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
@@ -105,6 +101,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=

+# Prompt playground
+# OPENAI_API_KEY=""
+# ANTHROPIC_API_KEY=""
+
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=

@@ -114,4 +114,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=

+# Encryption
+# ENCRYPTION_KEY=
+
+# Admin API
+# ADMIN_API_KEY=
+
 ### END Langfuse Cloud Config
@@ -1 +1,2 @@
-* @langfuse/founders
+# Currently inactive
+# * @langfuse/maintainers
@@ -7,6 +7,7 @@ version: 2
 updates:
  - package-ecosystem: npm
    directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
    schedule:
      interval: "daily"
    versioning-strategy: "increase"
@@ -19,7 +19,7 @@ jobs:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v3
        with:
-          version: 8
+          version: 8.15.5
      - uses: actions/setup-node@v3
        with:
          node-version: 20
@@ -54,7 +54,7 @@ jobs:
          file: ./worker/Dockerfile
          push: false

-  tests:
+  tests-web:
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -68,7 +68,7 @@ jobs:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v3
        with:
-          version: 8
+          version: 8.15.5
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
@@ -102,7 +102,56 @@ jobs:
        run: (pnpm run start&)

      - name: run tests
-        run: pnpm run test
+        run: pnpm --filter=web run test
+
+  tests-worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 8.15.5
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+
+      - name: Build
+        run: pnpm --filter=worker... run build
+
+      - name: run tests
+        run: pnpm --filter=worker run test

  e2e-tests:
    runs-on: ubuntu-latest
@@ -110,7 +159,7 @@ jobs:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v3
        with:
-          version: 8
+          version: 8.15.5
      - uses: actions/setup-node@v3
        with:
          node-version: 20
@@ -148,7 +197,7 @@ jobs:
  all-ci-passed:
    # This allows us to have a branch protection rule for tests and deploys with matrix
    runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests, test-docker-build]
+    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
    if: always()
    steps:
      - name: Successful deploy
@@ -165,9 +214,6 @@ jobs:
    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
    environment: "protected branches"
    runs-on: ubuntu-latest
-    env:
-      REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
    permissions:
      packages: write
      contents: read
@@ -175,7 +221,7 @@ jobs:
    steps:
      - uses: pnpm/action-setup@v3
        with:
-          version: 8
+          version: 8.15.5
      - name: Setup node
        uses: actions/setup-node@v3
        with:
@@ -185,19 +231,26 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v3

-      - name: Log in to the Container registry
+      - name: Log in to the GitHub Container registry
        uses: docker/login-action@v2
        with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
        uses: docker/metadata-action@v4
        with:
          images: |
-            ${{ env.REGISTRY }}/${{ github.repository }}
+            ghcr.io/langfuse/langfuse # GitHub
+            langfuse/langfuse # Docker Hub
          tags: |
            type=ref,event=branch
            type=ref,event=pr
@@ -206,11 +259,35 @@ jobs:
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}

-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./web/Dockerfile
          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/langfuse/langfuse-worker # GitHub
+            langfuse/langfuse-worker # Docker Hub
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}
@@ -1,3 +1,5 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 # Contributing to Langfuse

 First off, thanks for taking the time to contribute! ❤️
@@ -118,9 +120,11 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/

 - `web`: is the main application package providing Frontend and Backend APIs for Langfuse.
 - `worker` (no production yet): contains an application for asynchronous processing of tasks. This package is not yet used in production.
- `shared`: contains shared code between the above packages.
- `config-eslint`: contains eslint configurations which are shared between the above packages.
- `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `packages`:
+  - `shared`: contains shared code between the above packages.
+  - `config-eslint`: contains eslint configurations which are shared between the above packages.
+  - `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `ee`: contains all enterprise features. See [EE README](ee/README.md) for more details.

 ## Development Setup

@@ -192,7 +196,7 @@ Requirements
  To run migrations, you can execute the following command.

  ```bash
-  pnpm --filter=shared run db:migrate
+  pnpm run db:migrate -- --name <name of the migration>
  ```

 > [!NOTE]
@@ -2,9 +2,9 @@ Copyright (c) 2023 Finto Technologies GmbH

 Portions of this software are licensed as follows:

-* All content that resides under the "ee/" directory of this repository, if that directory exists, is licensed under the license defined in "ee/LICENSE".
-* All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
-* Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
+- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -22,4 +22,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
@@ -1,25 +1,19 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 <div align="center">
-   <a href="https://langfuse.com">
-      <h1>🪢 Langfuse</h1>
-   </a>
- <div>
-      <h3> <a href="https://cloud.langfuse.com">
-         <strong>Sign up</strong>
-       </a> · 
-      <a href="https://langfuse.com/docs/deployment/self-host">
-         <strong>Self Host Langfuse</strong>
-      </a> · 
-      <a href="https://langfuse.com/demo">
-         <strong>Demo Project (live data)</strong>
-      </a>
+   <div>
+      <h3>
+         <a href="https://cloud.langfuse.com">
+            <strong>Sign up</strong>
+         </a> · 
+         <a href="https://langfuse.com/docs/deployment/self-host">
+            <strong>Self Host</strong>
+         </a> · 
+         <a href="https://langfuse.com/demo">
+            <strong>Demo (live data)</strong>
+         </a>
      </h3>
-   <h3>
-      Langfuse is the open source LLM engineering platform.   
-   </h3>
-    <div>
-     Debug, analyze and iterate - together
   </div>
-   </br>
   <div>
      <a href="https://langfuse.com/docs"><strong>Docs</strong></a> ·
      <a href="https://langfuse.com/issues"><strong>Report Bug</strong></a> ·
@@ -28,7 +22,10 @@
      <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
      <a href="https://langfuse.com/discord"><strong>Discord</strong></a> 
   </div>
-   </br>
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <br/>
+   <br/>
+   <div>
      <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
      <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
      <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
@@ -60,12 +57,6 @@
  - Datasets let you test expected in and output pairs and benchmark performance before deploying ([Datasets](https://langfuse.com/docs/datasets))
  - Track versions and releases in your application ([Experimentation](https://langfuse.com/docs/experimentation), [Prompt Management](https://langfuse.com/docs/prompts))

-### Video: Langfuse in two minutes
-
-https://github.com/langfuse/langfuse/assets/2834609/6041347a-b517-4a11-8737-93ef8f8af49f
-
-_Muted by default, enable sound for voice-over_
-
 ## Get started

 ### Langfuse Cloud
@@ -98,25 +89,25 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe

 ### API Keys

-You require a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
+You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.

 ### Ingesting Data · Instrumenting Your Application

-Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
+Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.

 You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.

-See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get started in integrating Langfuse.
+See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.

 ### Integrations

-| Integration                                                      | Supports      | Description                                                      |
-| ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.      |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python        | Automated instrumentation via drop-in replacement of OpenAI SDK. |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.        |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python       | Automated instrumentation via LlamaIndex callback system.        |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.            |
+| Integration                                                      | Supports      | Description                                                 |
+| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
+| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
+| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
+| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |

 External projects/packages that integrate with Langfuse:

@@ -140,8 +131,8 @@ External projects/packages that integrate with Langfuse:
 In order of preference the best way to communicate with us:

 - [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
- [Discord](https://langfuse.com/discord): For community support and to chat directly with maintainers
- Privately: Email contact at langfuse dot com
+- [Discord](https://langfuse.com/discord): community support
+- Privately: contact at langfuse dot com

 ## Contributing to Langfuse

@@ -151,7 +142,7 @@ In order of preference the best way to communicate with us:

 ## License

-This repository is MIT licensed, except for the `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
+This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

 ## Misc

@@ -16,11 +16,9 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  worker:
@@ -30,7 +28,6 @@ services:
    depends_on:
      - db
      - redis
-      - server
    ports:
      - "3030:3030"
    environment:
@@ -39,9 +36,10 @@ services:
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
      - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
      - REDIS_PORT=${REDIS_PORT:-6379}
      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  redis:
@@ -13,7 +13,6 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

  db:
@@ -0,0 +1,8 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  extends: ["@repo/eslint-config/library.js"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: true,
+  },
+};
@@ -1,3 +1,5 @@
 # Enterprise Edition

-While we are currently working 100% on the open source version, we consider financing the project by developing some premium features as part of an enterprise version under a commercial license. Please reach out to us if you have specific requirements: enterprise@langfuse.com
+This folder includes features that are only available in the Enterprise Edition of Langfuse and on Langfuse Cloud.
+
+See [LICENSE](../LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
@@ -0,0 +1,46 @@
+{
+  "name": "@langfuse/ee",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./sso": {
+      "import": "./dist/src/sso/index.js",
+      "require": "./dist/src/sso/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
+  },
+  "dependencies": {
+    "@langfuse/shared": "workspace:*",
+    "next": "^14.2.1",
+    "next-auth": "^4.24.7",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@repo/eslint-config": "*",
+    "@repo/typescript-config": "*",
+    "@types/node": "^20.11.29",
+    "@typescript-eslint/parser": "^7.6.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "prettier": "^3.2.5",
+    "ts-node": "^10.9.2",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5"
+  }
+}
@@ -0,0 +1,4 @@
+{
+  "trailingComma": "es5",
+  "printWidth": 120
+}
@@ -0,0 +1,4 @@
+import { env } from "../env";
+
+export const isEeAvailable: boolean =
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;
@@ -0,0 +1,58 @@
+import crypto from "crypto";
+import { env } from "../env";
+
+const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const IV_LENGTH: number = 16; // For AES, this is always 16
+
+// Alternatively: openssl rand -hex 32
+export function keyGen() {
+  return crypto.randomBytes(32).toString("hex");
+}
+
+/**
+ * Encrypts the given plain text using AES-256-GCM algorithm.
+ *
+ * @param {string} plainText - The text to encrypt.
+ * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
+ * @returns {string} The encrypted data in hex format, including IV and authentication tag.
+ */
+export function encrypt(
+  plainText: string,
+  keyHex: string = ENCRYPTION_KEY
+): string {
+  const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
+  const cipher = crypto.createCipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  let encrypted = cipher.update(plainText, "utf8", "hex");
+  encrypted += cipher.final("hex");
+  const authTag = cipher.getAuthTag();
+
+  // Return iv, encrypted data, and authTag as hex, combined in one line
+  return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
+}
+
+export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+  const [ivHex, encryptedHex, authTagHex] = text.split(":");
+  if (!ivHex || !encryptedHex || !authTagHex) {
+    throw new Error("Invalid or corrupted cipher format");
+  }
+
+  const iv = Buffer.from(ivHex, "hex");
+  const encryptedText = Buffer.from(encryptedHex, "hex");
+  const authTag = Buffer.from(authTagHex, "hex");
+
+  const decipher = crypto.createDecipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  decipher.setAuthTag(authTag);
+
+  let decrypted = decipher.update(encryptedText, undefined, "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted.toString();
+}
@@ -0,0 +1,15 @@
+import { z } from "zod";
+
+const EnvSchema = z.object({
+  NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
+  ENCRYPTION_KEY: z
+    .string()
+    .length(
+      64,
+      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
+    )
+    .optional(),
+  ADMIN_API_KEY: z.string().optional(),
+});
+
+export const env = EnvSchema.parse(process.env);
@@ -0,0 +1 @@
+export { isEeAvailable } from "./ee-license-check";
@@ -0,0 +1,236 @@
+import { type Provider } from "next-auth/providers/index";
+import GoogleProvider from "next-auth/providers/google";
+import GitHubProvider from "next-auth/providers/github";
+import OktaProvider from "next-auth/providers/okta";
+import Auth0Provider from "next-auth/providers/auth0";
+import AzureADProvider from "next-auth/providers/azure-ad";
+import { isEeAvailable } from "..";
+import { prisma } from "@langfuse/shared/src/db";
+import { SsoProviderSchema } from "./types";
+import { decrypt, encrypt } from "../encryption";
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { env } from "../env";
+
+// Local cache for SSO configurations
+let cachedSsoConfigs: {
+  data: SsoProviderSchema[];
+  timestamp: number;
+} | null = null;
+
+/**
+ * Get all SSO configurations from the database or from local cache and parse them into SsoProviderSchema objects.
+ *
+ * @returns {Promise<SsoProviderSchema[]>} - A list of all SSO configurations. Empty array if none are configured or EE is not available.
+ */
+async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
+  if (!isEeAvailable) return [];
+  const CACHE_TTL = 60 * 1000; // 1 minute
+
+  // Set/refresh the cache if it's empty or expired
+  if (
+    cachedSsoConfigs === null ||
+    Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
+  ) {
+    // transform into zod object
+    const dbConfigs = await prisma.ssoConfig.findMany();
+    const parsedSsoConfigs = dbConfigs
+      .map((v) => {
+        try {
+          const parsedValue = SsoProviderSchema.parse(v);
+          return parsedValue;
+        } catch (e) {
+          console.error(
+            `Failed to parse SSO provider config for domain ${v.domain}`,
+            e
+          );
+          return null;
+        }
+      })
+      .filter((parsed) => parsed !== null) as SsoProviderSchema[];
+
+    cachedSsoConfigs = {
+      data: parsedSsoConfigs,
+      timestamp: Date.now(),
+    };
+  }
+
+  return cachedSsoConfigs.data;
+}
+
+/**
+ * Load all custom SSO providers from the database. To be used within `providers` in NextAuth backend configuration.
+ *
+ * @returns {Promise<Provider[]>} - A list of all custom SSO providers.
+ */
+export async function loadSsoProviders(): Promise<Provider[]> {
+  if (!isEeAvailable) return [];
+
+  const ssoConfigs = await getSsoConfigs();
+
+  const providers: Provider[] = [];
+
+  for (const dbSsoConfig of ssoConfigs) {
+    const provider = dbToNextAuthProvider(dbSsoConfig);
+    if (provider !== null) providers.push(provider);
+  }
+
+  return providers;
+}
+
+/**
+ * @returns `true` if any custom SSO provider is configured in the database.
+ */
+export async function isAnySsoConfigured(): Promise<boolean> {
+  if (!isEeAvailable) return false;
+  const ssoConfigs = await getSsoConfigs();
+  return ssoConfigs.length > 0;
+}
+
+/**
+ * Get the custom SSO providerId for a domain. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param domain - The domain to check for a custom SSO provider, e.g. "example.com".
+ * @returns `providerId` or null if none is configured or EE is not available.
+ */
+export async function getSsoAuthProviderIdForDomain(
+  domain: string
+): Promise<string | null> {
+  if (!isEeAvailable) return null;
+  const ssoConfig = (await getSsoConfigs()).find(
+    (ssoConfig) => ssoConfig.domain === domain.toLowerCase()
+  );
+
+  if (!ssoConfig) return null;
+  return getAuthProviderIdForSsoConfig(ssoConfig);
+}
+
+/**
+ * Converts a SsoProviderConfig to a NextAuth Provider instance.
+ *
+ * @param {SsoProviderSchema} provider - The SSO configuration from the database.
+ * @returns {Provider | null} - A NextAuth Provider instance or null if parsing fails or no custom credentials are used for this SSO config.
+ */
+const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
+  // If the SsoConfig does not use custom credentials, return null as no additional provider needs to be added to NextAuth
+  if (!provider.authConfig) return null;
+
+  if (provider.authProvider === "google")
+    return GoogleProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "github")
+    return GitHubProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "auth0")
+    return Auth0Provider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "okta")
+    return OktaProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "azure-ad")
+    return AzureADProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else {
+    // Type check to ensure we handle all providers
+    // eslint-disable-next-line no-unused-vars
+    const _: never = provider;
+    throw new Error(
+      `Unrecognized SSO provider for domain ${(provider as any).domain}`
+    );
+  }
+};
+
+/**
+ * Get the custom SSO providerId for a database SSO configuration. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param {DbSsoConfig} dbSsoConfig - The SSO configuration from the database.
+ * @returns {string} - The providerId used in NextAuth.
+ */
+const getAuthProviderIdForSsoConfig = (
+  dbSsoConfig: SsoProviderSchema
+): string => {
+  if (!dbSsoConfig.authConfig) return dbSsoConfig.authProvider;
+  return `${dbSsoConfig.domain}.${dbSsoConfig.authProvider}`;
+};
+
+export async function createNewSsoConfigHandler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  try {
+    if (!isEeAvailable) {
+      res.status(403).json({ error: "EE is not available" });
+      return;
+    }
+    // allow only POST requests
+    if (req.method !== "POST") {
+      res.status(405).json({ error: "Method Not Allowed" });
+      return;
+    }
+    // check if ADMIN_API_KEY is set
+    if (!env.ADMIN_API_KEY) {
+      res.status(500).json({ error: "ADMIN_API_KEY is not set" });
+      return;
+    }
+    if (!env.ENCRYPTION_KEY) {
+      res.status(500).json({ error: "ENCRYPTION_KEY is not set" });
+      return;
+    }
+    // check bearer token
+    const { authorization } = req.headers;
+    if (!authorization) {
+      res
+        .status(401)
+        .json({ error: "Unauthorized: No authorization header provided" });
+      return;
+    }
+    const [scheme, token] = authorization.split(" ");
+    if (scheme !== "Bearer" || !token || token !== env.ADMIN_API_KEY) {
+      res.status(401).json({ error: "Unauthorized: Invalid token" });
+      return;
+    }
+
+    const body = SsoProviderSchema.safeParse(req.body);
+    if (!body.success) {
+      res.status(400).json({ error: body.error });
+      return;
+    }
+
+    const { domain, authProvider, authConfig } = body.data;
+
+    const encryptedClientSecret = authConfig
+      ? {
+          ...authConfig,
+          clientSecret: encrypt(authConfig.clientSecret),
+        }
+      : undefined;
+
+    await prisma.ssoConfig.create({
+      data: {
+        domain,
+        authProvider,
+        authConfig: encryptedClientSecret,
+      },
+    });
+    res.status(201).json({
+      message: "SSO configuration created successfully",
+    });
+  } catch (e) {
+    console.log(e);
+    res.status(500).json({ error: "Internal Server Error" });
+  }
+}
@@ -0,0 +1,81 @@
+import { z } from "zod";
+
+const base = z.object({
+  domain: z.string().refine((v) => v === v.toLowerCase(), {
+    message: "Domain must be lowercase",
+  }),
+});
+
+export const GoogleProviderSchema = base.extend({
+  authProvider: z.literal("google"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const GithubProviderSchema = base.extend({
+  authProvider: z.literal("github"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const Auth0ProviderSchema = base.extend({
+  authProvider: z.literal("auth0"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const OktaProviderSchema = base.extend({
+  authProvider: z.literal("okta"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const AzureAdProviderSchema = base.extend({
+  authProvider: z.literal("azure-ad"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      tenantId: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
+export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
+export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
+export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+
+export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
+  GoogleProviderSchema,
+  GithubProviderSchema,
+  Auth0ProviderSchema,
+  OktaProviderSchema,
+  AzureAdProviderSchema,
+]);
+
+export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;
@@ -0,0 +1,14 @@
+{
+  "extends": "@repo/typescript-config/base.json",
+  "compilerOptions": {
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "types": ["node"],
+    "target": "ES2020",
+    "rootDir": "."
+  },
+  "include": ["."],
+  "exclude": ["node_modules", "dist"]
+}
@@ -127,6 +127,7 @@ types:
      id: string
      name: string
      description: optional<string>
+      metadata: optional<unknown>
      projectId: string
      createdAt: datetime
      updatedAt: datetime
@@ -151,15 +152,18 @@ types:
      status: DatasetStatus
      input: optional<unknown>
      expectedOutput: optional<unknown>
+      metadata: optional<unknown>
      sourceTraceId: optional<string>
      sourceObservationId: optional<string>
      datasetId: string
+      datasetName: string
      createdAt: datetime
      updatedAt: datetime
  DatasetRunItem:
    properties:
      id: string
      datasetRunId: string
+      datasetRunName: string
      datasetItemId: string
      traceId: string
      observationId: optional<string>
@@ -207,6 +211,7 @@ types:
    enum:
      - API
      - REVIEW
+      - EVAL

 errors:
  Error:
@@ -26,6 +26,9 @@ types:
      datasetName: string
      input: optional<unknown>
      expectedOutput: optional<unknown>
+      metadata: optional<unknown>
+      sourceTraceId: optional<string>
+      sourceObservationId: optional<string>
      id:
        type: optional<string>
        docs: Dataset items are upserted on their id
@@ -47,3 +47,4 @@ types:
    properties:
      name: string
      description: optional<string>
+      metadata: optional<unknown>
@@ -29,6 +29,12 @@ service:
            type: optional<string>
            allow-multiple: true
            docs: Optional filter for metrics where traces include all of these tags
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces on or after a certain timestamp
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces before a certain timestamp
      response: DailyMetrics
 types:
  DailyMetrics:
@@ -25,6 +25,20 @@ service:

 types:
  CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
    properties:
      name: string
      isActive:
@@ -34,8 +48,29 @@ types:
      config: optional<unknown>

  Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
    properties:
      name: string
      version: integer
-      prompt: string
      config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>
@@ -30,6 +30,15 @@ service:
          fromTimestamp:
            type: optional<datetime>
            docs: Retrieve only scores newer than this timestamp.
+          source:
+            type: commons.ScoreSource
+            docs: Retrieve only scores from a specific source.
+          operator:
+            type: optional<string>
+            docs: Retrieve only scores with <operator> value.
+          value:
+            type: optional<double>
+            docs: Retrieve only scores with <operator> value.
      response: Scores
    get-by-id:
      docs: Get a score
@@ -1,4 +1,4 @@
 {
  "organization": "finto",
-  "version": "0.19.30"
+  "version": "0.21.0"
 }
@@ -65,7 +65,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -245,7 +245,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\",\n    \"metadata\": \"UNKNOWN\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -379,7 +379,7 @@
          "request": {
            "description": "Get daily metrics of the Langfuse project",
            "url": {
-              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=",
+              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=&fromTimestamp=&toTimestamp=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -414,6 +414,16 @@
                  "key": "tags",
                  "value": "",
                  "description": "Optional filter for metrics where traces include all of these tags"
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces on or after a certain timestamp"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces before a certain timestamp"
                }
              ],
              "variable": []
@@ -628,7 +638,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -684,7 +694,7 @@
          "request": {
            "description": "Get a list of scores",
            "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -718,6 +728,21 @@
                  "key": "fromTimestamp",
                  "value": "",
                  "description": "Retrieve only scores newer than this timestamp."
+                },
+                {
+                  "key": "source",
+                  "value": "",
+                  "description": "Retrieve only scores from a specific source."
+                },
+                {
+                  "key": "operator",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "value",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
                }
              ],
              "variable": []
@@ -1,10 +1,12 @@
 {
  "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.29.2",
  "author": "engineering@langfuse.com",
  "license": "MIT",
+  "private": true,
  "engines": {
-    "node": ">=20"
+    "node": ">=20",
+    "pnpm": "<9"
  },
  "scripts": {
    "preinstall": "npx only-allow pnpm",
@@ -21,11 +23,51 @@
    "dev": "turbo run dev",
    "lint": "turbo run lint",
    "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate"
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
  },
  "devDependencies": {
+    "@release-it/bumper": "^6.0.1",
    "dotenv-cli": "^7.4.1",
    "prettier": "^3.2.5",
+    "release-it": "^17.2.0",
    "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
  }
 }
@@ -8,12 +8,13 @@
    "next.js"
  ],
  "devDependencies": {
-    "@vercel/style-guide": "^6.0.0",
-    "eslint-config-turbo": "^1.12.4",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-only-warn": "^1.1.0",
-    "@typescript-eslint/parser": "^7.1.0",
    "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "typescript": "^5.4.4"
+    "@typescript-eslint/parser": "^7.6.0",
+    "@vercel/style-guide": "^6.0.0",
+    "eslint-config-next": "^14.2.1",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-turbo": "^1.13.2",
+    "eslint-plugin-only-warn": "^1.1.0",
+    "typescript": "^5.4.5"
  }
 }
@@ -5,12 +5,23 @@
  "private": true,
  "main": "./dist/src/index.js",
  "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
  "engines": {
    "node": ">=20"
  },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    }
+  },
  "scripts": {
    "build": "tsc",
    "dev": "tsc --watch",
@@ -28,26 +39,26 @@
    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
  },
  "dependencies": {
-    "@langchain/anthropic": "^0.1.10",
-    "@langchain/core": "^0.1.54",
-    "@langchain/openai": "^0.0.26",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.58",
+    "@langchain/openai": "^0.0.28",
    "@prisma/client": "^5.12.1",
    "@types/bcryptjs": "^2.4.6",
    "bcryptjs": "^2.4.3",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.31",
+    "langchain": "^0.1.33",
    "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
  },
  "devDependencies": {
    "@repo/eslint-config": "*",
    "@repo/typescript-config": "*",
    "@types/lodash": "^4.14.202",
    "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.4",
+    "@types/pg": "^8.11.5",
    "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.2.0",
+    "@typescript-eslint/parser": "^7.6.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
@@ -57,10 +68,10 @@
    "prettier": "^3.2.5",
    "prisma": "^5.12.1",
    "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
    "ts-node": "^10.9.2",
-    "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
-    "typescript": "^5.4.4",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5",
    "vitest": "^1.3.1"
  }
 }
@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
    API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,22 @@ export const DatasetStatus = {
    ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobConfigState = {
+    ACTIVE: "ACTIVE",
+    INACTIVE: "INACTIVE"
+} as const;
+export type JobConfigState = (typeof JobConfigState)[keyof typeof JobConfigState];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
    id: string;
    user_id: string;
@@ -96,6 +113,7 @@ export type Dataset = {
    id: string;
    name: string;
    description: string | null;
+    metadata: unknown | null;
    project_id: string;
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
@@ -105,6 +123,7 @@ export type DatasetItem = {
    status: Generated<DatasetStatus>;
    input: unknown | null;
    expected_output: unknown | null;
+    metadata: unknown | null;
    source_trace_id: string | null;
    source_observation_id: string | null;
    dataset_id: string;
@@ -129,6 +148,19 @@ export type DatasetRuns = {
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
    id: string;
    created_at: Generated<Timestamp>;
@@ -139,6 +171,34 @@ export type Events = {
    url: string | null;
    method: string | null;
 };
+export type JobConfiguration = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    status: Generated<JobConfigState>;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
+};
 export type Membership = {
    project_id: string;
    user_id: string;
@@ -253,11 +313,13 @@ export type Prompt = {
    updated_at: Generated<Timestamp>;
    project_id: string;
    created_by: string;
-    prompt: string;
+    prompt: unknown;
    name: string;
    version: number;
+    type: Generated<string>;
    is_active: boolean;
    config: Generated<unknown>;
+    tags: Generated<string[]>;
 };
 export type Score = {
    id: string;
@@ -275,6 +337,13 @@ export type Session = {
    user_id: string;
    expires: Timestamp;
 };
+export type SsoConfig = {
+    domain: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    auth_provider: string;
+    auth_config: unknown | null;
+};
 export type Trace = {
    id: string;
    external_id: string | null;
@@ -344,7 +413,10 @@ export type DB = {
    dataset_run_items: DatasetRunItems;
    dataset_runs: DatasetRuns;
    datasets: Dataset;
+    eval_templates: EvalTemplate;
    events: Events;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
    membership_invitations: MembershipInvitation;
    memberships: Membership;
    models: Model;
@@ -355,6 +427,7 @@ export type DB = {
    prompts: Prompt;
    scores: Score;
    Session: Session;
+    sso_configs: SsoConfig;
    trace_sessions: TraceSession;
    traces: Trace;
    traces_view: TraceView;
@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;
@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_tags_idx" ON "prompts" USING GIN ("tags" array_ops);
@@ -0,0 +1,30 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://ai.google.dev/models/gemini#model-variations
+  -- https://cloud.google.com/vertex-ai/generative-ai/pricing
+  ('cluv2sjeo000008ih0fv23hi0', NULL, 'gemini-1.0-pro-latest', '(?i)^(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  -- stable versions priced differently
+  -- https://developers.googleblog.com/2024/02/gemini-15-available-for-private-preview-in-google-ai-studio.html
+  ('cluv2subq000108ih2mlrga6a', NULL, 'gemini-1.0-pro', '(?i)^(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2sx04000208ihbek75lsz', NULL, 'gemini-1.0-pro-001', '(?i)^(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2szw0000308ihch3n79x7', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+
+  ('cluv2t2x0000408ihfytl45l1', NULL, 'gemini-1.5-pro-latest', '(?i)^(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
+
+
+  
@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "JobConfigState" AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- AlterTable
+ALTER TABLE "job_configurations" ADD COLUMN     "status" "JobConfigState" NOT NULL DEFAULT 'ACTIVE';
@@ -0,0 +1,16 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluvpl4ls000008l6h2gx3i07', NULL, 'gpt-4-turbo', '(?i)^(gpt-4-turbo)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
@@ -0,0 +1,10 @@
+-- CreateTable
+CREATE TABLE "sso_configs" (
+    "domain" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "auth_provider" TEXT NOT NULL,
+    "auth_config" JSONB,
+
+    CONSTRAINT "sso_configs_pkey" PRIMARY KEY ("domain")
+);
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "models_model_name_idx" ON "models"("model_name");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_internal_model_idx" ON "observations"("internal_model");
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "metadata" JSONB;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "metadata" JSONB;
@@ -92,22 +92,25 @@ model VerificationToken {
 }

 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id               String                 @id @default(cuid())
+    createdAt        DateTime               @default(now()) @map("created_at")
+    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
+    name             String
+    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members          Membership[]
+    traces           Trace[]
+    observations     Observation[]
+    apiKeys          ApiKey[]
+    dataset          Dataset[]
+    RawEvents        Events[]
+    invitations      MembershipInvitation[]
+    sessions         TraceSession[]
+    Prompt           Prompt[]
+    Model            Model[]
+    AuditLog         AuditLog[]
+    EvalTemplate     EvalTemplate[]
+    JobConfiguration JobConfiguration[]
+    JobExecution     JobExecution[]

    @@map("projects")
 }
@@ -209,6 +212,7 @@ model Trace {
    scores          Score[]
    DatasetRunItems DatasetRunItems[]
    DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]

    @@index([projectId])
    @@index([sessionId])
@@ -298,6 +302,7 @@ model Observation {
    @@index(projectId)
    @@index(parentObservationId)
    @@index(model)
+    @@index(internalModel)
    @@index(promptId)
    @@index([projectId, startTime, type])
    @@map("observations")
@@ -362,16 +367,17 @@ enum ObservationLevel {
 }

 model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
    name          String
    value         Float
    source        ScoreSource
    comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]

    @@unique([id, traceId]) // used for upsert
    @@index(timestamp)
@@ -385,6 +391,7 @@ model Score {
 enum ScoreSource {
    API
    REVIEW
+    EVAL
 }

 enum PricingUnit {
@@ -423,6 +430,7 @@ model Dataset {
    id           String        @id @default(cuid())
    name         String
    description  String?
+    metadata     Json?
    projectId    String        @map("project_id")
    project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
    createdAt    DateTime      @default(now()) @map("created_at")
@@ -440,6 +448,7 @@ model DatasetItem {
    status              DatasetStatus     @default(ACTIVE)
    input               Json?
    expectedOutput      Json?             @map("expected_output")
+    metadata            Json?
    sourceTraceId       String?           @map("source_trace_id")
    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
    sourceObservationId String?           @map("source_observation_id")
@@ -521,17 +530,20 @@ model Prompt {

    createdBy String @map("created_by")

-    prompt      String
+    prompt      Json
    name        String
    version     Int
+    type        String        @default("text")
    isActive    Boolean       @map("is_active")
    config      Json          @default("{}")
+    tags        String[]      @default([])
    Observation Observation[]

    @@unique([projectId, name, version])
    @@index([projectId, name, version])
    @@index([projectId, id])
    @@index([projectId])
+    @@index([tags(ops: ArrayOps)], type: Gin)
    @@map("prompts")
 }

@@ -557,6 +569,7 @@ model Model {
    @@unique([projectId, modelName, startDate, unit])
    @@index([projectId, modelName, startDate, unit])
    @@index([projectId, modelName])
+    @@index(modelName)
    @@map("models")
 }

@@ -579,3 +592,110 @@ model AuditLog {
    @@index([createdAt])
    @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+enum JobConfigState {
+    ACTIVE
+    INACTIVE
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType        @map("job_type")
+    status         JobConfigState @default(ACTIVE)
+    evalTemplateId String?        @map("eval_template_id")
+    evalTemplate   EvalTemplate?  @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}
+
+// Single Sign-On configuration for a domain
+// This feature is part of the Enterprise Edition
+model SsoConfig {
+    domain       String   @id @default(cuid()) // e.g. "google.com"
+    createdAt    DateTime @default(now()) @map("created_at")
+    updatedAt    DateTime @default(now()) @updatedAt @map("updated_at")
+    authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
+
+    authConfig Json? @map("auth_config")
+    // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
+    // secrets like clientSecret are encrypted on the application level
+
+    @@map("sso_configs")
+}
@@ -4,13 +4,14 @@ import {
  type Prisma,
  ObservationType,
  ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";

 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";

 const LOAD_TRACE_VOLUME = 10_000;

@@ -164,6 +165,74 @@ async function main() {

    await uploadObjects(traces, observations, scores, sessions, events);

+    // add eval objects
+    const evalTemplate = await prisma.evalTemplate.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project1.id,
+          name: "toxicity-template",
+          version: 1,
+        },
+      },
+      create: {
+        projectId: project1.id,
+        name: "toxicity-template",
+        version: 1,
+        prompt:
+          "Please evaluate the toxicity of the following text {{input}} {{output}}",
+        model: "gpt-3.5-turbo",
+        vars: ["input", "output"],
+        outputSchema: {
+          score: "provide a score between 0 and 1",
+          reasoning: "one sentence reasoning for the score",
+        },
+        modelParams: {
+          temperature: 0.7,
+          outputTokenLimit: 100,
+          topP: 0.9,
+        },
+      },
+      update: {},
+    });
+
+    await prisma.jobConfiguration.upsert({
+      where: {
+        id: "toxicity-job",
+      },
+      create: {
+        id: "toxicity-job",
+        evalTemplateId: evalTemplate.id,
+        projectId: project1.id,
+        jobType: "EVAL",
+        status: "ACTIVE",
+        scoreName: "toxicity",
+        filter: [
+          {
+            type: "string",
+            value: "user",
+            column: "User ID",
+            operator: "contains",
+          },
+        ],
+        variableMapping: [
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "input",
+            templateVariable: "input",
+          },
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "metadata",
+            templateVariable: "output",
+          },
+        ],
+        targetObject: "trace",
+        sampling: 1,
+        delay: 5_000,
+      },
+      update: {},
+    });
+
    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
      const dataset = await prisma.dataset.create({
        data: {
@@ -171,6 +240,7 @@ async function main() {
          description:
            datasetNumber === 0 ? "Dataset test description" : undefined,
          projectId: project2.id,
+          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
        },
      });

@@ -199,6 +269,7 @@ async function main() {
              Math.random() > 0.3
                ? "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both."
                : undefined,
+            metadata: Math.random() > 0.5 ? { key: "value" } : undefined,
          },
        });
        datasetItemIds.push(datasetItem.id);
@@ -713,6 +784,16 @@ async function generatePrompts(project: Project) {
      version: 1,
      isActive: true,
    },
+    {
+      id: `prompt-${v4()}`,
+      projectId: project.id,
+      createdBy: "user-1",
+      prompt: "Prompt 4 content",
+      name: "Prompt 4",
+      version: 1,
+      isActive: true,
+      tags: ["tag1", "tag2"],
+    },
  ];

  for (const prompt of prompts) {
@@ -732,6 +813,7 @@ async function generatePrompts(project: Project) {
        name: prompt.name,
        version: prompt.version,
        isActive: prompt.isActive,
+        tags: prompt.tags,
      },
      update: {
        id: prompt.id,
@@ -3,6 +3,14 @@

 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";

 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
@@ -16,12 +24,32 @@ const prismaClientSingleton = () => {
  });
 };

+const kyselySingleton = (prismaClient: PrismaClient) => {
+  return prismaClient.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    })
+  );
+};
 declare global {
  // eslint-disable-next-line no-var
  var prisma: undefined | ReturnType<typeof prismaClientSingleton>;
+  var kyselyPrisma: undefined | ReturnType<typeof kyselySingleton>;
 }

 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+export const kyselyPrisma = globalThis.kyselyPrisma ?? kyselySingleton(prisma);
+
 export * from "@prisma/client";

 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;
@@ -0,0 +1,113 @@
+import z from "zod";
+import { ModelProvider, observationsTableCols } from "../..";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+// variable mapping stored in the db for eval templates
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(), // variable name in the template
+    // name of the observation to extract the variable from
+    // not required for trace, as we only have one.
+    objectName: z.string().nullish(),
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});
+
+export const availableEvalVariables = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 't."metadata"',
+      },
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];
+
+export const evalLLMModels = [
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-3.5-turbo",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-4-turbo-preview",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+] as const;
+
+export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;
@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
  type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";

 const operatorReplacements = {
  "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
  if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const internalFilters = filters.map((filter) => {
    // Get column definition to map column to internal name, e.g. "t.id"
    const col = tableColumns.find(
      (c) =>
        // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
    );
    if (!col) {
      console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
        ? Prisma.raw(
            arrayOperatorReplacements[
              filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
          )
        : filter.operator in operatorReplacements
          ? Prisma.raw(
              operatorReplacements[
                filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
            )
          : Prisma.raw(filter.operator); //checked by zod

@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
        break;
      case "stringOptions":
        valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
        )})`;
        break;
      case "arrayOptions":
        valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
          filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
        )}] `;
        break;

@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
      filter.type === "string" || filter.type === "stringObject"
        ? [
            ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw("'%' || ")
              : Prisma.empty,
            ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw(" || '%'")
              : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(

 const castValueToPostgresTypes = (
  column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
  return column.name === "type" &&
    (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
  safeColumn: string,
  operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
  if (!dateOperators.includes(operator)) {
    throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
  }

  return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
  )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };
@@ -1,9 +1,22 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./observationsTable";

+// llm api
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";

-export { ModelUsageUnit };
+// evals
+export * from "./features/evals/types";
+
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";
@@ -20,7 +20,7 @@ export const filterOperators = {
 export const timeFilter = z.object({
  column: z.string(),
  operator: z.enum(filterOperators.datetime),
-  value: z.date(),
+  value: z.coerce.date(), // coerce required to parse stringified dates from the db in evals
  type: z.literal("datetime"),
 });
 export const stringFilter = z.object({
@@ -41,6 +41,7 @@ export const tableNames = [
  "traces_metrics",
  "traces_parent_observation_scores",
  "sessions",
+  "prompts",
 ] as const;

 export type TableNames = (typeof tableNames)[number];
@@ -1,8 +1,8 @@
+import { ObservationLevel } from "@prisma/client";
 import {
  type OptionsDefinition,
  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";

 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -164,7 +164,7 @@ export type ObservationOptions = {
 };

 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
  return observationsTableCols.map((col) => {
    if (col.id === "model") {
@@ -0,0 +1,36 @@
+import { z } from "zod";
+
+export const TraceUpsertEvent = z.object({
+  projectId: z.string(),
+  traceId: z.string(),
+});
+
+export const EvalExecutionEvent = z.object({
+  projectId: z.string(),
+  jobExecutionId: z.string(),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    timestamp: Date;
+    id: string;
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    timestamp: Date;
+    id: string;
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};
@@ -15,6 +15,7 @@ export function getDisplaySecretKey(secretKey: string) {
 }

 export async function hashSecretKey(key: string) {
+  // legacy, uses bcrypt, transformed into hashed key upon first use
  const hashedKey = await hash(key, 11);
  return hashedKey;
 }
@@ -4,6 +4,7 @@ import {
  HumanMessage,
  SystemMessage,
 } from "@langchain/core/messages";
+import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 import {
  BytesOutputParser,
  StringOutputParser,
@@ -14,13 +15,18 @@ import { ChatOpenAI } from "@langchain/openai";
 import {
  ChatMessage,
  ChatMessageRole,
+  LLMFunctionCall,
  ModelParams,
  ModelProvider,
 } from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";

 type LLMCompletionParams = {
  messages: ChatMessage[];
  modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
+  callbacks?: BaseCallbackHandler[];
 };

 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -39,10 +45,17 @@ export async function fetchLLMCompletion(
  }
 ): Promise<string>;

+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array>> {
-  const { messages, modelParams, streaming } = params;
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
+  const { messages, modelParams, streaming, callbacks } = params;
  const finalMessages = messages.map((message) => {
    if (message.role === ChatMessageRole.User)
      return new HumanMessage(message.content);
@@ -60,6 +73,7 @@ export async function fetchLLMCompletion(
      temperature: modelParams.temperature,
      maxTokens: modelParams.max_tokens,
      topP: modelParams.top_p,
+      callbacks,
    });
  } else {
    chatModel = new ChatOpenAI({
@@ -68,9 +82,26 @@ export async function fetchLLMCompletion(
      temperature: modelParams.temperature,
      maxTokens: modelParams.max_tokens,
      topP: modelParams.top_p,
+      callbacks,
    });
  }

+  console.log("Making LLM call with params: ", modelParams);
+
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
  if (streaming) {
    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
  }
@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
  role: ChatMessageRole;
  content: string;
@@ -23,11 +26,13 @@ export type UIModelParams = Required<
 };

 // Generic config
-export type ModelConfig = {
-  max_tokens?: number;
-  temperature?: number;
-  top_p?: number;
-};
+export type ModelConfig = z.infer<typeof ZodModelConfig>;
+
+export const ZodModelConfig = z.object({
+  max_tokens: z.number().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+});

 // OpenAI
 export type OpenAIModelParams = {
@@ -72,3 +77,9 @@ export const supportedModels = {
  [ModelProvider.Anthropic]: anthropicModels,
  [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};
@@ -1,8 +1,5 @@
-import {
-  type OptionsDefinition,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
 import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";

 export const tracesTableCols: ColumnDefinition[] = [
  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
@@ -122,7 +119,7 @@ export type TraceOptions = {
 };

 export function tracesTableColsWithOptions(
-  options?: TraceOptions,
+  options?: TraceOptions
 ): ColumnDefinition[] {
  return tracesTableCols.map((col) => {
    if (col.id === "scores_avg") {
@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";

 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;
@@ -29,4 +29,5 @@ export type TableName =
  | "generations"
  | "sessions"
  | "scores"
+  | "prompts"
  | "dashboard";
@@ -6,7 +6,8 @@
    "lib": ["ES2020"],
    "outDir": "./dist",
    "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
  },
  "include": ["."],
  "exclude": ["node_modules", "dist"]
@@ -3,3 +3,4 @@ packages:
  - "web"
  - "worker"
  - "packages/**"
+  - "ee"
@@ -5,10 +5,11 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat

 FROM alpine AS base
-RUN npm install pnpm turbo --global
+RUN npm install turbo --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
+RUN corepack prepare pnpm@8.15.5 --activate


 FROM base AS pruner
@@ -482,6 +482,24 @@ paths:
            items:
              type: string
              nullable: true
+        - name: fromTimestamp
+          in: query
+          description: >-
+            Optional filter to only include traces on or after a certain
+            timestamp
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: Optional filter to only include traces before a certain timestamp
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
      responses:
        '200':
          description: ''
@@ -879,6 +897,27 @@ paths:
            type: string
            format: date-time
            nullable: true
+        - name: source
+          in: query
+          description: Retrieve only scores from a specific source.
+          required: true
+          schema:
+            $ref: '#/components/schemas/ScoreSource'
+        - name: operator
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: value
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: number
+            format: double
+            nullable: true
      responses:
        '200':
          description: ''
@@ -1490,6 +1529,8 @@ components:
        description:
          type: string
          nullable: true
+        metadata:
+          nullable: true
        projectId:
          type: string
        createdAt:
@@ -1553,6 +1594,8 @@ components:
          nullable: true
        expectedOutput:
          nullable: true
+        metadata:
+          nullable: true
        sourceTraceId:
          type: string
          nullable: true
@@ -1561,6 +1604,8 @@ components:
          nullable: true
        datasetId:
          type: string
+        datasetName:
+          type: string
        createdAt:
          type: string
          format: date-time
@@ -1571,6 +1616,7 @@ components:
        - id
        - status
        - datasetId
+        - datasetName
        - createdAt
        - updatedAt
    DatasetRunItem:
@@ -1581,6 +1627,8 @@ components:
          type: string
        datasetRunId:
          type: string
+        datasetRunName:
+          type: string
        datasetItemId:
          type: string
        traceId:
@@ -1597,6 +1645,7 @@ components:
      required:
        - id
        - datasetRunId
+        - datasetRunName
        - datasetItemId
        - traceId
        - createdAt
@@ -1679,6 +1728,7 @@ components:
      enum:
        - API
        - REVIEW
+        - EVAL
    CreateDatasetItemRequest:
      title: CreateDatasetItemRequest
      type: object
@@ -1689,6 +1739,14 @@ components:
          nullable: true
        expectedOutput:
          nullable: true
+        metadata:
+          nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
+        sourceObservationId:
+          type: string
+          nullable: true
        id:
          type: string
          nullable: true
@@ -1744,6 +1802,8 @@ components:
        description:
          type: string
          nullable: true
+        metadata:
+          nullable: true
      required:
        - name
    HealthResponse:
@@ -2412,6 +2472,50 @@ components:
        - name
    CreatePromptRequest:
      title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
      type: object
      properties:
        name:
@@ -2429,20 +2533,75 @@ components:
        - prompt
    Prompt:
      title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
      type: object
      properties:
        name:
          type: string
        version:
          type: integer
-        prompt:
-          type: string
        config: {}
      required:
        - name
        - version
-        - prompt
        - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
    CreateScoreRequest:
      title: CreateScoreRequest
      type: object
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "2.16.2",
+  "version": "2.29.2",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -18,21 +18,22 @@
    "test": "dotenv -e ../.env -- jest --runInBand",
    "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
    "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.540.0",
-    "@aws-sdk/s3-request-presigner": "^3.540.0",
-    "@headlessui/react": "^1.7.18",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@codemirror/lang-json": "^6.0.1",
+    "@headlessui/react": "^1.7.19",
    "@heroicons/react": "^2.1.3",
    "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.10",
-    "@langchain/core": "^0.1.54",
-    "@langchain/openai": "^0.0.26",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.58",
+    "@langchain/openai": "^0.0.28",
    "@langfuse/shared": "workspace:*",
+    "@langfuse/ee": "workspace:*",
    "@marsidev/react-turnstile": "^0.5.4",
    "@next-auth/prisma-adapter": "^1.0.7",
    "@radix-ui/react-accordion": "^1.1.2",
@@ -59,10 +60,10 @@
    "@react-email/render": "^0.0.12",
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.109.0",
-    "@sentry/node": "^7.109.0",
-    "@sentry/profiling-node": "^7.109.0",
-    "@sentry/types": "^7.109.0",
+    "@sentry/nextjs": "^7.110.1",
+    "@sentry/node": "^7.110.1",
+    "@sentry/profiling-node": "^7.110.1",
+    "@sentry/types": "^7.110.1",
    "@t3-oss/env-nextjs": "^0.8.0",
    "@tailwindcss/forms": "^0.5.7",
    "@tanstack/react-query": "^4.36.1",
@@ -72,32 +73,34 @@
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
-    "ai": "^3.0.18",
+    "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/react-codemirror": "^4.21.25",
+    "ai": "^3.0.23",
    "bcryptjs": "^2.4.3",
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.0",
-    "cmdk": "^0.2.1",
+    "cmdk": "^1.0.0",
    "core-js": "^3.36.1",
    "cors": "^2.8.5",
    "date-fns": "^3.3.1",
    "decimal.js": "^10.4.3",
    "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.10",
+    "js-tiktoken": "^1.0.11",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.31",
+    "langchain": "^0.1.33",
    "lodash": "^4.17.21",
    "lucide-react": "^0.364.0",
-    "next": "^14.1.4",
+    "next": "^14.2.1",
    "next-auth": "^4.24.7",
    "next-query-params": "^5.0.0",
    "nodemailer": "^6.9.13",
-    "posthog-js": "^1.105.9",
+    "posthog-js": "^1.122.0",
    "posthog-node": "^3.6.3",
    "prisma": "^5.12.1",
    "react": "18.2.0",
-    "react-day-picker": "^8.10.0",
+    "react-day-picker": "^8.10.1",
    "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.2",
+    "react-hook-form": "^7.51.3",
    "react-icons": "^5.0.1",
    "react-responsive": "^9.0.2",
    "react18-json-view": "^0.2.8-canary.6",
@@ -112,10 +115,9 @@
  "devDependencies": {
    "@jedmao/location": "^3.0.0",
    "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.1",
    "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^14.2.2",
+    "@testing-library/react": "^15.0.2",
    "@types/bcryptjs": "^2.4.6",
    "@types/cors": "^2.8.17",
    "@types/eslint": "^8.56.7",
@@ -123,57 +125,31 @@
    "@types/lodash": "^4.14.202",
    "@types/node": "20.10.5",
    "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.74",
-    "@types/react-dom": "^18.2.24",
+    "@types/react": "^18.2.79",
+    "@types/react-dom": "^18.2.25",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^6.21.0",
+    "@typescript-eslint/parser": "^7.6.0",
    "autoprefixer": "^10.4.19",
    "dotenv-cli": "^7.3.0",
    "eslint": "^8.56.0",
-    "eslint-config-next": "^14.1.4",
+    "eslint-config-next": "^14.2.1",
    "jest": "^29.7.0",
    "jest-environment-jsdom": "^29.7.0",
    "node-mocks-http": "^1.14.1",
    "postcss": "^8.4.38",
    "prettier": "^3.2.5",
-    "prettier-plugin-tailwindcss": "^0.5.13",
-    "release-it": "^17.1.1",
+    "prettier-plugin-tailwindcss": "^0.5.14",
    "tailwindcss": "^3.4.3",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
    "tsx": "^4.7.2",
-    "typescript": "^5.4.4"
+    "typescript": "^5.4.5"
  },
  "ct3aMetadata": {
    "initVersion": "7.13.0"
  },
  "optionalDependencies": {
    "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
  }
 }
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>
@@ -10,7 +10,7 @@ test("should redirect to home if signed in", async ({ page }) => {
  await page.goto("/auth/sign-in");
  await page.fill('input[name="email"]', "demo@langfuse.com");
  await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');

  // wait 2 seconds
  await page.waitForTimeout(2000);
@@ -32,7 +32,7 @@ test("Successfully sign up & able to go to homepage", async ({ page }) => {
  await page.fill('input[name="name"]', "demo lang");
  await page.fill('input[name="email"]', randomEmailAddress());
  await page.fill('input[type="password"]', "password2");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
  await page.waitForTimeout(2000);
  // see get started page
  await expect(page).toHaveURL("/?getStarted=1");
@@ -45,7 +45,7 @@ test("Successfully sign up & able to go to homepage with uppercase email", async
  await page.fill('input[name="name"]', "demo lang");
  await page.fill('input[name="email"]', "A" + randomEmailAddress());
  await page.fill('input[type="password"]', "password3");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
  await page.waitForTimeout(2000);
  // see get started page
  await expect(page).toHaveURL("/?getStarted=1");
@@ -55,13 +55,13 @@ test("Signup input validation", async ({ page }) => {
  await page.goto("auth/sign-up");
  await page.fill('input[name="email"]', "notanemail");
  await page.fill('input[type="password"]', "shortPw");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
  await page.waitForTimeout(2000);
  await expect(page.getByText("Invalid email")).toBeVisible();
  await expect(
    page.getByText("Password must be at least 8 characters long"),
  ).toBeVisible();
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
  await page.waitForTimeout(2000);
  // don't see get started page
  await expect(page).not.toHaveURL("/?getStarted=1");
@@ -7,7 +7,7 @@ test("should see new projects dialog open after clicking new project btn", async
  await page.fill('input[name="name"]', "demo user");
  await page.fill('input[name="email"]', randomEmailAddress());
  await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
  await page.waitForTimeout(2000);
  await page.waitForTimeout(2000);
  expect(await page.innerHTML("data-testid=create-new-project-title")).toBe(
@@ -24,7 +24,7 @@ test("Create a project with provided name", async ({ page }) => {
  await page.goto("/auth/sign-in");
  await page.fill('input[name="email"]', "demo@langfuse.com");
  await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
  await page.waitForTimeout(2000);
  await page.isVisible('[data-testid="new-project-form"]');
  await page.click('[data-testid="create-project-btn"]');
@@ -1,4 +1,7 @@
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
+import {
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server/auth";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { prisma } from "@langfuse/shared/src/db";

@@ -5,56 +5,12 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 } from "uuid";

 describe("/api/public/datasets and /api/public/dataset-items API Endpoints", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
-
-  it("should create and get a dataset", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name",
-      description: "dataset-description",
-    });
-
-    const dbDataset = await prisma.dataset.findMany({
-      where: {
-        name: "dataset-name",
-      },
-    });
-
-    expect(dbDataset.length).toBeGreaterThan(0);
-
-    const getDataset = await makeAPICall(
-      "GET",
-      `/api/public/datasets/dataset-name`,
-    );
-
-    expect(getDataset.status).toBe(200);
-    expect(getDataset.body).toMatchObject({
-      name: "dataset-name",
-      description: "dataset-description",
-    });
-  });
-
-  it("GET datasets", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-1",
-    });
-
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-2",
-    });
-
-    const datasetItemId = v4();
-
-    await makeAPICall("POST", "/api/public/dataset-items", {
-      datasetName: "dataset-name-2",
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-      id: datasetItemId,
-    });
-
-    const traceId = v4();
-    const observationId = v4();
+  const traceId = v4();
+  const observationId = v4();
+  beforeEach(async () => {
+    await pruneDatabase();

+    // create sample trace and observation
    const response = await makeAPICall("POST", "/api/public/ingestion", {
      batch: [
        {
@@ -90,6 +46,62 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      ],
    });
    expect(response.status).toBe(207);
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("should create and get a dataset", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
+    });
+
+    const dbDataset = await prisma.dataset.findMany({
+      where: {
+        name: "dataset-name",
+      },
+    });
+
+    expect(dbDataset.length).toBeGreaterThan(0);
+
+    const getDataset = await makeAPICall(
+      "GET",
+      `/api/public/datasets/dataset-name`,
+    );
+
+    expect(getDataset.status).toBe(200);
+    expect(getDataset.body).toMatchObject({
+      name: "dataset-name",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
+    });
+  });
+
+  it("GET datasets", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-1",
+    });
+
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-2",
+    });
+
+    const datasetItemId = v4();
+
+    const createItemRes = await makeAPICall<{
+      datasetName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name-2",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
+      id: datasetItemId,
+    });
+
+    expect(createItemRes.status).toBe(200);
+    expect(createItemRes.body).toMatchObject({
+      datasetName: "dataset-name-2", // not included in db table
+    });

    await makeAPICall("POST", "/api/public/dataset-run-items", {
      datasetItemId: datasetItemId,
@@ -129,6 +141,9 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      datasetName: "dataset-name",
      input: { key: "value" },
      expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
+      sourceTraceId: traceId,
+      sourceObservationId: observationId,
    });
    const dbDatasetItem = await prisma.datasetItem.findFirst({
      where: {
@@ -152,6 +167,8 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
          id: dbDatasetItem!.id,
          input: { key: "value" },
          expectedOutput: { key: "value" },
+          metadata: { key: "value-dataset-item" },
+          datasetName: "dataset-name", // not included in db table
        },
      ],
    });
@@ -165,6 +182,10 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      id: dbDatasetItem!.id,
      input: { key: "value" },
      expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
+      datasetName: "dataset-name", // not included in db table
+      sourceTraceId: traceId,
+      sourceObservationId: observationId,
    });
  });

@@ -177,6 +198,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      id: "dataset-item-id",
      datasetName: "dataset-name",
      input: { key: "value" },
+      metadata: { key: "value-dataset-item" },
    });
    expect(item1.status).toBe(200);
    expect(item1.body).toMatchObject({
@@ -187,6 +209,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      id: "dataset-item-id",
      datasetName: "dataset-name",
      input: { key: "value2" },
+      metadata: ["hello-world"],
    });
    expect(item2.status).toBe(200);
    expect(item2.body).toMatchObject({
@@ -199,6 +222,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
    });
    expect(dbDatasetItem).not.toBeNull();
    expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
+    expect(dbDatasetItem?.metadata).toMatchObject(["hello-world"]);
  });

  it("should create and get a dataset run", async () => {
@@ -303,20 +327,25 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
          datasetItemId: "dataset-item-id",
          observationId: observationId,
          traceId: traceId,
+          datasetRunName: "run-only-observation",
        }),
      ]),
    });

-    const runItemTrace = await makeAPICall(
-      "POST",
-      "/api/public/dataset-run-items",
-      {
-        datasetItemId: "dataset-item-id",
-        traceId: traceId,
-        runName: "run-only-trace",
-        metadata: { key: "value" },
-      },
-    );
+    const runItemTrace = await makeAPICall<{
+      datasetRunName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-run-items", {
+      datasetItemId: "dataset-item-id",
+      traceId: traceId,
+      runName: "run-only-trace",
+      metadata: { key: "value" },
+    });
+
+    expect(runItemTrace.status).toBe(200);
+    expect(runItemTrace.body).toMatchObject({
+      datasetRunName: "run-only-trace", // not included in db table
+    });
+
    const dbRunTrace = await prisma.datasetRuns.findFirst({
      where: {
        name: "run-only-trace",
@@ -520,8 +520,6 @@ describe("/api/public/ingestion API Endpoint", () => {

      expect(response.status).toBe(207);

-      console.log("response body", response.body);
-
      const dbGeneration = await prisma.observation.findUnique({
        where: {
          id: generationId,
@@ -851,7 +849,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -901,7 +898,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -1167,8 +1163,6 @@ describe("/api/public/ingestion API Endpoint", () => {
    });
    expect(responseOne.status).toBe(207);

-    console.log(responseOne.body);
-
    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
    const errors = responseOne.body.errors;

@@ -1419,7 +1413,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
      input,
    )} ${JSON.stringify(expected)}`, () => {
      const cleanedEvent = cleanEvent(input);
-      console.log(cleanedEvent);
      expect(cleanedEvent).toStrictEqual(expected);
    });
  });
@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared/src/db";
+import { type ObservationView } from "@langfuse/shared";

 describe("/api/public/observations API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -41,6 +41,19 @@ describe("/api/public/observations API Endpoint", () => {
      },
    });

+    const prompt = await prisma.prompt.create({
+      data: {
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
    await prisma.observation.create({
      data: {
        id: uuidv4(),
@@ -62,6 +75,9 @@ describe("/api/public/observations API Endpoint", () => {
        },
        internalModel: "gpt-3.5-turbo",
        unit: ModelUsageUnit.Tokens,
+        prompt: {
+          connect: { id: prompt.id },
+        },
      },
    });

@@ -95,6 +111,7 @@ describe("/api/public/observations API Endpoint", () => {
    expect(
      fetchedObservations.body.data[0]?.calculatedTotalCost,
    ).toBeGreaterThan(0);
+    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
  });
  it("should fetch all observations, filtered by generations", async () => {
    await pruneDatabase();
@@ -1,5 +1,5 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+import { tracesTableCols } from "@langfuse/shared";

 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
@@ -3,7 +3,12 @@
 import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
-import { type Prompt } from "@langfuse/shared/src/db";
+import { type Prompt } from "@langfuse/shared";
+import {
+  PromptSchema,
+  PromptType,
+  type ValidatedPrompt,
+} from "@/src/features/prompts/server/validation";

 describe("/api/public/prompts API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -43,10 +48,12 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptId);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
+    expect(fetchedObservations.body.tags).toEqual([]);
  });

  it("should fetch active prompt only if no prompt version is given", async () => {
@@ -190,6 +197,7 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptIdTwo);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(2);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -221,6 +229,7 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
@@ -342,8 +351,6 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(response.status).toBe(207);

-    console.log("response body", response.body);
-
    const dbGeneration = await prisma.observation.findUnique({
      where: {
        id: generationId,
@@ -375,11 +382,185 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
    expect(fetchedObservations.body.config).toEqual({});
  });
+
+  it("should create and fetch a chat prompt", async () => {
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(201);
+
+    const { body: fetchedPrompt } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+
+    const validatedPrompt = validatePrompt(fetchedPrompt);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+  });
+
+  it("should fail if chat prompt has string prompt", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if chat prompt has incorrect messages format", async () => {
+    const promptName = "prompt-name";
+    const incorrectChatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", message: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: incorrectChatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+  it("should fail if text prompt has message format", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: [{ role: "system", content: "You are a bot" }],
+      type: "text",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if previous versions have different prompt type", async () => {
+    // Create a chat prompt
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const postResponse1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse1.status).toBe(201);
+
+    // Try creating a text prompt with the same name
+    const postResponse2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "text",
+      isActive: true,
+      version: 2,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse2.status).toBe(400);
+    expect(postResponse2.body).toEqual({
+      error: "ValidationError",
+      message:
+        "Previous versions have different prompt type. Create a new prompt with a different name.",
+    });
+
+    // Check if the prompt is still the chat prompt
+    const getResponse1 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(getResponse1.status).toBe(200);
+
+    const validatedPrompt = validatePrompt(getResponse1.body);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+
+    // Check that the text prompt has not been created
+    const getResponse2 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}&version=2`,
+      undefined,
+    );
+    expect(getResponse2.status).toBe(404);
+    expect(getResponse2.body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
 });

 const isPrompt = (x: unknown): x is Prompt => {
@@ -393,6 +574,18 @@ const isPrompt = (x: unknown): x is Prompt => {
    typeof prompt.isActive === "boolean" &&
    typeof prompt.projectId === "string" &&
    typeof prompt.createdBy === "string" &&
-    typeof prompt.config === "object"
+    typeof prompt.config === "object" &&
+    Object.values(PromptType).includes(prompt.type as PromptType)
  );
 };
+
+const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+  Object.keys(obj).forEach((key) => {
+    obj[key] =
+      key === "createdAt" || key === "updatedAt"
+        ? new Date(obj[key] as string)
+        : obj[key];
+  });
+
+  return PromptSchema.parse(obj);
+};
@@ -5,8 +5,13 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";

 describe("/api/public/scores API Endpoint", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
+  let should_prune_db = true;
+  beforeEach(async () => {
+    if (should_prune_db) await pruneDatabase();
+  });
+  afterEach(async () => {
+    if (should_prune_db) await pruneDatabase();
+  });

  it("should create score for a trace", async () => {
    await pruneDatabase();
@@ -316,4 +321,274 @@ describe("/api/public/scores API Endpoint", () => {
      observationId: generationId,
    });
  });
+
+  describe("should Filter scores", () => {
+    const userId = "user-name";
+    const scoreName = "score-name";
+    const queryUserName = `userId=${userId}&name=${scoreName}`;
+    const traceId = uuidv4();
+    const generationId = uuidv4();
+    const scoreId_1 = uuidv4();
+    const scoreId_2 = uuidv4();
+    const scoreId_3 = uuidv4();
+    interface GetScoresAPIResponse {
+      data: [
+        {
+          id: string;
+          name: string;
+          value: number;
+        },
+      ];
+      meta: object;
+    }
+
+    beforeAll(async () => {
+      should_prune_db = false;
+      await pruneDatabase();
+
+      await makeAPICall("POST", "/api/public/traces", {
+        id: traceId,
+        userId: userId,
+      });
+      await makeAPICall("POST", "/api/public/generations", {
+        id: generationId,
+      });
+      await makeAPICall("POST", "/api/public/scores", {
+        id: scoreId_1,
+        observationId: generationId,
+        name: scoreName,
+        value: 10.5,
+        traceId: traceId,
+        comment: "comment",
+      });
+      await makeAPICall("POST", "/api/public/scores", {
+        id: scoreId_2,
+        observationId: generationId,
+        name: scoreName,
+        value: 50.5,
+        traceId: traceId,
+        comment: "comment",
+      });
+      await makeAPICall("POST", "/api/public/scores", {
+        id: scoreId_3,
+        observationId: generationId,
+        name: scoreName,
+        value: 100.8,
+        traceId: traceId,
+        comment: "comment",
+      });
+    });
+    afterAll(async () => {
+      await pruneDatabase();
+    });
+
+    it("get all scores", async () => {
+      const getAllScore = await makeAPICall<{
+        data: [
+          {
+            traceId: string;
+            observationId: string;
+          },
+        ];
+        meta: object;
+      }>("GET", `/api/public/scores?${queryUserName}`);
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+        });
+      }
+    });
+
+    it("test only operator", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test only value", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&value=0.8`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test operator <", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<&value=50`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
+    it("test operator >", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>&value=100`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_3,
+          name: scoreName,
+          value: 100.8,
+        },
+      ]);
+    });
+    it("test operator <=", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<=&value=50.5`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+      ]);
+    });
+    it("test operator >=", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>=&value=50.5`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+        {
+          id: scoreId_3,
+          name: scoreName,
+          value: 100.8,
+        },
+      ]);
+    });
+    it("test operator !=", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=!=&value=50.5`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+        {
+          id: scoreId_3,
+          name: scoreName,
+          value: 100.8,
+        },
+      ]);
+    });
+    it("test operator =", async () => {
+      const getScore = await makeAPICall<GetScoresAPIResponse>(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator==&value=50.5`,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+      ]);
+    });
+    it("test invalid operator", async () => {
+      const getScore = await makeAPICall(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=op&value=50.5`,
+      );
+      expect(getScore.status).toBe(400);
+      expect(getScore.body).toMatchObject({
+        message: "Invalid request data",
+      });
+    });
+    it("test invalid value", async () => {
+      const getScore = await makeAPICall(
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<&value=myvalue`,
+      );
+      expect(getScore.status).toBe(400);
+      expect(getScore.body).toMatchObject({
+        message: "Invalid request data",
+      });
+    });
+  });
 });
@@ -0,0 +1,33 @@
+import * as React from "react";
+import { Switch } from "@headlessui/react";
+import { cn } from "@/src/utils/tailwind";
+
+export const Slider = (props: {
+  disabled: boolean; // whether the slider is clickable in the UI
+  loading?: boolean;
+  onChecked?: (checked: boolean) => void;
+  isChecked?: boolean; // whether the slider is active
+}) => {
+  console.log("props", props);
+
+  return (
+    <Switch
+      checked={props.isChecked}
+      disabled={props.loading || props.disabled}
+      onChange={props.onChecked}
+      className={cn(
+        props.isChecked ? "bg-black" : "bg-gray-200",
+        "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-black focus:ring-offset-2",
+      )}
+    >
+      <span className="sr-only">Use setting</span>
+      <span
+        aria-hidden="true"
+        className={cn(
+          props.isChecked ? "translate-x-5" : "translate-x-0",
+          "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out",
+        )}
+      />
+    </Switch>
+  );
+};
@@ -40,11 +40,13 @@ export function DatePicker({
  onChange,
  clearable = false,
  className,
+  disabled,
 }: {
  date?: Date | undefined;
  onChange: (date: Date | undefined) => void;
  clearable?: boolean;
  className?: string;
+  disabled?: boolean;
 }) {
  return (
    <div className="flex flex-row gap-2 align-middle">
@@ -52,6 +54,7 @@ export function DatePicker({
        <PopoverTrigger asChild>
          <Button
            variant={"outline"}
+            disabled={disabled}
            className={cn(
              "justify-start text-left font-normal",
              !date && "text-muted-foreground",
@@ -0,0 +1,46 @@
+import { Button } from "@/src/components/ui/button";
+import { AlertCircle } from "lucide-react";
+import { useSession } from "next-auth/react";
+import { useRouter } from "next/router";
+import { captureException } from "@sentry/nextjs";
+import { useEffect } from "react";
+
+export const ErrorPage = ({
+  title = "Error",
+  message,
+}: {
+  title?: string;
+  message: string;
+}) => {
+  const session = useSession();
+  const router = useRouter();
+
+  return (
+    <div className="flex h-screen flex-col items-center justify-center">
+      <AlertCircle className="mb-4 h-12 w-12 text-red-500" />
+      <h1 className="mb-4 text-xl font-bold">{title}</h1>
+      <p className="mb-8 text-center">{message}</p>
+      {session.status === "unauthenticated" ? (
+        <Button onClick={() => void router.push("/")}>Sign In</Button>
+      ) : null}
+    </div>
+  );
+};
+
+export const ErrorPageWithSentry = ({
+  title = "Error",
+  message,
+}: {
+  title?: string;
+  message: string;
+}) => {
+  useEffect(() => {
+    // Capture the error with Sentry
+    if (window !== undefined)
+      captureException(
+        new Error(`ErrorPageWithSentry rendered: ${title}, ${message}`),
+      );
+  }, [title, message]); // Empty dependency array means this effect runs once on mount
+
+  return <ErrorPage title={title} message={message} />;
+};
@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
              <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
                <MessageCircle size={12} />
              </HoverCardTrigger>
-              <HoverCardContent>
+              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                <p>{s.comment}</p>
              </HoverCardContent>
            </HoverCard>
@@ -0,0 +1,33 @@
+import CodeMirror, { EditorView } from "@uiw/react-codemirror";
+import { githubLight } from "@uiw/codemirror-theme-github";
+import { json } from "@codemirror/lang-json";
+
+// todo: add json linting
+
+export function JsonEditor({
+  defaultValue,
+  onChange,
+  editable = true,
+  lineWrapping = true,
+}: {
+  defaultValue: string;
+  onChange?: (value: string) => void;
+  editable?: boolean;
+  lineWrapping?: boolean;
+}) {
+  return (
+    <CodeMirror
+      value={defaultValue}
+      theme={githubLight}
+      basicSetup={{
+        foldGutter: true,
+      }}
+      lang={"json"}
+      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      defaultValue={defaultValue}
+      onChange={onChange}
+      className="overflow-hidden rounded-md border"
+      editable={editable}
+    />
+  );
+}
@@ -5,6 +5,7 @@ import { useSession } from "next-auth/react";
 import DocPopup from "@/src/components/layouts/doc-popup";
 import { type Status, StatusBadge } from "./status-badge";
 import { cn } from "@/src/utils/tailwind";
+import { Badge } from "@/src/components/ui/badge";

 export default function Header({
  level = "h2",
@@ -14,6 +15,7 @@ export default function Header({
  breadcrumb?: { name: string; href?: string }[];
  status?: Status;
  help?: { description: string; href?: string };
+  featureBetaURL?: string;
  actionButtons?: React.ReactNode;
  level?: "h2" | "h3";
 }) {
@@ -40,7 +42,7 @@ export default function Header({
    [...props.breadcrumb.map((i) => i.href).filter(Boolean)].pop();

  return (
-    <div className={cn(level === "h2" ? "mb-4" : "mb-1")}>
+    <div className={cn(level === "h2" ? "mb-4" : "mb-2")}>
      <div>
        {backHref ? (
          <nav className="sm:hidden" aria-label="Back">
@@ -89,7 +91,7 @@ export default function Header({
      </div>
      <div className="mt-2 flex flex-wrap items-center justify-between gap-2">
        <div className="flex items-center gap-3 md:gap-5">
-          <div className="flex min-w-0 flex-row">
+          <div className="flex min-w-0 flex-row justify-center align-middle">
            {level === "h2" ? (
              <h2 className="text-2xl font-bold leading-7 text-gray-900 sm:truncate sm:text-3xl sm:tracking-tight">
                {props.title}
@@ -106,10 +108,25 @@ export default function Header({
                size="sm"
              />
            ) : null}
+            {props.featureBetaURL ? (
+              <Link
+                href={props.featureBetaURL}
+                rel="noopener noreferrer"
+                target="_blank"
+                className="flex items-center"
+              >
+                <Badge
+                  title="Feature is currently in beta. Click to learn more."
+                  className="ml-2"
+                >
+                  Beta
+                </Badge>
+              </Link>
+            ) : null}
          </div>
          {props.status && <StatusBadge type={props.status} />}
        </div>
-        <div className="flex items-center gap-3">
+        <div className="flex flex-wrap items-center gap-3">
          {props.actionButtons ?? null}
        </div>
      </div>
@@ -43,7 +43,11 @@ const userNavigation = [
 ];

 const pathsWithoutNavigation: string[] = ["/onboarding"];
-const unauthenticatedPaths: string[] = ["/auth/sign-in", "/auth/sign-up"];
+const unauthenticatedPaths: string[] = [
+  "/auth/sign-in",
+  "/auth/sign-up",
+  "/auth/error",
+];
 const publishablePaths: string[] = [
  "/project/[projectId]/sessions/[sessionId]",
  "/project/[projectId]/traces/[traceId]",
@@ -75,6 +79,14 @@ export default function Layout(props: PropsWithChildren) {
    )
      return null;

+    // cloud only
+    if (
+      route.cloudOnly !== undefined &&
+      // the feature should be available in local development
+      route.cloudOnly !== (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined)
+    )
+      return null;
+
    // RBAC
    if (
      route.rbacScope !== undefined &&
@@ -309,8 +321,8 @@ export default function Layout(props: PropsWithChildren) {
              </ul>
            </nav>

-            <Menu as="div" className="relative left-1">
-              <Menu.Button className="flex w-full items-center gap-x-4 p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+            <Menu as="div" className="relative">
+              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
                <span className="sr-only">Open user menu</span>
                <Avatar className="h-8 w-8">
                  <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -342,7 +354,10 @@ export default function Layout(props: PropsWithChildren) {
                leaveFrom="transform opacity-100 scale-100"
                leaveTo="transform opacity-0 scale-95"
              >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                    {session.data?.user?.email}
+                  </span>
                  {userNavigation.map((item) => (
                    <Menu.Item key={item.name}>
                      {({ active }) => (
@@ -403,7 +418,10 @@ export default function Layout(props: PropsWithChildren) {
              leaveFrom="transform opacity-100 scale-100"
              leaveTo="transform opacity-0 scale-95"
            >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  {session.data?.user?.email}
+                </span>
                {userNavigation.map((item) => (
                  <Menu.Item key={item.name}>
                    {({ active }) => (
@@ -496,6 +514,7 @@ const MainNavigation: React.FC<{
                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
                )}
                onClick={onNavitemClick}
+                target={item.newTab ? "_blank" : undefined}
              >
                {item.icon && (
                  <item.icon
@@ -574,6 +593,7 @@ const MainNavigation: React.FC<{
                                : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
                            )}
+                            target={subItem.newTab ? "_blank" : undefined}
                          >
                            {subItem.name}
                            {subItem.label && (
@@ -9,7 +9,9 @@ import {
  Settings,
  UsersIcon,
  PenSquareIcon,
+  LibraryBig,
  TerminalIcon,
+  Lightbulb,
 } from "lucide-react";

 export type Route = {
@@ -21,6 +23,8 @@ export type Route = {
  pathname?: string; // link, ignored if children
  children?: Array<Route>; // folder
  bottom?: boolean; // bottom of the sidebar, only for first level routes
+  newTab?: boolean; // open in new tab
+  cloudOnly?: boolean; // only available in cloud
 };

 export const ROUTES: Route[] = [
@@ -55,6 +59,26 @@ export const ROUTES: Route[] = [
      },
    ],
  },
+  {
+    name: "Evaluation",
+    icon: Lightbulb,
+    cloudOnly: true,
+    featureFlag: "evals",
+    children: [
+      {
+        name: "Templates",
+        pathname: `/project/[projectId]/evals/templates`,
+        cloudOnly: true,
+        featureFlag: "evals",
+      },
+      {
+        name: "Configs",
+        pathname: `/project/[projectId]/evals/configs`,
+        cloudOnly: true,
+        featureFlag: "evals",
+      },
+    ],
+  },
  {
    name: "Users",
    pathname: `/project/[projectId]/users`,
@@ -71,6 +95,8 @@ export const ROUTES: Route[] = [
    pathname: "/project/[projectId]/playground",
    icon: TerminalIcon,
    featureFlag: "playground",
+    cloudOnly: true,
+    label: "Beta",
  },
  {
    name: "Datasets",
@@ -83,6 +109,13 @@ export const ROUTES: Route[] = [
    icon: Settings,
    bottom: true,
  },
+  {
+    name: "Docs",
+    pathname: "https://langfuse.com/docs",
+    icon: LibraryBig,
+    bottom: true,
+    newTab: true,
+  },
  {
    name: "Support",
    pathname: "/project/[projectId]/support",
@@ -1,7 +1,7 @@
 import { cn } from "@/src/utils/tailwind";

 const statusCategories = {
-  active: ["production", "live"],
+  active: ["production", "live", "active"],
  inactive: ["disabled", "inactive"],
 };

@@ -1,22 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import { AlertCircle } from "lucide-react";
-import { useSession } from "next-auth/react";
-import { useRouter } from "next/router";
-
-export const NoAccessError = () => {
-  const session = useSession();
-  const router = useRouter();
-
-  return (
-    <div className="flex h-screen flex-col items-center justify-center">
-      <AlertCircle className="mb-4 h-12 w-12 text-red-500" />
-      <h1 className="mb-4 text-xl font-bold">Error</h1>
-      <p className="mb-8 text-center">
-        You do not have access to this resource.
-      </p>
-      {session.status === "unauthenticated" ? (
-        <Button onClick={() => void router.push("/")}>Sign In</Button>
-      ) : null}
-    </div>
-  );
-};
@@ -7,7 +7,6 @@ import {
  SelectValue,
 } from "@/src/components/ui/select";
 import { cn } from "@/src/utils/tailwind";
-import { Code } from "lucide-react";
 import { useRouter } from "next/router";

 interface Project {
@@ -1,9 +1,10 @@
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import Header from "@/src/components/layouts/header";
-import { NoAccessError } from "@/src/components/no-access";
+import { ErrorPage } from "@/src/components/error-page";
 import { PublishSessionSwitch } from "@/src/components/publish-object-switch";
 import { StarSessionToggle } from "@/src/components/star-toggle";
 import { IOPreview } from "@/src/components/trace/IOPreview";
+import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 import { Badge } from "@/src/components/ui/badge";
 import { Card } from "@/src/components/ui/card";
 import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
@@ -41,7 +42,8 @@ export const SessionPage: React.FC<{
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [session.isSuccess, session.data]);

-  if (session.error?.data?.code === "UNAUTHORIZED") return <NoAccessError />;
+  if (session.error?.data?.code === "UNAUTHORIZED")
+    return <ErrorPage message="You do not have access to this session." />;

  return (
    <div className="flex flex-col overflow-hidden xl:container">
@@ -101,20 +103,7 @@ export const SessionPage: React.FC<{
            className="border-border-gray-150 group grid gap-3 p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
            key={trace.id}
          >
-            <div className="col-span-2 flex flex-col gap-2 p-0">
-              {trace.input || trace.output ? (
-                <IOPreview
-                  key={trace.id}
-                  input={trace.input}
-                  output={trace.output}
-                  hideIfNull
-                />
-              ) : (
-                <div className="p-2 text-xs text-gray-500">
-                  This trace has no input or output.
-                </div>
-              )}
-            </div>
+            <SessionIO traceId={trace.id} />
            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
              <Link
                href={`/project/${projectId}/traces/${trace.id}`}
@@ -142,3 +131,39 @@ export const SessionPage: React.FC<{
    </div>
  );
 };
+
+const SessionIO = ({ traceId }: { traceId: string }) => {
+  const trace = api.traces.byId.useQuery(
+    { traceId: traceId },
+    {
+      enabled: typeof traceId === "string",
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+  return (
+    <div className="col-span-2 flex flex-col gap-2 p-0">
+      {!trace.data ? (
+        <JsonSkeleton
+          className="h-full w-full overflow-hidden px-2 py-1"
+          numRows={4}
+        />
+      ) : trace.data.input || trace.data.output ? (
+        <IOPreview
+          key={traceId}
+          input={trace.data.input}
+          output={trace.data.output}
+          hideIfNull
+        />
+      ) : (
+        <div className="p-2 text-xs text-gray-500">
+          This trace has no input or output.
+        </div>
+      )}
+    </div>
+  );
+};
@@ -12,7 +12,7 @@ import {
  DropdownMenuCheckboxItem,
 } from "@/src/components/ui/dropdown-menu";
 import { type VisibilityState } from "@tanstack/react-table";
-import { ChevronDownIcon } from "lucide-react";
+import { ChevronDown, Columns } from "lucide-react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";

 interface DataTableColumnVisibilityFilterProps<TData, TValue> {
@@ -70,14 +70,16 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
        className="select-none"
        asChild
      >
-        <Button variant="outline" className="ml-auto">
-          Columns {count <= total ? `${count}/${total}` : ""}
-          <ChevronDownIcon className="ml-2 h-4 w-4" />
+        <Button variant="outline" title="Show/hide columns">
+          <Columns className="mr-2 h-4 w-4" />
+          <span className="text-xs text-gray-500">{`(${count}/${total})`}</span>
+          <ChevronDown className="ml-2 h-4 w-4" />
        </Button>
      </DropdownMenuTrigger>
      <DropdownMenuContent
        align="end"
        onPointerDownOutside={() => setIsOpen(false)}
+        className="max-h-96 overflow-y-auto"
      >
        {columns.map(
          (column, index) =>
@@ -46,11 +46,7 @@ export function TraceTableMultiSelectAction({
    <>
      <DropdownMenu>
        <DropdownMenuTrigger asChild>
-          <Button
-            variant="outline"
-            className="bg-white p-2 font-medium text-black"
-            disabled={selectedTraceIds.length < 1}
-          >
+          <Button disabled={selectedTraceIds.length < 1}>
            Actions ({selectedTraceIds.length} selected)
            <ChevronDown className="h-5 w-5" />
          </Button>
@@ -0,0 +1,59 @@
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import {
+  MdDensityLarge,
+  MdDensityMedium,
+  MdDensitySmall,
+} from "react-icons/md";
+
+const heightOptions = [
+  { id: "s", label: "Small", value: "h-6", icon: <MdDensitySmall /> },
+  { id: "m", label: "Medium", value: "h-24", icon: <MdDensityMedium /> },
+  { id: "l", label: "Large", value: "h-64", icon: <MdDensityLarge /> },
+] as const;
+
+export type RowHeight = (typeof heightOptions)[number]["id"];
+
+export const getRowHeightTailwindClass = (rowHeight: RowHeight | undefined) =>
+  heightOptions.find((h) => h.id === rowHeight)?.value;
+
+export function useRowHeightLocalStorage(
+  tableName: string,
+  defaultValue: RowHeight,
+) {
+  const [rowHeight, setRowHeight, clearRowHeight] = useLocalStorage<RowHeight>(
+    `${tableName}Height`,
+    defaultValue,
+  );
+
+  return [rowHeight, setRowHeight, clearRowHeight] as const;
+}
+
+export const DataTableRowHeightSwitch = ({
+  rowHeight,
+  setRowHeight,
+}: {
+  rowHeight: RowHeight;
+  setRowHeight: (e: RowHeight) => void;
+}) => (
+  <Tabs
+    //defaultValue={height}
+    value={rowHeight}
+    onValueChange={(e) => setRowHeight(e as any)}
+    key="height"
+  >
+    <TabsList className="gap-1 border bg-transparent px-2">
+      {heightOptions.map(({ id, label, icon }) => (
+        <TabsTrigger
+          key={id}
+          value={id}
+          className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
+        >
+          <span role="img" aria-label={`${label} size`}>
+            {icon}
+          </span>
+        </TabsTrigger>
+      ))}
+    </TabsList>
+  </Tabs>
+);
@@ -2,11 +2,16 @@ import { Button } from "@/src/components/ui/button";
 import React, { type Dispatch, type SetStateAction, useState } from "react";
 import { Input } from "@/src/components/ui/input";
 import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-table-column-visibility-filter";
-import { type FilterState } from "@/src/features/filters/types";
-import { FilterBuilder } from "@/src/features/filters/components/filter-builder";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import { type FilterState } from "@langfuse/shared";
+import { PopoverFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import { type ColumnDefinition } from "@langfuse/shared";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import {
+  DataTableRowHeightSwitch,
+  type RowHeight,
+} from "@/src/components/table/data-table-row-height-switch";
+import { Search } from "lucide-react";

 interface SearchConfig {
  placeholder: string;
@@ -16,13 +21,15 @@ interface SearchConfig {

 interface DataTableToolbarProps<TData, TValue> {
  columns: LangfuseColumnDef<TData, TValue>[];
-  filterColumnDefinition: ColumnDefinition[];
+  filterColumnDefinition?: ColumnDefinition[];
  searchConfig?: SearchConfig;
  actionButtons?: React.ReactNode;
-  filterState: FilterState;
-  setFilterState: Dispatch<SetStateAction<FilterState>>;
+  filterState?: FilterState;
+  setFilterState?: Dispatch<SetStateAction<FilterState>>;
  columnVisibility?: VisibilityState;
  setColumnVisibility?: Dispatch<SetStateAction<VisibilityState>>;
+  rowHeight?: RowHeight;
+  setRowHeight?: Dispatch<SetStateAction<RowHeight>>;
 }

 export function DataTableToolbar<TData, TValue>({
@@ -34,42 +41,46 @@ export function DataTableToolbar<TData, TValue>({
  setFilterState,
  columnVisibility,
  setColumnVisibility,
+  rowHeight,
+  setRowHeight,
 }: DataTableToolbarProps<TData, TValue>) {
  const [searchString, setSearchString] = useState(
    searchConfig?.currentQuery ?? "",
  );

  return (
-    <div className="my-2 flex max-w-full items-center justify-between overflow-x-auto">
-      <div className="flex flex-1 items-center space-x-2">
-        {searchConfig && (
-          <div className="flex max-w-md items-center space-x-2">
-            <Input
-              autoFocus
-              placeholder={searchConfig.placeholder}
-              value={searchString}
-              onChange={(event) => setSearchString(event.currentTarget.value)}
-              onKeyDown={(event) => {
-                if (event.key === "Enter") {
-                  searchConfig.updateQuery(searchString);
-                }
-              }}
-              className="h-10 w-[200px] lg:w-[350px]"
-            />
-            <Button
-              variant="outline"
-              onClick={() => searchConfig.updateQuery(searchString)}
-            >
-              Search
-            </Button>
-          </div>
-        )}
-        <FilterBuilder
+    <div className="my-2 flex flex-1 flex-wrap items-center gap-2">
+      {searchConfig && (
+        <div className="flex max-w-md items-center">
+          <Input
+            autoFocus
+            placeholder={searchConfig.placeholder}
+            value={searchString}
+            onChange={(event) => setSearchString(event.currentTarget.value)}
+            onKeyDown={(event) => {
+              if (event.key === "Enter") {
+                searchConfig.updateQuery(searchString);
+              }
+            }}
+            className="h-10 w-[200px] rounded-r-none lg:w-[250] 2xl:w-[350px]"
+          />
+          <Button
+            variant="outline"
+            onClick={() => searchConfig.updateQuery(searchString)}
+            className="rounded-l-none border-l-0 p-3"
+          >
+            <Search className="h-4 w-4" />
+          </Button>
+        </div>
+      )}
+      {!!filterColumnDefinition && !!filterState && !!setFilterState && (
+        <PopoverFilterBuilder
          columns={filterColumnDefinition}
          filterState={filterState}
          onChange={setFilterState}
        />
-        <div className="flex-1" />
+      )}
+      <div className="flex flex-row flex-wrap gap-2 lg:ml-auto">
        {!!columnVisibility && !!setColumnVisibility && (
          <DataTableColumnVisibilityFilter
            columns={columns}
@@ -77,6 +88,12 @@ export function DataTableToolbar<TData, TValue>({
            setColumnVisibility={setColumnVisibility}
          />
        )}
+        {!!rowHeight && !!setRowHeight && (
+          <DataTableRowHeightSwitch
+            rowHeight={rowHeight}
+            setRowHeight={setRowHeight}
+          />
+        )}
        {actionButtons}
      </div>
    </div>
@@ -2,6 +2,10 @@

 import DocPopup from "@/src/components/layouts/doc-popup";
 import { DataTablePagination } from "@/src/components/table/data-table-pagination";
+import {
+  type RowHeight,
+  getRowHeightTailwindClass,
+} from "@/src/components/table/data-table-row-height-switch";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { type ModelTableRow } from "@/src/components/table/use-cases/models";
 import {
@@ -43,6 +47,7 @@ interface DataTableProps<TData, TValue> {
  orderBy?: OrderByState;
  setOrderBy?: (s: OrderByState) => void;
  help?: { description: string; href: string };
+  rowHeight?: RowHeight;
 }

 export interface AsyncTableData<T> {
@@ -63,8 +68,10 @@ export function DataTable<TData extends object, TValue>({
  help,
  orderBy,
  setOrderBy,
+  rowHeight,
 }: DataTableProps<TData, TValue>) {
  const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
+  const rowheighttw = getRowHeightTailwindClass(rowHeight);

  const table = useReactTable({
    data: data.data ?? [],
@@ -185,10 +192,12 @@ export function DataTable<TData extends object, TValue>({
                        key={cell.id}
                        className="overflow-hidden whitespace-nowrap px-2 py-1 text-xs first:pl-2"
                      >
-                        {flexRender(
-                          cell.column.columnDef.cell,
-                          cell.getContext(),
-                        )}
+                        <div className={cn("flex items-center", rowheighttw)}>
+                          {flexRender(
+                            cell.column.columnDef.cell,
+                            cell.getContext(),
+                          )}
+                        </div>
                      </TableCell>
                    ))}
                  </TableRow>
@@ -4,22 +4,26 @@ export type TableLinkProps = {
  path: string;
  value: string;
  truncateAt?: number;
+  icon?: React.ReactNode;
 };

 export default function TableLink({
  path,
  value,
  truncateAt = 7,
+  icon,
 }: TableLinkProps) {
+  const truncatedValue =
+    value.length - truncateAt > 3
+      ? `...${value.substring(value.length - truncateAt)}`
+      : value;
  return (
    <Link
      className="inline-block rounded bg-indigo-50 px-2 py-1 text-xs font-semibold text-blue-600 shadow-sm hover:bg-indigo-100"
      href={path}
      title={value}
    >
-      {value.length - truncateAt > 3
-        ? `...${value.substring(value.length - truncateAt)}`
-        : value}
+      {icon ? icon : truncatedValue}
    </Link>
  );
 }
@@ -1,54 +0,0 @@
-import { JSONView } from "@/src/components/ui/code";
-import { cn } from "@/src/utils/tailwind";
-import { randomIntFromInterval } from "@/src/utils/numbers";
-import { Skeleton } from "@/src/components/ui/skeleton";
-import React from "react";
-
-export const IOCell = ({
-  isLoading,
-  data,
-}: {
-  isLoading: boolean;
-  data: unknown;
-}) => {
-  return (
-    <>
-      {isLoading  ? (
-        <JsonSkeleton className="h-[250px] w-[500px] px-3 py-1" />
-      ) : (
-        <JSONView json={data} className="h-[250px] w-[500px] overflow-y-auto" />
-      )}
-    </>
-  );
-};
-
-export const JsonSkeleton = ({
-  className,
-  numRows = 10,
-}: {
-  numRows?: number;
-  className?: string;
-}) => {
-  const sizingOptions = [
-    "h-5 w-full",
-    "h-5 w-[400px]",
-    "h-5 w-[450px]",
-    "h-5 w-[475px]",
-  ];
-
-  const generateRandomSize = () =>
-    sizingOptions[randomIntFromInterval(0, sizingOptions.length - 1)];
-
-  return (
-    <div className={cn("w-[500px] rounded-md border", className)}>
-      <div className="flex flex-col gap-1">
-        {[...Array<number>(numRows)].map((_) => (
-          <>
-            <Skeleton className={generateRandomSize()} />
-          </>
-        ))}
-        <br />
-      </div>
-    </div>
-  );
-};
@@ -22,7 +22,6 @@ import {
  withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
 import {
  formatIntervalSeconds,
  intervalInSeconds,
@@ -30,19 +29,20 @@ import {
 } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type ObservationLevel } from "@langfuse/shared/src/db";
+import { type Prisma, type ObservationLevel } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
 import { usdFormatter } from "@/src/utils/numbers";
 import {
  exportOptions,
  type ExportFileFormats,
-} from "@/src/server/api/interfaces/exportTypes";
+  observationsTableColsWithOptions,
+} from "@langfuse/shared";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
-import { IOCell } from "./IOCell";
-import { setSmallPaginationIfColumnsVisible } from "../../../features/column-visibility/hooks/setSmallPaginationIfColumnsVisible";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";

 export type GenerationsTableRow = {
  id: string;
@@ -62,7 +62,7 @@ export type GenerationsTableRow = {
  outputCost?: Decimal;
  totalCost?: Decimal;
  traceName?: string;
-  metadata?: string;
+  metadata?: Prisma.JsonValue;
  scores?: ScoreSimplified[];
  usage: {
    promptTokens: number;
@@ -91,6 +91,11 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
    pageSize: withDefault(NumberParam, 50),
  });

+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "generations",
+    "s",
+  );
+
  const [filterState, setFilterState] = useQueryFilterState(
    [
      {
@@ -510,8 +515,10 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
      accessorKey: "metadata",
      header: "Metadata",
      cell: ({ row }) => {
-        const values: string | undefined = row.getValue("metadata");
-        return <div className="flex flex-wrap gap-x-3 gap-y-1">{values}</div>;
+        const values = row.getValue(
+          "metadata",
+        ) as GenerationsTableRow["metadata"];
+        return !!values ? <IOTableCell data={values} /> : null;
      },
      enableHiding: true,
      defaultHidden: true,
@@ -552,13 +559,6 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
      columns,
    );

-  const smallTableRequired = setSmallPaginationIfColumnsVisible(
-    columnVisibility,
-    ["input", "output"],
-    paginationState,
-    setPaginationState,
-  );
-
  const rows: GenerationsTableRow[] = generations.isSuccess
    ? generations.data.generations.map((generation) => {
        return {
@@ -577,9 +577,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
          model: generation.model ?? "",
          scores: generation.scores,
          level: generation.level,
-          metadata: generation.metadata
-            ? JSON.stringify(generation.metadata)
-            : undefined,
+          metadata: generation.metadata,
          statusMessage: generation.statusMessage ?? undefined,
          usage: {
            promptTokens: generation.promptTokens,
@@ -609,14 +607,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
        }}
        columnVisibility={columnVisibility}
        setColumnVisibility={setColumnVisibilityState}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
        actionButtons={
          <DropdownMenu>
            <DropdownMenuTrigger asChild>
-              <Button
-                variant="outline"
-                className="ml-auto whitespace-nowrap"
-                size="sm"
-              >
+              <Button variant="outline" className="ml-auto whitespace-nowrap">
                {filterState.length > 0 || searchQuery
                  ? "Export selection"
                  : "Export all"}{" "}
@@ -662,13 +658,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
          pageCount: Math.ceil(totalCount / paginationState.pageSize),
          onChange: setPaginationState,
          state: paginationState,
-          // enforce a minimum page size of 10 if input or output columns are visible
-          options: smallTableRequired ? [10] : undefined,
        }}
        setOrderBy={setOrderByState}
        orderBy={orderByState}
        columnVisibility={columnVisibility}
        onColumnVisibilityChange={setColumnVisibilityState}
+        rowHeight={rowHeight}
      />
    </div>
  );
@@ -695,14 +690,16 @@ const GenerationsIOCell = ({
          skipBatch: true,
        },
      },
+      refetchOnMount: false, // prevents refetching loops
    },
  );
  return (
-    <IOCell
+    <IOTableCell
      isLoading={observation.isLoading}
      data={
        io === "output" ? observation.data?.output : observation.data?.input
      }
+      className={cn(io === "output" && "bg-green-50")}
    />
  );
 };
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`export { isEeAvailable } from "./ee-license-check";`
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>