chore: release v2.21.2

fix(cloud): signup blocked due to turnstile (#1667 )
feat(security): add AUTH_DISABLE_SIGNUP env to disable all new signups (#1666 )
2024-04-11 00:27:40 +02:00 · 2024-04-11 00:26:06 +02:00 · 2024-04-11 00:16:20 +02:00 · 2024-04-10 16:49:55 +00:00 · 2024-04-10 16:30:51 +00:00 · 2024-04-10 18:25:21 +02:00
213 changed files with 7601 additions and 2324 deletions
@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""

 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
@@ -16,4 +16,11 @@ SALT="salt"

 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
@@ -37,6 +37,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
+# AUTH_DISABLE_SIGNUP=true

 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -76,11 +77,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # DB_EXPORT_PAGE_SIZE=1000


-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
@@ -105,6 +101,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=

+# Prompt playground
+# OPENAI_API_KEY=""
+# ANTHROPIC_API_KEY=""
+
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=

@@ -1 +1,2 @@
-* @langfuse/founders
+# Currently inactive
+# * @langfuse/maintainers
@@ -7,6 +7,7 @@ version: 2
 updates:
  - package-ecosystem: npm
    directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
    schedule:
      interval: "daily"
    versioning-strategy: "increase"
@@ -54,7 +54,7 @@ jobs:
          file: ./worker/Dockerfile
          push: false

-  tests:
+  tests-web:
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -102,7 +102,56 @@ jobs:
        run: (pnpm run start&)

      - name: run tests
-        run: pnpm run test
+        run: pnpm --filter=web run test
+
+  tests-worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 8
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+
+      - name: Build
+        run: pnpm --filter=worker... run build
+
+      - name: run tests
+        run: pnpm --filter=worker run test

  e2e-tests:
    runs-on: ubuntu-latest
@@ -148,7 +197,7 @@ jobs:
  all-ci-passed:
    # This allows us to have a branch protection rule for tests and deploys with matrix
    runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests, test-docker-build]
+    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
    if: always()
    steps:
      - name: Successful deploy
@@ -193,7 +242,7 @@ jobs:
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
        uses: docker/metadata-action@v4
        with:
          images: |
@@ -206,11 +255,34 @@ jobs:
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}

-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./web/Dockerfile
          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ github.repository }}-worker
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}
@@ -192,7 +192,7 @@ Requirements
  To run migrations, you can execute the following command.

  ```bash
-  pnpm --filter=shared run db:migrate
+  pnpm run db:migrate -- --name <name of the migration>
  ```

 > [!NOTE]
@@ -16,11 +16,9 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  worker:
@@ -30,7 +28,6 @@ services:
    depends_on:
      - db
      - redis
-      - server
    ports:
      - "3030:3030"
    environment:
@@ -39,9 +36,10 @@ services:
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
      - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
      - REDIS_PORT=${REDIS_PORT:-6379}
      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always

  redis:
@@ -13,7 +13,6 @@ services:
      - SALT=mysalt
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

  db:
@@ -154,12 +154,14 @@ types:
      sourceTraceId: optional<string>
      sourceObservationId: optional<string>
      datasetId: string
+      datasetName: string
      createdAt: datetime
      updatedAt: datetime
  DatasetRunItem:
    properties:
      id: string
      datasetRunId: string
+      datasetRunName: string
      datasetItemId: string
      traceId: string
      observationId: optional<string>
@@ -25,6 +25,20 @@ service:

 types:
  CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
    properties:
      name: string
      isActive:
@@ -34,8 +48,29 @@ types:
      config: optional<unknown>

  Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
    properties:
      name: string
      version: integer
-      prompt: string
      config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>
@@ -628,7 +628,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -1,8 +1,9 @@
 {
  "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.21.2",
  "author": "engineering@langfuse.com",
  "license": "MIT",
+  "private": true,
  "engines": {
    "node": ">=20"
  },
@@ -21,11 +22,51 @@
    "dev": "turbo run dev",
    "lint": "turbo run lint",
    "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate"
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
  },
  "devDependencies": {
+    "@release-it/bumper": "^6.0.1",
    "dotenv-cli": "^7.4.1",
    "prettier": "^3.2.5",
+    "release-it": "^17.1.1",
    "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
  }
 }
@@ -5,12 +5,23 @@
  "private": true,
  "main": "./dist/src/index.js",
  "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
  "engines": {
    "node": ">=20"
  },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    }
+  },
  "scripts": {
    "build": "tsc",
    "dev": "tsc --watch",
@@ -28,24 +39,24 @@
    "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
  },
  "dependencies": {
-    "@langchain/anthropic": "^0.1.10",
-    "@langchain/core": "^0.1.54",
+    "@langchain/anthropic": "^0.1.12",
+    "@langchain/core": "^0.1.55",
    "@langchain/openai": "^0.0.26",
    "@prisma/client": "^5.12.1",
    "@types/bcryptjs": "^2.4.6",
    "bcryptjs": "^2.4.3",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.31",
+    "langchain": "^0.1.32",
    "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
  },
  "devDependencies": {
    "@repo/eslint-config": "*",
    "@repo/typescript-config": "*",
    "@types/lodash": "^4.14.202",
    "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.4",
+    "@types/pg": "^8.11.5",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/parser": "^7.2.0",
    "eslint": "^8.57.0",
@@ -57,9 +68,9 @@
    "prettier": "^3.2.5",
    "prisma": "^5.12.1",
    "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
    "ts-node": "^10.9.2",
-    "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
+    "tsc-watch": "^6.2.0",
    "typescript": "^5.4.4",
    "vitest": "^1.3.1"
  }
@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
    API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,17 @@ export const DatasetStatus = {
    ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
    id: string;
    user_id: string;
@@ -129,6 +141,19 @@ export type DatasetRuns = {
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
    id: string;
    created_at: Generated<Timestamp>;
@@ -139,6 +164,33 @@ export type Events = {
    url: string | null;
    method: string | null;
 };
+export type JobConfiguration = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
+};
 export type Membership = {
    project_id: string;
    user_id: string;
@@ -253,11 +305,13 @@ export type Prompt = {
    updated_at: Generated<Timestamp>;
    project_id: string;
    created_by: string;
-    prompt: string;
+    prompt: unknown;
    name: string;
    version: number;
+    type: Generated<string>;
    is_active: boolean;
    config: Generated<unknown>;
+    tags: Generated<string[]>;
 };
 export type Score = {
    id: string;
@@ -344,7 +398,10 @@ export type DB = {
    dataset_run_items: DatasetRunItems;
    dataset_runs: DatasetRuns;
    datasets: Dataset;
+    eval_templates: EvalTemplate;
    events: Events;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
    membership_invitations: MembershipInvitation;
    memberships: Membership;
    models: Model;
@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;
@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_tags_idx" ON "prompts" USING GIN ("tags" array_ops);
@@ -92,22 +92,25 @@ model VerificationToken {
 }

 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id               String                 @id @default(cuid())
+    createdAt        DateTime               @default(now()) @map("created_at")
+    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
+    name             String
+    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members          Membership[]
+    traces           Trace[]
+    observations     Observation[]
+    apiKeys          ApiKey[]
+    dataset          Dataset[]
+    RawEvents        Events[]
+    invitations      MembershipInvitation[]
+    sessions         TraceSession[]
+    Prompt           Prompt[]
+    Model            Model[]
+    AuditLog         AuditLog[]
+    EvalTemplate     EvalTemplate[]
+    JobConfiguration JobConfiguration[]
+    JobExecution     JobExecution[]

    @@map("projects")
 }
@@ -209,6 +212,7 @@ model Trace {
    scores          Score[]
    DatasetRunItems DatasetRunItems[]
    DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]

    @@index([projectId])
    @@index([sessionId])
@@ -362,16 +366,17 @@ enum ObservationLevel {
 }

 model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
    name          String
    value         Float
    source        ScoreSource
    comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]

    @@unique([id, traceId]) // used for upsert
    @@index(timestamp)
@@ -385,6 +390,7 @@ model Score {
 enum ScoreSource {
    API
    REVIEW
+    EVAL
 }

 enum PricingUnit {
@@ -521,17 +527,20 @@ model Prompt {

    createdBy String @map("created_by")

-    prompt      String
+    prompt      Json
    name        String
    version     Int
+    type        String        @default("text")
    isActive    Boolean       @map("is_active")
    config      Json          @default("{}")
+    tags        String[]      @default([])
    Observation Observation[]

    @@unique([projectId, name, version])
    @@index([projectId, name, version])
    @@index([projectId, id])
    @@index([projectId])
+    @@index([tags(ops: ArrayOps)], type: Gin)
    @@map("prompts")
 }

@@ -579,3 +588,89 @@ model AuditLog {
    @@index([createdAt])
    @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType       @map("job_type")
+    evalTemplateId String?       @map("eval_template_id")
+    evalTemplate   EvalTemplate? @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}
@@ -4,13 +4,14 @@ import {
  type Prisma,
  ObservationType,
  ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";

 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";

 const LOAD_TRACE_VOLUME = 10_000;

@@ -713,6 +714,16 @@ async function generatePrompts(project: Project) {
      version: 1,
      isActive: true,
    },
+    {
+      id: `prompt-${v4()}`,
+      projectId: project.id,
+      createdBy: "user-1",
+      prompt: "Prompt 4 content",
+      name: "Prompt 4",
+      version: 1,
+      isActive: true,
+      tags: ["tag1", "tag2"],
+    },
  ];

  for (const prompt of prompts) {
@@ -732,6 +743,7 @@ async function generatePrompts(project: Project) {
        name: prompt.name,
        version: prompt.version,
        isActive: prompt.isActive,
+        tags: prompt.tags,
      },
      update: {
        id: prompt.id,
@@ -3,6 +3,14 @@

 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";

 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
@@ -22,6 +30,23 @@ declare global {
 }

 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+
+export const kyselyPrisma = prisma.$extends(
+  kyselyExtension({
+    kysely: (driver) =>
+      new Kysely<DB>({
+        dialect: {
+          // This is where the magic happens!
+          createDriver: () => driver,
+          // Don't forget to customize these to match your database!
+          createAdapter: () => new PostgresAdapter(),
+          createIntrospector: (db) => new PostgresIntrospector(db),
+          createQueryCompiler: () => new PostgresQueryCompiler(),
+        },
+      }),
+  })
+);
+
 export * from "@prisma/client";

 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;
@@ -0,0 +1,31 @@
+import z from "zod";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(),
+    objectName: z.string().nullish(), // can be null as this is only required for langfuseObjects other than trace
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});
@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
  type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";

 const operatorReplacements = {
  "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
  if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
  const internalFilters = filters.map((filter) => {
    // Get column definition to map column to internal name, e.g. "t.id"
    const col = tableColumns.find(
      (c) =>
        // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
    );
    if (!col) {
      console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
        ? Prisma.raw(
            arrayOperatorReplacements[
              filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
          )
        : filter.operator in operatorReplacements
          ? Prisma.raw(
              operatorReplacements[
                filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
            )
          : Prisma.raw(filter.operator); //checked by zod

@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
        break;
      case "stringOptions":
        valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
        )})`;
        break;
      case "arrayOptions":
        valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
          filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
        )}] `;
        break;

@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
      filter.type === "string" || filter.type === "stringObject"
        ? [
            ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw("'%' || ")
              : Prisma.empty,
            ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
            )
              ? Prisma.raw(" || '%'")
              : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(

 const castValueToPostgresTypes = (
  column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
  return column.name === "type" &&
    (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
  safeColumn: string,
  operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
  if (!dateOperators.includes(operator)) {
    throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
  }

  return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
  )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };
@@ -1,9 +1,19 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
-
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./features/evals/types";
+export * from "./observationsTable";
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
+export * from "./server/evals/types";

-export { ModelUsageUnit };
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";
@@ -41,6 +41,7 @@ export const tableNames = [
  "traces_metrics",
  "traces_parent_observation_scores",
  "sessions",
+  "prompts",
 ] as const;

 export type TableNames = (typeof tableNames)[number];
@@ -1,8 +1,8 @@
+import { ObservationLevel } from "@prisma/client";
 import {
  type OptionsDefinition,
  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";

 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -164,7 +164,7 @@ export type ObservationOptions = {
 };

 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
  return observationsTableCols.map((col) => {
    if (col.id === "model") {
@@ -0,0 +1,41 @@
+import { z } from "zod";
+
+export const QueueEnvelope = z.object({
+  timestamp: z.string().datetime({ offset: true }),
+  id: z.string(),
+});
+
+export const TraceUpsertEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    traceId: z.string(),
+  }),
+});
+
+export const EvalExecutionEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    jobExecutionId: z.string(),
+  }),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};
@@ -0,0 +1,77 @@
+import z from "zod";
+import { ModelProvider, observationsTableCols, tracesTableCols } from "../..";
+
+export const evalObjects = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      ...tracesTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];
+
+export const evalModels = [
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-3.5-turbo",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-4-turbo-preview",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+] as const;
+
+export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;
@@ -14,13 +14,17 @@ import { ChatOpenAI } from "@langchain/openai";
 import {
  ChatMessage,
  ChatMessageRole,
+  LLMFunctionCall,
  ModelParams,
  ModelProvider,
 } from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";

 type LLMCompletionParams = {
  messages: ChatMessage[];
  modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
 };

 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -30,18 +34,27 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
  params: LLMCompletionParams & {
    streaming: true;
+    functionCall: undefined;
  }
 ): Promise<IterableReadableStream<Uint8Array>>;

 export async function fetchLLMCompletion(
  params: LLMCompletionParams & {
    streaming: false;
+    functionCall: undefined;
  }
 ): Promise<string>;

+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array>> {
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
  const { messages, modelParams, streaming } = params;
  const finalMessages = messages.map((message) => {
    if (message.role === ChatMessageRole.User)
@@ -71,6 +84,22 @@ export async function fetchLLMCompletion(
    });
  }

+  console.log("Making LLM call with params: ", modelParams);
+
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
  if (streaming) {
    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
  }
@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
  role: ChatMessageRole;
  content: string;
@@ -23,11 +26,13 @@ export type UIModelParams = Required<
 };

 // Generic config
-export type ModelConfig = {
-  max_tokens?: number;
-  temperature?: number;
-  top_p?: number;
-};
+export type ModelConfig = z.infer<typeof ZodModelConfig>;
+
+export const ZodModelConfig = z.object({
+  max_tokens: z.number().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+});

 // OpenAI
 export type OpenAIModelParams = {
@@ -72,3 +77,9 @@ export const supportedModels = {
  [ModelProvider.Anthropic]: anthropicModels,
  [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};
@@ -1,8 +1,5 @@
-import {
-  type OptionsDefinition,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
 import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";

 export const tracesTableCols: ColumnDefinition[] = [
  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
@@ -122,7 +119,7 @@ export type TraceOptions = {
 };

 export function tracesTableColsWithOptions(
-  options?: TraceOptions,
+  options?: TraceOptions
 ): ColumnDefinition[] {
  return tracesTableCols.map((col) => {
    if (col.id === "scores_avg") {
@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";

 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;
@@ -29,4 +29,5 @@ export type TableName =
  | "generations"
  | "sessions"
  | "scores"
+  | "prompts"
  | "dashboard";
@@ -6,7 +6,8 @@
    "lib": ["ES2020"],
    "outDir": "./dist",
    "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
  },
  "include": ["."],
  "exclude": ["node_modules", "dist"]
@@ -1561,6 +1561,8 @@ components:
          nullable: true
        datasetId:
          type: string
+        datasetName:
+          type: string
        createdAt:
          type: string
          format: date-time
@@ -1571,6 +1573,7 @@ components:
        - id
        - status
        - datasetId
+        - datasetName
        - createdAt
        - updatedAt
    DatasetRunItem:
@@ -1581,6 +1584,8 @@ components:
          type: string
        datasetRunId:
          type: string
+        datasetRunName:
+          type: string
        datasetItemId:
          type: string
        traceId:
@@ -1597,6 +1602,7 @@ components:
      required:
        - id
        - datasetRunId
+        - datasetRunName
        - datasetItemId
        - traceId
        - createdAt
@@ -2412,6 +2418,50 @@ components:
        - name
    CreatePromptRequest:
      title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
      type: object
      properties:
        name:
@@ -2429,20 +2479,75 @@ components:
        - prompt
    Prompt:
      title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
      type: object
      properties:
        name:
          type: string
        version:
          type: integer
-        prompt:
-          type: string
        config: {}
      required:
        - name
        - version
-        - prompt
        - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
    CreateScoreRequest:
      title: CreateScoreRequest
      type: object
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "2.16.2",
+  "version": "2.21.2",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -18,19 +18,19 @@
    "test": "dotenv -e ../.env -- jest --runInBand",
    "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
    "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.540.0",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
    "@aws-sdk/s3-request-presigner": "^3.540.0",
+    "@codemirror/lang-json": "^6.0.1",
    "@headlessui/react": "^1.7.18",
    "@heroicons/react": "^2.1.3",
    "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.10",
-    "@langchain/core": "^0.1.54",
+    "@langchain/anthropic": "^0.1.12",
+    "@langchain/core": "^0.1.55",
    "@langchain/openai": "^0.0.26",
    "@langfuse/shared": "workspace:*",
    "@marsidev/react-turnstile": "^0.5.4",
@@ -72,11 +72,13 @@
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
-    "ai": "^3.0.18",
+    "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/react-codemirror": "^4.21.25",
+    "ai": "^3.0.20",
    "bcryptjs": "^2.4.3",
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.0",
-    "cmdk": "^0.2.1",
+    "cmdk": "^1.0.0",
    "core-js": "^3.36.1",
    "cors": "^2.8.5",
    "date-fns": "^3.3.1",
@@ -84,7 +86,7 @@
    "exponential-backoff": "^3.1.1",
    "js-tiktoken": "^1.0.10",
    "kysely": "^0.27.3",
-    "langchain": "^0.1.31",
+    "langchain": "^0.1.32",
    "lodash": "^4.17.21",
    "lucide-react": "^0.364.0",
    "next": "^14.1.4",
@@ -112,8 +114,7 @@
  "devDependencies": {
    "@jedmao/location": "^3.0.0",
    "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.0",
    "@testing-library/jest-dom": "^6.4.2",
    "@testing-library/react": "^14.2.2",
    "@types/bcryptjs": "^2.4.6",
@@ -123,7 +124,7 @@
    "@types/lodash": "^4.14.202",
    "@types/node": "20.10.5",
    "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.74",
+    "@types/react": "^18.2.75",
    "@types/react-dom": "^18.2.24",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/eslint-plugin": "^6.21.0",
@@ -138,7 +139,6 @@
    "postcss": "^8.4.38",
    "prettier": "^3.2.5",
    "prettier-plugin-tailwindcss": "^0.5.13",
-    "release-it": "^17.1.1",
    "tailwindcss": "^3.4.3",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
@@ -150,30 +150,5 @@
  },
  "optionalDependencies": {
    "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
  }
 }
@@ -1,4 +1,7 @@
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
+import {
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server/auth";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { prisma } from "@langfuse/shared/src/db";

@@ -45,13 +45,20 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()

    const datasetItemId = v4();

-    await makeAPICall("POST", "/api/public/dataset-items", {
+    const createItemRes = await makeAPICall<{
+      datasetName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-items", {
      datasetName: "dataset-name-2",
      input: { key: "value" },
      expectedOutput: { key: "value" },
      id: datasetItemId,
    });

+    expect(createItemRes.status).toBe(200);
+    expect(createItemRes.body).toMatchObject({
+      datasetName: "dataset-name-2", // not included in db table
+    });
+
    const traceId = v4();
    const observationId = v4();

@@ -152,6 +159,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
          id: dbDatasetItem!.id,
          input: { key: "value" },
          expectedOutput: { key: "value" },
+          datasetName: "dataset-name", // not included in db table
        },
      ],
    });
@@ -165,6 +173,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
      id: dbDatasetItem!.id,
      input: { key: "value" },
      expectedOutput: { key: "value" },
+      datasetName: "dataset-name", // not included in db table
    });
  });

@@ -303,20 +312,25 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
          datasetItemId: "dataset-item-id",
          observationId: observationId,
          traceId: traceId,
+          datasetRunName: "run-only-observation",
        }),
      ]),
    });

-    const runItemTrace = await makeAPICall(
-      "POST",
-      "/api/public/dataset-run-items",
-      {
-        datasetItemId: "dataset-item-id",
-        traceId: traceId,
-        runName: "run-only-trace",
-        metadata: { key: "value" },
-      },
-    );
+    const runItemTrace = await makeAPICall<{
+      datasetRunName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-run-items", {
+      datasetItemId: "dataset-item-id",
+      traceId: traceId,
+      runName: "run-only-trace",
+      metadata: { key: "value" },
+    });
+
+    expect(runItemTrace.status).toBe(200);
+    expect(runItemTrace.body).toMatchObject({
+      datasetRunName: "run-only-trace", // not included in db table
+    });
+
    const dbRunTrace = await prisma.datasetRuns.findFirst({
      where: {
        name: "run-only-trace",
@@ -520,8 +520,6 @@ describe("/api/public/ingestion API Endpoint", () => {

      expect(response.status).toBe(207);

-      console.log("response body", response.body);
-
      const dbGeneration = await prisma.observation.findUnique({
        where: {
          id: generationId,
@@ -851,7 +849,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -901,7 +898,6 @@ describe("/api/public/ingestion API Endpoint", () => {
      ],
    });

-    console.log(responseOne.body);
    expect(responseOne.status).toBe(207);

    expect("errors" in responseOne.body).toBe(true);
@@ -1167,8 +1163,6 @@ describe("/api/public/ingestion API Endpoint", () => {
    });
    expect(responseOne.status).toBe(207);

-    console.log(responseOne.body);
-
    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
    const errors = responseOne.body.errors;

@@ -1419,7 +1413,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
      input,
    )} ${JSON.stringify(expected)}`, () => {
      const cleanedEvent = cleanEvent(input);
-      console.log(cleanedEvent);
      expect(cleanedEvent).toStrictEqual(expected);
    });
  });
@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared/src/db";
+import { type ObservationView } from "@langfuse/shared";

 describe("/api/public/observations API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -41,6 +41,19 @@ describe("/api/public/observations API Endpoint", () => {
      },
    });

+    const prompt = await prisma.prompt.create({
+      data: {
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
    await prisma.observation.create({
      data: {
        id: uuidv4(),
@@ -62,6 +75,9 @@ describe("/api/public/observations API Endpoint", () => {
        },
        internalModel: "gpt-3.5-turbo",
        unit: ModelUsageUnit.Tokens,
+        prompt: {
+          connect: { id: prompt.id },
+        },
      },
    });

@@ -95,6 +111,7 @@ describe("/api/public/observations API Endpoint", () => {
    expect(
      fetchedObservations.body.data[0]?.calculatedTotalCost,
    ).toBeGreaterThan(0);
+    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
  });
  it("should fetch all observations, filtered by generations", async () => {
    await pruneDatabase();
@@ -1,5 +1,5 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+import { tracesTableCols } from "@langfuse/shared";

 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
@@ -3,7 +3,12 @@
 import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
-import { type Prompt } from "@langfuse/shared/src/db";
+import { type Prompt } from "@langfuse/shared";
+import {
+  PromptSchema,
+  PromptType,
+  type ValidatedPrompt,
+} from "@/src/features/prompts/server/validation";

 describe("/api/public/prompts API Endpoint", () => {
  beforeEach(async () => await pruneDatabase());
@@ -43,10 +48,12 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptId);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
+    expect(fetchedObservations.body.tags).toEqual([]);
  });

  it("should fetch active prompt only if no prompt version is given", async () => {
@@ -190,6 +197,7 @@ describe("/api/public/prompts API Endpoint", () => {
    expect(fetchedObservations.body.id).toBe(promptIdTwo);
    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(2);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -221,6 +229,7 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
@@ -342,8 +351,6 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(response.status).toBe(207);

-    console.log("response body", response.body);
-
    const dbGeneration = await prisma.observation.findUnique({
      where: {
        id: generationId,
@@ -375,11 +382,185 @@ describe("/api/public/prompts API Endpoint", () => {

    expect(fetchedObservations.body.name).toBe("prompt-name");
    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
    expect(fetchedObservations.body.version).toBe(1);
    expect(fetchedObservations.body.isActive).toBe(true);
    expect(fetchedObservations.body.createdBy).toBe("API");
    expect(fetchedObservations.body.config).toEqual({});
  });
+
+  it("should create and fetch a chat prompt", async () => {
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(201);
+
+    const { body: fetchedPrompt } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+
+    const validatedPrompt = validatePrompt(fetchedPrompt);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+  });
+
+  it("should fail if chat prompt has string prompt", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if chat prompt has incorrect messages format", async () => {
+    const promptName = "prompt-name";
+    const incorrectChatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", message: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: incorrectChatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+  it("should fail if text prompt has message format", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: [{ role: "system", content: "You are a bot" }],
+      type: "text",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if previous versions have different prompt type", async () => {
+    // Create a chat prompt
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const postResponse1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse1.status).toBe(201);
+
+    // Try creating a text prompt with the same name
+    const postResponse2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "text",
+      isActive: true,
+      version: 2,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse2.status).toBe(400);
+    expect(postResponse2.body).toEqual({
+      error: "ValidationError",
+      message:
+        "Previous versions have different prompt type. Create a new prompt with a different name.",
+    });
+
+    // Check if the prompt is still the chat prompt
+    const getResponse1 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(getResponse1.status).toBe(200);
+
+    const validatedPrompt = validatePrompt(getResponse1.body);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+
+    // Check that the text prompt has not been created
+    const getResponse2 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}&version=2`,
+      undefined,
+    );
+    expect(getResponse2.status).toBe(404);
+    expect(getResponse2.body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
 });

 const isPrompt = (x: unknown): x is Prompt => {
@@ -393,6 +574,18 @@ const isPrompt = (x: unknown): x is Prompt => {
    typeof prompt.isActive === "boolean" &&
    typeof prompt.projectId === "string" &&
    typeof prompt.createdBy === "string" &&
-    typeof prompt.config === "object"
+    typeof prompt.config === "object" &&
+    Object.values(PromptType).includes(prompt.type as PromptType)
  );
 };
+
+const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+  Object.keys(obj).forEach((key) => {
+    obj[key] =
+      key === "createdAt" || key === "updatedAt"
+        ? new Date(obj[key] as string)
+        : obj[key];
+  });
+
+  return PromptSchema.parse(obj);
+};
@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
              <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
                <MessageCircle size={12} />
              </HoverCardTrigger>
-              <HoverCardContent>
+              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                <p>{s.comment}</p>
              </HoverCardContent>
            </HoverCard>
@@ -0,0 +1,33 @@
+import CodeMirror, { EditorView } from "@uiw/react-codemirror";
+import { githubLight } from "@uiw/codemirror-theme-github";
+import { json } from "@codemirror/lang-json";
+
+// todo: add json linting
+
+export function JsonEditor({
+  defaultValue,
+  onChange,
+  editable = true,
+  lineWrapping = true,
+}: {
+  defaultValue: string;
+  onChange?: (value: string) => void;
+  editable?: boolean;
+  lineWrapping?: boolean;
+}) {
+  return (
+    <CodeMirror
+      value={defaultValue}
+      theme={githubLight}
+      basicSetup={{
+        foldGutter: true,
+      }}
+      lang={"json"}
+      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      defaultValue={defaultValue}
+      onChange={onChange}
+      className="overflow-hidden rounded-md border"
+      editable={editable}
+    />
+  );
+}
@@ -309,8 +309,8 @@ export default function Layout(props: PropsWithChildren) {
              </ul>
            </nav>

-            <Menu as="div" className="relative left-1">
-              <Menu.Button className="flex w-full items-center gap-x-4 p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+            <Menu as="div" className="relative">
+              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
                <span className="sr-only">Open user menu</span>
                <Avatar className="h-8 w-8">
                  <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -342,7 +342,10 @@ export default function Layout(props: PropsWithChildren) {
                leaveFrom="transform opacity-100 scale-100"
                leaveTo="transform opacity-0 scale-95"
              >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                    {session.data?.user?.email}
+                  </span>
                  {userNavigation.map((item) => (
                    <Menu.Item key={item.name}>
                      {({ active }) => (
@@ -403,7 +406,10 @@ export default function Layout(props: PropsWithChildren) {
              leaveFrom="transform opacity-100 scale-100"
              leaveTo="transform opacity-0 scale-95"
            >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  {session.data?.user?.email}
+                </span>
                {userNavigation.map((item) => (
                  <Menu.Item key={item.name}>
                    {({ active }) => (
@@ -496,6 +502,7 @@ const MainNavigation: React.FC<{
                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
                )}
                onClick={onNavitemClick}
+                target={item.newTab ? "_blank" : undefined}
              >
                {item.icon && (
                  <item.icon
@@ -574,6 +581,7 @@ const MainNavigation: React.FC<{
                                : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
                            )}
+                            target={subItem.newTab ? "_blank" : undefined}
                          >
                            {subItem.name}
                            {subItem.label && (
@@ -9,6 +9,7 @@ import {
  Settings,
  UsersIcon,
  PenSquareIcon,
+  LibraryBig,
  TerminalIcon,
 } from "lucide-react";

@@ -21,6 +22,7 @@ export type Route = {
  pathname?: string; // link, ignored if children
  children?: Array<Route>; // folder
  bottom?: boolean; // bottom of the sidebar, only for first level routes
+  newTab?: boolean; // open in new tab
 };

 export const ROUTES: Route[] = [
@@ -53,6 +55,16 @@ export const ROUTES: Route[] = [
        name: "Models",
        pathname: `/project/[projectId]/models`,
      },
+      {
+        name: "Templates",
+        pathname: `/project/[projectId]/evals/templates`,
+        featureFlag: "evals",
+      },
+      {
+        name: "Configs",
+        pathname: `/project/[projectId]/evals/configs`,
+        featureFlag: "evals",
+      },
    ],
  },
  {
@@ -83,6 +95,13 @@ export const ROUTES: Route[] = [
    icon: Settings,
    bottom: true,
  },
+  {
+    name: "Docs",
+    pathname: "https://langfuse.com/docs",
+    icon: LibraryBig,
+    bottom: true,
+    newTab: true,
+  },
  {
    name: "Support",
    pathname: "/project/[projectId]/support",
@@ -7,7 +7,6 @@ import {
  SelectValue,
 } from "@/src/components/ui/select";
 import { cn } from "@/src/utils/tailwind";
-import { Code } from "lucide-react";
 import { useRouter } from "next/router";

 interface Project {
@@ -2,9 +2,9 @@ import { Button } from "@/src/components/ui/button";
 import React, { type Dispatch, type SetStateAction, useState } from "react";
 import { Input } from "@/src/components/ui/input";
 import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-table-column-visibility-filter";
-import { type FilterState } from "@/src/features/filters/types";
-import { FilterBuilder } from "@/src/features/filters/components/filter-builder";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import { type FilterState } from "@langfuse/shared";
+import { PopoverFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import { type ColumnDefinition } from "@langfuse/shared";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";

@@ -16,11 +16,11 @@ interface SearchConfig {

 interface DataTableToolbarProps<TData, TValue> {
  columns: LangfuseColumnDef<TData, TValue>[];
-  filterColumnDefinition: ColumnDefinition[];
+  filterColumnDefinition?: ColumnDefinition[];
  searchConfig?: SearchConfig;
  actionButtons?: React.ReactNode;
-  filterState: FilterState;
-  setFilterState: Dispatch<SetStateAction<FilterState>>;
+  filterState?: FilterState;
+  setFilterState?: Dispatch<SetStateAction<FilterState>>;
  columnVisibility?: VisibilityState;
  setColumnVisibility?: Dispatch<SetStateAction<VisibilityState>>;
 }
@@ -64,11 +64,13 @@ export function DataTableToolbar<TData, TValue>({
            </Button>
          </div>
        )}
-        <FilterBuilder
-          columns={filterColumnDefinition}
-          filterState={filterState}
-          onChange={setFilterState}
-        />
+        {!!filterColumnDefinition && !!filterState && !!setFilterState && (
+          <PopoverFilterBuilder
+            columns={filterColumnDefinition}
+            filterState={filterState}
+            onChange={setFilterState}
+          />
+        )}
        <div className="flex-1" />
        {!!columnVisibility && !!setColumnVisibility && (
          <DataTableColumnVisibilityFilter
@@ -4,22 +4,26 @@ export type TableLinkProps = {
  path: string;
  value: string;
  truncateAt?: number;
+  icon?: React.ReactNode;
 };

 export default function TableLink({
  path,
  value,
  truncateAt = 7,
+  icon,
 }: TableLinkProps) {
+  const truncatedValue =
+    value.length - truncateAt > 3
+      ? `...${value.substring(value.length - truncateAt)}`
+      : value;
  return (
    <Link
      className="inline-block rounded bg-indigo-50 px-2 py-1 text-xs font-semibold text-blue-600 shadow-sm hover:bg-indigo-100"
      href={path}
      title={value}
    >
-      {value.length - truncateAt > 3
-        ? `...${value.substring(value.length - truncateAt)}`
-        : value}
+      {icon ? icon : truncatedValue}
    </Link>
  );
 }
@@ -13,7 +13,7 @@ export const IOCell = ({
 }) => {
  return (
    <>
-      {isLoading  ? (
+      {isLoading ? (
        <JsonSkeleton className="h-[250px] w-[500px] px-3 py-1" />
      ) : (
        <JSONView json={data} className="h-[250px] w-[500px] overflow-y-auto" />
@@ -42,10 +42,8 @@ export const JsonSkeleton = ({
  return (
    <div className={cn("w-[500px] rounded-md border", className)}>
      <div className="flex flex-col gap-1">
-        {[...Array<number>(numRows)].map((_) => (
-          <>
-            <Skeleton className={generateRandomSize()} />
-          </>
+        {[...Array<number>(numRows)].map((_, i) => (
+          <Skeleton className={generateRandomSize()} key={i} />
        ))}
        <br />
      </div>
@@ -22,7 +22,6 @@ import {
  withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
 import {
  formatIntervalSeconds,
  intervalInSeconds,
@@ -30,14 +29,15 @@ import {
 } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type ObservationLevel } from "@langfuse/shared/src/db";
+import { type ObservationLevel } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
 import { usdFormatter } from "@/src/utils/numbers";
 import {
  exportOptions,
  type ExportFileFormats,
-} from "@/src/server/api/interfaces/exportTypes";
+  observationsTableColsWithOptions,
+} from "@langfuse/shared";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
@@ -695,6 +695,7 @@ const GenerationsIOCell = ({
          skipBatch: true,
        },
      },
+      refetchOnMount: false, // prevents refetching loops
    },
  );
  return (
@@ -6,11 +6,11 @@ import useColumnVisibility from "@/src/features/column-visibility/hooks/useColum
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
-  ScoreOptions,
+  type ScoreOptions,
  scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { RouterOutput, type RouterInput } from "@/src/utils/types";
+import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";

 export type ScoresTableRow = {
@@ -6,7 +6,7 @@ import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsTableColsWithOptions } from "@/src/server/api/definitions/sessionsView";
@@ -10,17 +10,9 @@ import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import { Checkbox } from "@/src/components/ui/checkbox";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
-import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
-import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
-import {
-  TraceOptions,
-  tracesTableColsWithOptions,
-} from "@/src/server/api/definitions/tracesTable";
 import { api } from "@/src/utils/api";
 import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
-import { type ObservationLevel, type Score } from "@langfuse/shared/src/db";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
 import {
@@ -37,6 +29,15 @@ import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
 import { IOCell } from "./IOCell";
 import { setSmallPaginationIfColumnsVisible } from "@/src/features/column-visibility/hooks/setSmallPaginationIfColumnsVisible";
+import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
+import {
+  type FilterState,
+  type TraceOptions,
+  tracesTableColsWithOptions,
+  type ObservationLevel,
+  type Score,
+} from "@langfuse/shared";

 export type TracesTableRow = {
  bookmarked: boolean;
@@ -466,9 +467,7 @@ export default function TracesTable({
      id: "input",
      cell: ({ row }) => {
        const traceId: string = row.getValue("id");
-        return (
-          <TracesIOCell traceId={traceId} projectId={projectId} io="input" />
-        );
+        return <TracesIOCell traceId={traceId} io="input" />;
      },
      enableHiding: true,
      defaultHidden: true,
@@ -479,9 +478,7 @@ export default function TracesTable({
      id: "output",
      cell: ({ row }) => {
        const traceId: string = row.getValue("id");
-        return (
-          <TracesIOCell traceId={traceId} projectId={projectId} io="output" />
-        );
+        return <TracesIOCell traceId={traceId} io="output" />;
      },
      enableHiding: true,
      defaultHidden: true,
@@ -643,11 +640,9 @@ export default function TracesTable({
 }

 const TracesIOCell = ({
-  projectId,
  traceId,
  io,
 }: {
-  projectId: string;
  traceId: string;
  io: "input" | "output";
 }) => {
@@ -660,6 +655,7 @@ const TracesIOCell = ({
          skipBatch: true,
        },
      },
+      refetchOnMount: false, // prevents refetching loops
    },
  );
  return (
@@ -1,6 +1,6 @@
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Observation } from "@langfuse/shared/src/db";
+import { type Observation } from "@langfuse/shared";

 export const TraceAggUsageBadge = (props: {
  observations: ObservationReturnType[];
@@ -136,18 +136,18 @@ const ChatMlMessageSchema = z
    content,
    json: Object.keys(other).length === 0 ? undefined : other,
  }));
-const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
+export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);

-const OpenAiMessageView: React.FC<{
+export const OpenAiMessageView: React.FC<{
+  title?: string;
  messages: z.infer<typeof ChatMlArraySchema>;
-}> = ({ messages }) => {
+}> = ({ title, messages }) => {
  const COLLAPSE_THRESHOLD = 3;
  const [isCollapsed, setCollapsed] = useState(
    messages.length > COLLAPSE_THRESHOLD ? true : null,
  );

  const transformedMessages = messages;
-  console.log(transformedMessages);
  // const transformedMessages = messages.map(
  //   ({ role, name, content, ...rest }) => ({
  //     role,
@@ -158,58 +158,65 @@ const OpenAiMessageView: React.FC<{
  // );

  return (
-    <div className="flex flex-col gap-2 rounded-md border p-3">
-      {transformedMessages
-        .filter(
-          (_, i) =>
-            // show all if not collapsed or null; show first and last n if collapsed
-            !isCollapsed || i == 0 || i > messages.length - COLLAPSE_THRESHOLD,
-        )
-        .map((message, index) => (
-          <Fragment key={index}>
-            <div>
-              {!!message.content && (
-                <JSONView
-                  title={message.name ?? message.role}
-                  json={message.content}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.json && "rounded-b-none",
-                  )}
-                />
-              )}
-              {!!message.json && (
-                <JSONView
-                  title={
-                    message.content ? undefined : message.name ?? message.role
-                  }
-                  json={message.json}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.content && "rounded-t-none border-t-0",
-                  )}
-                />
-              )}
-            </div>
-            {isCollapsed !== null && index === 0 ? (
-              <Button
-                variant="ghost"
-                size="xs"
-                onClick={() => setCollapsed((v) => !v)}
-              >
-                {isCollapsed
-                  ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
-                  : "Hide history"}
-              </Button>
-            ) : null}
-          </Fragment>
-        ))}
+    <div className="rounded-md border">
+      {title && (
+        <div className="border-b px-3 py-1 text-xs font-medium">{title}</div>
+      )}
+      <div className="flex flex-col gap-2 p-3">
+        {transformedMessages
+          .filter(
+            (_, i) =>
+              // show all if not collapsed or null; show first and last n if collapsed
+              !isCollapsed ||
+              i == 0 ||
+              i > messages.length - COLLAPSE_THRESHOLD,
+          )
+          .map((message, index) => (
+            <Fragment key={index}>
+              <div>
+                {!!message.content && (
+                  <JSONView
+                    title={message.name ?? message.role}
+                    json={message.content}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.json && "rounded-b-none",
+                    )}
+                  />
+                )}
+                {!!message.json && (
+                  <JSONView
+                    title={
+                      message.content ? undefined : message.name ?? message.role
+                    }
+                    json={message.json}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.content && "rounded-t-none border-t-0",
+                    )}
+                  />
+                )}
+              </div>
+              {isCollapsed !== null && index === 0 ? (
+                <Button
+                  variant="ghost"
+                  size="xs"
+                  onClick={() => setCollapsed((v) => !v)}
+                >
+                  {isCollapsed
+                    ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
+                    : "Hide history"}
+                </Button>
+              ) : null}
+            </Fragment>
+          ))}
+      </div>
    </div>
  );
 };
@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/code";
-import { type Score } from "@langfuse/shared/src/db";
+import { type Score } from "@langfuse/shared";
 import {
  Card,
  CardContent,
@@ -1,6 +1,6 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, $Enums } from "@langfuse/shared/src/db";
+import { type Trace, type Score, $Enums } from "@langfuse/shared";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/code";
-import { type Trace, type Score } from "@langfuse/shared/src/db";
+import { type Trace, type Score } from "@langfuse/shared";
 import {
  Card,
  CardContent,
@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared/src/db";
+import { type Trace, type Score } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";
@@ -120,7 +120,7 @@ const CommandItem = React.forwardRef<
  <CommandPrimitive.Item
    ref={ref}
    className={cn(
-      "relative flex cursor-pointer select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none aria-selected:bg-slate-100 aria-selected:text-slate-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:aria-selected:bg-slate-800 dark:aria-selected:text-slate-50",
+      "relative flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none aria-selected:bg-slate-100 aria-selected:text-slate-900 dark:aria-selected:bg-slate-800 dark:aria-selected:text-slate-50",
      className,
    )}
    {...props}
@@ -1 +1 @@
-export const VERSION = "v2.16.2";
+export const VERSION = "v2.21.2";
@@ -61,6 +61,7 @@ export const env = createEnv({
    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
    AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
    AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
+    AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
    // EMAIL
    EMAIL_FROM_ADDRESS: z.string().optional(),
    SMTP_CONNECTION_URL: z.string().optional(),
@@ -72,6 +73,9 @@ export const env = createEnv({
    S3_REGION: z.string().optional(),
    // Database exports
    DB_EXPORT_PAGE_SIZE: z.number().optional(),
+    // Worker
+    LANGFUSE_WORKER_HOST: z.string().optional(),
+    LANGFUSE_WORKER_PASSWORD: z.string().optional(),
    // Prompt playground
    OPENAI_API_KEY: z.string().optional(),
    ANTHROPIC_API_KEY: z.string().optional(),
@@ -82,6 +86,8 @@ export const env = createEnv({
   * Specify your client-side environment variables schema here. This way you can ensure the app
   * isn't built with invalid env vars. To expose them to the client, prefix them with
   * `NEXT_PUBLIC_`.
+   * 
+   * WARNING: They do not work when used in Docker builds as NEXT_PUBLIC variables are not runtime but compile-time.
   */
  client: {
    // NEXT_PUBLIC_CLIENTVAR: z.string().min(1),
@@ -137,14 +143,17 @@ export const env = createEnv({
    AUTH_OKTA_CLIENT_ID: process.env.AUTH_OKTA_CLIENT_ID,
    AUTH_OKTA_CLIENT_SECRET: process.env.AUTH_OKTA_CLIENT_SECRET,
    AUTH_OKTA_ISSUER: process.env.AUTH_OKTA_ISSUER,
-    AUTH_OKTA_ALLOW_ACCOUNT_LINKING: process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
+    AUTH_OKTA_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
    AUTH_AUTH0_CLIENT_ID: process.env.AUTH_AUTH0_CLIENT_ID,
    AUTH_AUTH0_CLIENT_SECRET: process.env.AUTH_AUTH0_CLIENT_SECRET,
    AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
-    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
    AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
      process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
    AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
+    AUTH_DISABLE_SIGNUP: process.env.AUTH_DISABLE_SIGNUP,
    // Email
    EMAIL_FROM_ADDRESS: process.env.EMAIL_FROM_ADDRESS,
    SMTP_CONNECTION_URL: process.env.SMTP_CONNECTION_URL,
@@ -156,6 +165,9 @@ export const env = createEnv({
    S3_REGION: process.env.S3_REGION,
    // Database exports
    DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
+    // Worker
+    LANGFUSE_WORKER_HOST: process.env.LANGFUSE_WORKER_HOST,
+    LANGFUSE_WORKER_PASSWORD: process.env.LANGFUSE_WORKER_PASSWORD,
    // Prompt playground
    OPENAI_API_KEY: process.env.OPENAI_API_KEY,
    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
@@ -1,5 +1,5 @@
 import { prisma as _prisma } from "@langfuse/shared/src/db";
-import { type MembershipRole } from "@langfuse/shared/src/db";
+import { type MembershipRole } from "@langfuse/shared";

 export type AuditableResource =
  | "membership"
@@ -13,7 +13,9 @@ export type AuditableResource =
  | "model"
  | "prompt"
  | "session"
-  | "apiKey";
+  | "apiKey"
+  | "evalTemplate"
+  | "job";

 type AuditLog = {
  resourceType: AuditableResource;
@@ -1,7 +1,6 @@
-import { type LangfuseColumnDef } from "@/src/components/table/types";
-import React from "react";
-import { TracesTableRow } from "@/src/components/table/use-cases/traces";
-import { GenerationsTableRow } from "../../../components/table/use-cases/generations";
+import type React from "react";
+import type { TracesTableRow } from "@/src/components/table/use-cases/traces";
+import type { GenerationsTableRow } from "../../../components/table/use-cases/generations";

 export type TableColumn =
  | (keyof GenerationsTableRow)[]
@@ -5,7 +5,7 @@ import {
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
  extractTimeSeriesData,
  fillMissingValuesAndTransform,
@@ -3,7 +3,7 @@ import {
  dateTimeAggregationSettings,
  type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
  getAllModels,
  extractTimeSeriesData,
@@ -1,7 +1,7 @@
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";

 import { type DatabaseRow } from "@/src/server/api/services/query-builder";
@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";
@@ -4,7 +4,7 @@ import {
  dateTimeAggregationSettings,
  type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";

 import {
  getAllModels,
@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { RightAlignedCell } from "./RightAlignedCell";
@@ -1,5 +1,5 @@
 import { api } from "@/src/utils/api";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { ExpandListButton } from "@/src/features/dashboard/components/cards/ChevronButton";
 import { useState } from "react";
 import DocPopup from "@/src/components/layouts/doc-popup";
@@ -3,7 +3,7 @@ import {
  dateTimeAggregationSettings,
  type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
@@ -1,6 +1,6 @@
 import { api } from "@/src/utils/api";
 import { type DateTimeAggregationOption } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TabComponent } from "@/src/features/dashboard/components/TabsComponent";
@@ -1,5 +1,5 @@
 import { type TimeSeriesChartDataPoint } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { type DatabaseRow } from "@/src/server/api/services/query-builder";
 import { api } from "@/src/utils/api";

@@ -1,4 +1,4 @@
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { usdFormatter } from "@/src/utils/numbers";

 // traces do not have a startTime or endTime column, so we need to map these to the timestamp column
@@ -11,16 +11,22 @@ import {
 } from "@/src/components/ui/dropdown-menu";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";

-import { Archive, MoreVertical } from "lucide-react";
+import { Archive, ListTree, MoreVertical } from "lucide-react";
 import { Button } from "@/src/components/ui/button";
-import { DatasetStatus, type DatasetItem } from "@langfuse/shared/src/db";
+import { type DatasetItem, DatasetStatus } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useEffect } from "react";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";

 type RowData = {
  id: string;
+  source?: {
+    traceId: string;
+    observationId?: string;
+  };
  status: DatasetItem["status"];
  createdAt: string;
  input: string;
@@ -67,6 +73,7 @@ export function DatasetItemsTable({
    {
      accessorKey: "id",
      header: "Item id",
+      id: "id",
      cell: ({ row }) => {
        const id: string = row.getValue("id");
        return (
@@ -78,9 +85,36 @@ export function DatasetItemsTable({
        );
      },
    },
+    {
+      accessorKey: "source",
+      header: "Source",
+      headerTooltip: {
+        description:
+          "Link to the source trace based on which this item was added",
+      },
+      id: "source",
+      cell: ({ row }) => {
+        const source: RowData["source"] = row.getValue("source");
+        if (!source) return null;
+        return source.observationId ? (
+          <TableLink
+            path={`/project/${projectId}/traces/${source.traceId}?observation=${source.observationId}`}
+            value={source.observationId}
+            icon={<ListTree className="h-4 w-4" />}
+          />
+        ) : (
+          <TableLink
+            path={`/project/${projectId}/traces/${source.traceId}`}
+            value={source.traceId}
+            icon={<ListTree className="h-4 w-4" />}
+          />
+        );
+      },
+    },
    {
      accessorKey: "status",
      header: "Status",
+      id: "status",
      cell: ({ row }) => {
        const status: DatasetStatus = row.getValue("status");
        return (
@@ -100,15 +134,21 @@ export function DatasetItemsTable({
    },
    {
      accessorKey: "createdAt",
-      header: "Created",
+      header: "Created At",
+      id: "createdAt",
+      enableHiding: true,
    },
    {
      accessorKey: "input",
      header: "Input",
+      id: "input",
+      enableHiding: true,
    },
    {
      accessorKey: "expectedOutput",
      header: "Expected Output",
+      id: "expectedOutput",
+      enableHiding: true,
    },
    {
      id: "actions",
@@ -165,6 +205,12 @@ export function DatasetItemsTable({

    return {
      id: item.id,
+      source: item.sourceTraceId
+        ? {
+            traceId: item.sourceTraceId,
+            observationId: item.sourceObservationId ?? undefined,
+          }
+        : undefined,
      status: item.status,
      createdAt: item.createdAt.toLocaleString(),
      input,
@@ -172,8 +218,18 @@ export function DatasetItemsTable({
    };
  };

+  const [columnVisibility, setColumnVisibility] = useColumnVisibility<RowData>(
+    "datasetItemsColumnVisibility",
+    columns,
+  );
+
  return (
    <div>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+      />
      <DataTable
        columns={columns}
        data={
@@ -200,6 +256,8 @@ export function DatasetItemsTable({
          onChange: setPaginationState,
          state: paginationState,
        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
      />
    </div>
  );
@@ -3,12 +3,17 @@ import { DataTable } from "@/src/components/table/data-table";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, intervalInSeconds } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";

-import { type Score } from "@langfuse/shared/src/db";
+import { type Score } from "@langfuse/shared";
 import { usdFormatter } from "../../../utils/numbers";
+import { IOCell } from "@/src/components/table/use-cases/IOCell";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { useEffect } from "react";

 type RowData = {
  id: string;
@@ -18,6 +23,11 @@ type RowData = {
    traceId: string;
    observationId?: string;
  };
+  // i/o not set explicitly, but fetched from the server from the cell
+  input?: unknown;
+  output?: unknown;
+  expectedOutput?: unknown;
+
  scores: Score[];
  latency?: number;
  totalCost?: string;
@@ -28,32 +38,50 @@ export function DatasetRunItemsTable(
    | {
        projectId: string;
        datasetId: string;
-        datasetRunId: string;
+        datasetRunId: string; // View from run page
      }
    | {
        projectId: string;
        datasetId: string;
-        datasetItemId: string;
+        datasetItemId: string; // View from item page
      },
 ) {
+  const { setDetailPageList } = useDetailPageLists();
  const [paginationState, setPaginationState] = useQueryParams({
    pageIndex: withDefault(NumberParam, 0),
-    pageSize: withDefault(NumberParam, 50),
+    pageSize: withDefault(NumberParam, 20),
  });
  const runItems = api.datasets.runitemsByRunIdOrItemId.useQuery({
    ...props,
    page: paginationState.pageIndex,
    limit: paginationState.pageSize,
  });
+  useEffect(() => {
+    if (runItems.isSuccess) {
+      setDetailPageList(
+        "traces",
+        runItems.data.runItems.filter((i) => !!i.trace).map((i) => i.trace!.id),
+      );
+      // set the datasetItems list only when viewing this table from the run page
+      if ("datasetRunId" in props)
+        setDetailPageList(
+          "datasetItems",
+          runItems.data.runItems.map((i) => i.datasetItemId),
+        );
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [runItems.isSuccess, runItems.data]);

  const columns: LangfuseColumnDef<RowData>[] = [
    {
      accessorKey: "runAt",
      header: "Run At",
+      id: "runAt",
    },
    {
      accessorKey: "datasetItemId",
      header: "Dataset Item",
+      id: "datasetItemId",
      cell: ({ row }) => {
        const datasetItemId: string = row.getValue("datasetItemId");
        return (
@@ -68,6 +96,7 @@ export function DatasetRunItemsTable(
    {
      accessorKey: "trace",
      header: "Trace",
+      id: "trace",
      cell: ({ row }) => {
        const trace: RowData["trace"] = row.getValue("trace");
        if (!trace) return null;
@@ -89,6 +118,8 @@ export function DatasetRunItemsTable(
    {
      accessorKey: "latency",
      header: "Latency",
+      id: "latency",
+      enableHiding: true,
      cell: ({ row }) => {
        const latency: RowData["latency"] = row.getValue("latency");
        return <>{!!latency ? formatIntervalSeconds(latency) : null}</>;
@@ -97,6 +128,8 @@ export function DatasetRunItemsTable(
    {
      accessorKey: "totalCost",
      header: "Total Cost",
+      id: "totalCost",
+      enableHiding: true,
      cell: ({ row }) => {
        const totalCost: RowData["totalCost"] = row.getValue("totalCost");
        return <>{totalCost}</>;
@@ -105,11 +138,62 @@ export function DatasetRunItemsTable(
    {
      accessorKey: "scores",
      header: "Scores",
+      id: "scores",
+      enableHiding: true,
      cell: ({ row }) => {
        const scores: RowData["scores"] = row.getValue("scores");
        return <GroupedScoreBadges scores={scores} variant="headings" />;
      },
    },
+    {
+      accessorKey: "input",
+      header: "Input",
+      id: "input",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const trace: RowData["trace"] = row.getValue("trace");
+        return trace ? (
+          <TraceObservationIOCell
+            traceId={trace.traceId}
+            observationId={trace.observationId}
+            io="input"
+          />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "output",
+      header: "Output",
+      id: "output",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const trace: RowData["trace"] = row.getValue("trace");
+        return trace ? (
+          <TraceObservationIOCell
+            traceId={trace.traceId}
+            observationId={trace.observationId}
+            io="output"
+          />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "expectedOutput",
+      header: "Expected Output",
+      id: "expectedOutput",
+      enableHiding: true,
+      cell: ({ row }) => {
+        const datasetItemId: string = row.getValue("datasetItemId");
+        return datasetItemId ? (
+          <DatasetItemIOCell
+            projectId={props.projectId}
+            datasetId={props.datasetId}
+            datasetItemId={datasetItemId}
+            io="expectedOutput"
+          />
+        ) : null;
+      },
+    },
  ];

  const convertToTableRow = (
@@ -133,31 +217,132 @@ export function DatasetRunItemsTable(
    };
  };

+  const [columnVisibility, setColumnVisibility] = useColumnVisibility<RowData>(
+    "datasetRunsItemsColumnVisibility",
+    columns,
+  );
+
  return (
-    <DataTable
-      columns={columns}
-      data={
-        runItems.isLoading
-          ? { isLoading: true, isError: false }
-          : runItems.isError
-            ? {
-                isLoading: false,
-                isError: true,
-                error: runItems.error.message,
-              }
-            : {
-                isLoading: false,
-                isError: false,
-                data: runItems.data.runItems.map((t) => convertToTableRow(t)),
-              }
-      }
-      pagination={{
-        pageCount: Math.ceil(
-          (runItems.data?.totalRunItems ?? 0) / paginationState.pageSize,
-        ),
-        onChange: setPaginationState,
-        state: paginationState,
-      }}
-    />
+    <div>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+      />
+      <DataTable
+        columns={columns}
+        data={
+          runItems.isLoading
+            ? { isLoading: true, isError: false }
+            : runItems.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: runItems.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: runItems.data.runItems.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(
+            (runItems.data?.totalRunItems ?? 0) / paginationState.pageSize,
+          ),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
  );
 }
+
+const TraceObservationIOCell = ({
+  traceId,
+  observationId,
+  io,
+}: {
+  traceId: string;
+  observationId?: string;
+  io: "input" | "output";
+}) => {
+  // conditionally fetch the trace or observation depending on the presence of observationId
+  const trace = api.traces.byId.useQuery(
+    { traceId: traceId },
+    {
+      enabled: observationId === undefined,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+  const observation = api.observations.byId.useQuery(
+    {
+      observationId: observationId as string, // disabled when observationId is undefined
+      traceId: traceId,
+    },
+    {
+      enabled: observationId !== undefined,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+
+  const data = observationId === undefined ? trace.data : observation.data;
+
+  return (
+    <IOCell
+      isLoading={!!!observationId ? trace.isLoading : observation.isLoading}
+      data={io === "output" ? data?.output : data?.input}
+    />
+  );
+};
+
+const DatasetItemIOCell = ({
+  projectId,
+  datasetId,
+  datasetItemId,
+  io,
+}: {
+  projectId: string;
+  datasetId: string;
+  datasetItemId: string;
+  io: "expectedOutput" | "input";
+}) => {
+  const datasetItem = api.datasets.itemById.useQuery(
+    {
+      projectId: projectId,
+      datasetId: datasetId,
+      datasetItemId: datasetItemId,
+    },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false, // prevents refetching loops
+    },
+  );
+
+  return (
+    <IOCell
+      isLoading={datasetItem.isLoading}
+      data={
+        io === "expectedOutput"
+          ? datasetItem.data?.expectedOutput
+          : datasetItem.data?.input
+      }
+    />
+  );
+};
@@ -10,6 +10,8 @@ import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 import { type RouterOutput } from "@/src/utils/types";
 import { useEffect } from "react";
 import { usdFormatter } from "../../../utils/numbers";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";

 type RowData = {
  key: {
@@ -53,6 +55,7 @@ export function DatasetRunsTable(props: {
    {
      accessorKey: "key",
      header: "Name",
+      id: "key",
      cell: ({ row }) => {
        const key: RowData["key"] = row.getValue("key");
        return (
@@ -67,14 +70,20 @@ export function DatasetRunsTable(props: {
    {
      accessorKey: "description",
      header: "Description",
+      id: "description",
+      enableHiding: true,
    },
    {
      accessorKey: "countRunItems",
      header: "Run Items",
+      id: "countRunItems",
+      enableHiding: true,
    },
    {
      accessorKey: "avgLatency",
      header: "Latency (avg)",
+      id: "avgLatency",
+      enableHiding: true,
      cell: ({ row }) => {
        const avgLatency: RowData["avgLatency"] = row.getValue("avgLatency");
        return <>{formatIntervalSeconds(avgLatency)}</>;
@@ -83,6 +92,8 @@ export function DatasetRunsTable(props: {
    {
      accessorKey: "avgTotalCost",
      header: "Total Cost (avg)",
+      id: "avgTotalCost",
+      enableHiding: true,
      cell: ({ row }) => {
        const avgTotalCost: RowData["avgTotalCost"] =
          row.getValue("avgTotalCost");
@@ -92,6 +103,8 @@ export function DatasetRunsTable(props: {
    {
      accessorKey: "scores",
      header: "Scores (avg)",
+      id: "scores",
+      enableHiding: true,
      cell: ({ row }) => {
        const scores: RowData["scores"] = row.getValue("scores");
        return (
@@ -108,10 +121,14 @@ export function DatasetRunsTable(props: {
    {
      accessorKey: "createdAt",
      header: "Created",
+      id: "createdAt",
+      enableHiding: true,
    },
    {
      accessorKey: "metadata",
      header: "Metadata",
+      id: "metadata",
+      enableHiding: true,
      cell: ({ row }) => {
        const metadata: RowData["metadata"] = row.getValue("metadata");
        return <div className="flex flex-wrap gap-x-3 gap-y-1">{metadata}</div>;
@@ -134,31 +151,45 @@ export function DatasetRunsTable(props: {
    };
  };

+  const [columnVisibility, setColumnVisibility] = useColumnVisibility<RowData>(
+    "datasetRunsColumnVisibility",
+    columns,
+  );
+
  return (
-    <DataTable
-      columns={columns}
-      data={
-        runs.isLoading
-          ? { isLoading: true, isError: false }
-          : runs.isError
-            ? {
-                isLoading: false,
-                isError: true,
-                error: runs.error.message,
-              }
-            : {
-                isLoading: false,
-                isError: false,
-                data: runs.data.runs.map((t) => convertToTableRow(t)),
-              }
-      }
-      pagination={{
-        pageCount: Math.ceil(
-          (runs.data?.totalRuns ?? 0) / paginationState.pageSize,
-        ),
-        onChange: setPaginationState,
-        state: paginationState,
-      }}
-    />
+    <div>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+      />
+      <DataTable
+        columns={columns}
+        data={
+          runs.isLoading
+            ? { isLoading: true, isError: false }
+            : runs.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: runs.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: runs.data.runs.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(
+            (runs.data?.totalRuns ?? 0) / paginationState.pageSize,
+          ),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
  );
 }
@@ -15,6 +15,8 @@ import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 import { type RouterOutput } from "@/src/utils/types";
 import { MoreVertical } from "lucide-react";
 import { useEffect } from "react";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";

 type RowData = {
  key: {
@@ -56,6 +58,7 @@ export function DatasetsTable(props: { projectId: string }) {
    {
      accessorKey: "key",
      header: "Name",
+      id: "key",
      cell: ({ row }) => {
        const key: RowData["key"] = row.getValue("key");
        return (
@@ -70,22 +73,32 @@ export function DatasetsTable(props: { projectId: string }) {
    {
      accessorKey: "description",
      header: "Description",
+      id: "description",
+      enableHiding: true,
    },
    {
      accessorKey: "countItems",
      header: "Items",
+      id: "countItems",
+      enableHiding: true,
    },
    {
      accessorKey: "countRuns",
      header: "Runs",
+      id: "countRuns",
+      enableHiding: true,
    },
    {
      accessorKey: "createdAt",
      header: "Created",
+      id: "createdAt",
+      enableHiding: true,
    },
    {
      accessorKey: "lastRunAt",
      header: "Last Run",
+      id: "lastRunAt",
+      enableHiding: true,
    },
    {
      id: "actions",
@@ -135,8 +148,18 @@ export function DatasetsTable(props: { projectId: string }) {
    };
  };

+  const [columnVisibility, setColumnVisibility] = useColumnVisibility<RowData>(
+    "datasetsColumnVisibility",
+    columns,
+  );
+
  return (
    <div>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+      />
      <DataTable
        columns={columns}
        data={
@@ -161,6 +184,8 @@ export function DatasetsTable(props: { projectId: string }) {
          onChange: setPaginationState,
          state: paginationState,
        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
      />
    </div>
  );
@@ -11,9 +11,10 @@ import {
  FormLabel,
  FormMessage,
 } from "@/src/components/ui/form";
-import { Textarea } from "@/src/components/ui/textarea";
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { JsonEditor } from "@/src/components/json-editor";
+import { type RouterOutput } from "@/src/utils/types";

 const formSchema = z.object({
  input: z.string().refine(
@@ -26,7 +27,10 @@ const formSchema = z.object({
        return false;
      }
    },
-    { message: "Invalid JSON" },
+    {
+      message:
+        "Invalid input. Please provide a JSON object or double-quoted string.",
+    },
  ),
  expectedOutput: z.string().refine(
    (value) => {
@@ -38,18 +42,19 @@ const formSchema = z.object({
        return false;
      }
    },
-    { message: "Invalid JSON" },
+    {
+      message:
+        "Invalid input. Please provide a JSON object or double-quoted string.",
+    },
  ),
 });

 export const EditDatasetItem = ({
  projectId,
-  datasetId,
-  itemId,
+  datasetItem,
 }: {
  projectId: string;
-  datasetId: string;
-  itemId: string;
+  datasetItem: RouterOutput["datasets"]["itemById"];
 }) => {
  const [formError, setFormError] = useState<string | null>(null);
  const [hasChanges, setHasChanges] = useState(false);
@@ -58,25 +63,20 @@ export const EditDatasetItem = ({
    scope: "datasets:CUD",
  });
  const utils = api.useUtils();
-  const item = api.datasets.itemById.useQuery({
-    datasetId,
-    projectId,
-    datasetItemId: itemId,
-  });

  useEffect(() => {
    form.setValue(
      "input",
-      item.data?.input ? JSON.stringify(item.data.input, null, 2) : "",
+      datasetItem?.input ? JSON.stringify(datasetItem.input, null, 2) : "",
    );
    form.setValue(
      "expectedOutput",
-      item.data?.expectedOutput
-        ? JSON.stringify(item.data.expectedOutput, null, 2)
+      datasetItem?.expectedOutput
+        ? JSON.stringify(datasetItem.expectedOutput, null, 2)
        : "",
    );
    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [item.data]);
+  }, [datasetItem]);

  const form = useForm<z.infer<typeof formSchema>>({
    resolver: zodResolver(formSchema),
@@ -92,10 +92,11 @@ export const EditDatasetItem = ({
  });

  function onSubmit(values: z.infer<typeof formSchema>) {
+    if (!!!datasetItem) return;
    updateDatasetItemMutation.mutate({
      projectId: projectId,
-      datasetId: datasetId,
-      datasetItemId: itemId,
+      datasetId: datasetItem.datasetId,
+      datasetItemId: datasetItem.id,
      input: values.input,
      expectedOutput: values.expectedOutput,
    });
@@ -119,10 +120,13 @@ export const EditDatasetItem = ({
                <FormItem>
                  <FormLabel>Input</FormLabel>
                  <FormControl>
-                    <Textarea
-                      {...field}
-                      className="min-h-[200px] font-mono text-xs"
-                      disabled={!hasAccess}
+                    <JsonEditor
+                      defaultValue={field.value}
+                      onChange={(v) => {
+                        setHasChanges(true);
+                        field.onChange(v);
+                      }}
+                      editable={hasAccess}
                    />
                  </FormControl>
                  <FormMessage />
@@ -136,10 +140,13 @@ export const EditDatasetItem = ({
                <FormItem>
                  <FormLabel>Expected output</FormLabel>
                  <FormControl>
-                    <Textarea
-                      {...field}
-                      className="min-h-[200px] font-mono text-xs"
-                      disabled={!hasAccess}
+                    <JsonEditor
+                      defaultValue={field.value}
+                      onChange={(v) => {
+                        setHasChanges(true);
+                        field.onChange(v);
+                      }}
+                      editable={hasAccess}
                    />
                  </FormControl>
                  <FormMessage />
@@ -37,14 +37,15 @@ export const NewDatasetItemButton = (props: {
          New item
        </Button>
      </DialogTrigger>
-      <DialogContent className="sm:max-w-3xl">
+      <DialogContent className="h-[calc(100vh-5rem)] max-h-none w-[calc(100vw-5rem)] max-w-none">
        <DialogHeader>
-          <DialogTitle className="mb-5">Create new dataset item</DialogTitle>
+          <DialogTitle>Create new dataset item</DialogTitle>
        </DialogHeader>
        <NewDatasetItemForm
          projectId={props.projectId}
          datasetId={props.datasetId}
          onFormSuccess={() => setOpen(false)}
+          className="h-full overflow-y-auto"
        />
      </DialogContent>
    </Dialog>
@@ -20,8 +20,9 @@ import {
 import { api } from "@/src/utils/api";
 import { useState } from "react";
 import { usePostHog } from "posthog-js/react";
-import { Textarea } from "@/src/components/ui/textarea";
+import { JsonEditor } from "@/src/components/json-editor";
 import { type Prisma } from "@langfuse/shared/src/db";
+import { cn } from "@/src/utils/tailwind";

 const formSchema = z.object({
  datasetId: z.string().min(1, "Select a dataset"),
@@ -37,7 +38,7 @@ const formSchema = z.object({
    },
    {
      message:
-        "Invalid input. Please provide a JSON object or a string value enclosed in double quotes.",
+        "Invalid input. Please provide a JSON object or double-quoted string.",
    },
  ),
  expectedOutput: z.string().refine(
@@ -52,7 +53,7 @@ const formSchema = z.object({
    },
    {
      message:
-        "Invalid input. Please provide a JSON object or a string value enclosed in double quotes.",
+        "Invalid input. Please provide a JSON object or double-quoted string.",
    },
  ),
 });
@@ -64,6 +65,7 @@ export const NewDatasetItemForm = (props: {
  input?: Prisma.JsonValue;
  output?: Prisma.JsonValue;
  datasetId?: string;
+  className?: string;
  onFormSuccess?: () => void;
 }) => {
  const [formError, setFormError] = useState<string | null>(null);
@@ -113,7 +115,7 @@ export const NewDatasetItemForm = (props: {
      <form
        // eslint-disable-next-line @typescript-eslint/no-misused-promises
        onSubmit={form.handleSubmit(onSubmit)}
-        className="flex flex-col gap-6"
+        className={cn("flex flex-col gap-6", props.className)}
      >
        <FormField
          control={form.control}
@@ -147,9 +149,9 @@ export const NewDatasetItemForm = (props: {
              <FormItem className="flex flex-col gap-2">
                <FormLabel>Input</FormLabel>
                <FormControl>
-                  <Textarea
-                    {...field}
-                    className="min-h-[150px] flex-1 font-mono text-xs"
+                  <JsonEditor
+                    defaultValue={field.value}
+                    onChange={field.onChange}
                  />
                </FormControl>
                <FormMessage />
@@ -163,9 +165,9 @@ export const NewDatasetItemForm = (props: {
              <FormItem className="flex flex-col gap-2">
                <FormLabel>Expected output</FormLabel>
                <FormControl>
-                  <Textarea
-                    {...field}
-                    className="min-h-[150px] flex-1 font-mono text-xs"
+                  <JsonEditor
+                    defaultValue={field.value}
+                    onChange={field.onChange}
                  />
                </FormControl>
                <FormMessage />
@@ -101,9 +101,9 @@ export const NewDatasetItemFromTrace = (props: {
        </Button>
      )}
      <Dialog open={hasAccess && open} onOpenChange={setOpen}>
-        <DialogContent className="sm:w-3xl lg:max-w-none">
+        <DialogContent className="h-[calc(100vh-5rem)] max-h-none w-[calc(100vw-5rem)] max-w-none">
          <DialogHeader>
-            <DialogTitle className="mb-5">Add to dataset</DialogTitle>
+            <DialogTitle>Add to dataset</DialogTitle>
          </DialogHeader>
          <NewDatasetItemForm
            traceId={props.traceId}
@@ -112,6 +112,7 @@ export const NewDatasetItemFromTrace = (props: {
            input={props.input}
            output={props.output}
            onFormSuccess={() => setOpen(false)}
+            className="h-full overflow-y-auto"
          />
        </DialogContent>
      </Dialog>
@@ -560,6 +560,11 @@ export const datasetRouter = createTRPCRouter({
            in: observationIds,
          },
        },
+        select: {
+          id: true,
+          latency: true,
+          calculatedTotalCost: true,
+        },
      });

      const traceIds = runItems
@@ -572,6 +577,10 @@ export const datasetRouter = createTRPCRouter({
          },
          projectId: ctx.session.projectId,
        },
+        select: {
+          id: true,
+          duration: true,
+        },
      });

      const items = runItems.map((ri) => {
@@ -0,0 +1,4 @@
+NOTE: We may transition this feature from our MIT licensed repository to the
+a commercial License (ee folder) once we release a first stable version.
+Please consider this when planning long-term use and integration of this functionality into your projects.
+For more information see https://langfuse.com/docs/open-source
@@ -0,0 +1,120 @@
+import { DataTable } from "@/src/components/table/data-table";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type JobConfiguration } from "@prisma/client";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+
+export type EvalConfigRow = {
+  id: string;
+  createdAt: string;
+  evalTemplateId?: string;
+  scoreName: string;
+  targetObject: string;
+  filter: string;
+  variableMapping: string;
+};
+
+export default function EvalConfigTable({ projectId }: { projectId: string }) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const templates = api.evals.allConfigs.useQuery({
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+    projectId,
+  });
+  const totalCount = templates.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<EvalConfigRow>[] = [
+    {
+      accessorKey: "id",
+      id: "id",
+      header: "ID",
+      enableHiding: false,
+    },
+    {
+      accessorKey: "createdAt",
+      id: "createdAt",
+      header: "Created At",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "evalTemplateId",
+      id: "evalTemplateId",
+      header: "Eval Template",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "scoreName",
+      id: "scoreName",
+      header: "Score Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "targetObject",
+      id: "targetObject",
+      header: "Target",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "filter",
+      id: "filter",
+      header: "Filter",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "variableMapping",
+      id: "variableMapping",
+      header: "Mapping",
+      enableHiding: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<EvalConfigRow>("evalConfigColumnVisibility", columns);
+
+  const convertToTableRow = (jobConfig: JobConfiguration): EvalConfigRow => {
+    return {
+      id: jobConfig.id,
+      createdAt: jobConfig.createdAt.toLocaleString(),
+      evalTemplateId: jobConfig.evalTemplateId?.toLocaleString(),
+      scoreName: jobConfig.scoreName,
+      targetObject: jobConfig.targetObject,
+      filter: JSON.stringify(jobConfig.filter),
+      variableMapping: JSON.stringify(jobConfig.variableMapping),
+    };
+  };
+
+  return (
+    <div>
+      <DataTable
+        columns={columns}
+        data={
+          templates.isLoading
+            ? { isLoading: true, isError: false }
+            : templates.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: templates.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: templates.data.configs.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(totalCount / paginationState.pageSize),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
+  );
+}
@@ -0,0 +1,160 @@
+import * as React from "react";
+import Header from "@/src/components/layouts/header";
+import { EvalTemplateForm } from "@/src/features/evals/components/new-template-form";
+import { PlaygroundProvider } from "@/src/features/playground/client/context";
+import { api } from "@/src/utils/api";
+import { EvalTemplate, evalModels } from "@langfuse/shared";
+import router, { useRouter } from "next/router";
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { Button } from "@/src/components/ui/button";
+import { Pencil } from "lucide-react";
+import { useState } from "react";
+
+export const EvalTemplateDetail = () => {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+  const templateId = router.query.id as string;
+
+  const [isEditing, setIsEditing] = useState(false);
+
+  // get the current template by id
+  const template = api.evals.templateById.useQuery({
+    projectId: projectId,
+    id: templateId,
+  });
+
+  // get all templates for the current template name
+  const allTemplates = api.evals.allTemplatesForName.useQuery(
+    {
+      projectId: projectId,
+      name: template.data?.name ?? "",
+    },
+    {
+      enabled:
+        !template.isLoading &&
+        !template.isError &&
+        template.data?.name !== undefined,
+    },
+  );
+
+  return (
+    <div className="md:container">
+      <Header
+        title={template.data?.name ?? "Loading..."}
+        help={{
+          description:
+            "A scores is an evaluation of a traces or observations. It can be created from user feedback, model-based evaluations, or manual review. See docs to learn more.",
+          href: "https://langfuse.com/docs/scores",
+        }}
+        actionButtons={
+          template.data && (
+            <>
+              {!isEditing && (
+                <UpdateTemplate
+                  projectId={projectId}
+                  isLoading={template.isLoading}
+                  setIsEditing={setIsEditing}
+                />
+              )}
+              <EvalVersionDropdown
+                disabled={allTemplates.isLoading}
+                options={allTemplates.data?.templates ?? []}
+                defaultOption={template.data ?? undefined}
+                onSelect={(template) => {
+                  router.push(
+                    `/project/${projectId}/evals/templates/${template.id}`,
+                  );
+                }}
+              />
+            </>
+          )
+        }
+      />
+      {allTemplates.isLoading || !allTemplates.data ? (
+        <div>Loading...</div>
+      ) : (
+        <PlaygroundProvider avilableModels={[...evalModels]}>
+          <EvalTemplateForm
+            projectId={projectId}
+            existingEvalTemplate={template.data ?? undefined}
+            isEditing={isEditing}
+            setIsEditing={setIsEditing}
+          />
+        </PlaygroundProvider>
+      )}
+    </div>
+  );
+};
+
+export function EvalVersionDropdown(props: {
+  disabled: boolean;
+  options?: EvalTemplate[];
+  defaultOption?: EvalTemplate;
+  onSelect?: (template: EvalTemplate) => void;
+}) {
+  const handleSelect = (value: string) => {
+    const selectedTemplate = props.options?.find(
+      (template) => template.id === value,
+    );
+    if (selectedTemplate && props.onSelect) {
+      props.onSelect(selectedTemplate);
+    }
+  };
+
+  return (
+    <Select
+      disabled={props.disabled}
+      onValueChange={handleSelect}
+      defaultValue={props.defaultOption ? props.defaultOption.id : undefined}
+    >
+      <SelectTrigger className="w-[180px]">
+        <SelectValue placeholder="Version" />
+      </SelectTrigger>
+      <SelectContent>
+        <SelectGroup>
+          {props.options?.map((template) => (
+            <SelectItem key={template.id} value={template.id}>
+              {template.version} - {template.createdAt.toLocaleDateString()}
+            </SelectItem>
+          ))}
+        </SelectGroup>
+      </SelectContent>
+    </Select>
+  );
+}
+
+export function UpdateTemplate({
+  projectId,
+  isLoading,
+  setIsEditing,
+}: {
+  projectId: string;
+  isLoading: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) {
+  const hasAccess = useHasAccess({ projectId, scope: "evalTemplate:create" });
+
+  const handlePromptEdit = () => {
+    setIsEditing(true);
+  };
+
+  return (
+    <Button
+      variant="outline"
+      size="icon"
+      onClick={() => handlePromptEdit()}
+      disabled={!hasAccess}
+      loading={isLoading}
+    >
+      <Pencil className="h-5 w-5" />
+    </Button>
+  );
+}
@@ -0,0 +1,120 @@
+import { DataTable } from "@/src/components/table/data-table";
+import TableLink from "@/src/components/table/table-link";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { RouterOutputs, api } from "@/src/utils/api";
+import { createColumnHelper } from "@tanstack/react-table";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+
+export type EvalsTemplateRow = {
+  name: string;
+  latestCreatedAt?: Date;
+  latestVersion?: number;
+  latestId?: string;
+};
+
+export default function EvalsTemplateTable({
+  projectId,
+}: {
+  projectId: string;
+}) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const templates = api.evals.templateNames.useQuery({
+    projectId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+  const totalCount = templates.data?.totalCount ?? 0;
+
+  const columnHelper = createColumnHelper<EvalsTemplateRow>();
+
+  const columns = [
+    columnHelper.accessor("name", {
+      header: "Name",
+      id: "name",
+      cell: (row) => {
+        const name = row.getValue();
+        const id = row.row.original.latestId;
+
+        if (!id) {
+          return name;
+        }
+
+        return name ? (
+          <TableLink
+            path={`/project/${projectId}/evals/templates/${encodeURIComponent(id)}`}
+            value={name}
+            truncateAt={50}
+          />
+        ) : undefined;
+      },
+    }),
+    columnHelper.accessor("latestCreatedAt", {
+      header: "Last Edit",
+      id: "latestCreatedAt",
+      cell: (row) => {
+        return row.getValue()?.toLocaleDateString();
+      },
+    }),
+    columnHelper.accessor("latestVersion", {
+      header: "Last Version",
+      id: "version",
+      cell: (row) => {
+        return row.getValue();
+      },
+    }),
+  ] as LangfuseColumnDef<EvalsTemplateRow>[];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<EvalsTemplateRow>(
+      "evalTemplatesColumnVisibility",
+      columns,
+    );
+
+  const convertToTableRow = (
+    template: RouterOutputs["evals"]["templateNames"]["templates"][number],
+  ): EvalsTemplateRow => {
+    return {
+      name: template.name,
+      latestCreatedAt: template.latestCreatedAt,
+      latestVersion: template.version,
+      latestId: template.latestId,
+    };
+  };
+
+  return (
+    <div>
+      <DataTable
+        columns={columns}
+        data={
+          templates.isLoading
+            ? { isLoading: true, isError: false }
+            : templates.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: templates.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: templates.data.templates.map((t) =>
+                    convertToTableRow(t),
+                  ),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(totalCount / paginationState.pageSize),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
+  );
+}
@@ -0,0 +1,423 @@
+import { usePostHog } from "posthog-js/react";
+import { useFieldArray, useForm } from "react-hook-form";
+import { Input } from "@/src/components/ui/input";
+import { Button } from "@/src/components/ui/button";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import { tracesTableColsWithOptions, singleFilter } from "@langfuse/shared";
+import { type FilterState } from "@langfuse/shared";
+import * as z from "zod";
+import { Card } from "@/src/components/ui/card";
+import { useEffect, useState } from "react";
+import { api } from "@/src/utils/api";
+import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import {
+  type EvalTemplate,
+  variableMapping,
+  wipVariableMapping,
+  evalObjects,
+} from "@langfuse/shared";
+import router from "next/router";
+
+const formSchema = z.object({
+  evalTemplateId: z.string(),
+  scoreName: z.string(),
+  target: z.string(),
+  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+  mapping: z.array(wipVariableMapping),
+  sampling: z.coerce.number().gte(0).lte(1),
+  delay: z.coerce.number().optional().default(10_000),
+});
+
+export const NewEvalConfigForm = (props: {
+  projectId: string;
+  evalTemplates: EvalTemplate[];
+  onFormSuccess?: () => void;
+}) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const posthog = usePostHog();
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    defaultValues: {
+      evalTemplateId: "",
+      scoreName: undefined,
+      target: "trace",
+      filter: [] as FilterState,
+      mapping: [],
+      sampling: 1,
+      delay: 10_000,
+    },
+  });
+
+  const traceFilterOptions = api.traces.filterOptions.useQuery({
+    projectId: props.projectId,
+    ...form.getFieldState("filter"),
+  });
+
+  const getSelectedEvalTemplate = props.evalTemplates.find(
+    (template) =>
+      `${template.name}-${template.version}` ===
+      form.getValues().evalTemplateId,
+  );
+
+  useEffect(() => {
+    if (getSelectedEvalTemplate) {
+      form.setValue("mapping", []);
+      form.setValue(
+        "mapping",
+        getSelectedEvalTemplate.vars.map((v) => ({
+          templateVariable: v,
+          langfuseObject: "trace" as const,
+        })),
+      );
+    }
+  }, [form, getSelectedEvalTemplate]);
+
+  const { fields } = useFieldArray({
+    control: form.control,
+    name: "mapping",
+  });
+
+  const utils = api.useUtils();
+  const createJobMutation = api.evals.createJob.useMutation({
+    onSuccess: () => utils.models.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    posthog.capture("models:new_template_form");
+    if (!getSelectedEvalTemplate) {
+      setFormError("Please select an eval template");
+      return;
+    }
+
+    // validate wip variable mapping
+    const validatedVarMapping = z.array(variableMapping).parse(values.mapping);
+
+    createJobMutation
+      .mutateAsync({
+        projectId: props.projectId,
+        evalTemplateId: getSelectedEvalTemplate.id,
+        scoreName: values.scoreName,
+        target: values.target,
+        filter: values.filter,
+        mapping: validatedVarMapping,
+        sampling: values.sampling,
+      })
+      .then(() => {
+        props.onFormSuccess?.();
+        form.reset();
+        void router.push(`/project/${props.projectId}/evals/configs/`);
+      })
+      .catch((error) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        if ("message" in error && typeof error.message === "string") {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          setFormError(error.message as string);
+          return;
+        } else {
+          setFormError(JSON.stringify(error));
+          console.error(error);
+        }
+      });
+  }
+
+  return (
+    <Form {...form}>
+      {JSON.stringify(form.watch(), null, 2)}
+      <form
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="flex flex-col gap-4"
+      >
+        <div className="grid gap-4">
+          <FormField
+            control={form.control}
+            name="evalTemplateId"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Eval Template</FormLabel>
+                <Select
+                  defaultValue={field.value}
+                  onValueChange={(value) => {
+                    field.onChange(value);
+                  }}
+                >
+                  <FormControl>
+                    <SelectTrigger>
+                      <SelectValue placeholder="Select a model to run this eval template" />
+                    </SelectTrigger>
+                  </FormControl>
+                  <SelectContent>
+                    {props.evalTemplates.map((template) => (
+                      <SelectItem
+                        value={`${template.name}-${template.version}`}
+                        key={template.id}
+                      >
+                        {`${template.name}-${template.version}`}
+                      </SelectItem>
+                    ))}
+                  </SelectContent>
+                </Select>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          <FormField
+            control={form.control}
+            name="scoreName"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Score Name</FormLabel>
+                <FormControl>
+                  <Input {...field} />
+                </FormControl>
+                <FormDescription>
+                  Optional score name, defaults to ABCDEFG
+                </FormDescription>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="target"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Eval target object</FormLabel>
+                  <FormControl>
+                    <Tabs defaultValue="trace">
+                      <TabsList {...field}>
+                        <TabsTrigger value="trace">Trace</TabsTrigger>
+                        <TabsTrigger value="observation" disabled={true}>
+                          Observation (coming soon)
+                        </TabsTrigger>
+                      </TabsList>
+                    </Tabs>
+                  </FormControl>
+                  <FormDescription>Description</FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="filter"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Target filter</FormLabel>
+                  <FormControl>
+                    <div className="w-1/2">
+                      <InlineFilterBuilder
+                        columns={tracesTableColsWithOptions(
+                          traceFilterOptions.data,
+                        )}
+                        filterState={field.value ?? []}
+                        onChange={(value) => field.onChange(value)}
+                      />
+                    </div>
+                  </FormControl>
+                  <FormDescription>
+                    This will run on all future and XX historical traces.
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+
+          <Card className="p-4">
+            <FormLabel>Variable mapping</FormLabel>
+            <FormControl>Here will some variable mapping be added.</FormControl>
+            <div className="mt-2 flex flex-col gap-2">
+              {fields.map((mappingField, index) => (
+                <div className="flex gap-2" key={index}>
+                  <span className="whitespace-nowrap rounded-md bg-slate-200 px-2 py-1 text-xs	">
+                    {mappingField.templateVariable}
+                  </span>
+                  <FormField
+                    control={form.control}
+                    key={`${mappingField.id}-langfuseObject`}
+                    name={`mapping.${index}.langfuseObject`}
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormControl>
+                          <Select
+                            defaultValue={
+                              evalObjects.find(
+                                (evalObject) => evalObject.id === field.value,
+                              )?.display
+                            }
+                            onValueChange={(value) => {
+                              const obj = evalObjects.find(
+                                (evalObject) => evalObject.display === value,
+                              );
+                              field.onChange(obj?.id);
+                            }}
+                          >
+                            <SelectTrigger>
+                              <SelectValue placeholder="Object type" />
+                            </SelectTrigger>
+                            <SelectContent>
+                              {evalObjects.map((evalObject) => (
+                                <SelectItem
+                                  value={evalObject.display}
+                                  key={evalObject.id}
+                                >
+                                  {evalObject.display}
+                                </SelectItem>
+                              ))}
+                            </SelectContent>
+                          </Select>
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+
+                  {form.watch(`mapping.${index}.langfuseObject`) !== "trace" ? (
+                    <FormField
+                      control={form.control}
+                      key={`${mappingField.id}-objectName`}
+                      name={`mapping.${index}.objectName`}
+                      render={({ field }) => (
+                        <FormItem>
+                          <FormControl>
+                            <Input {...field} value={field.value ?? ""} />
+                          </FormControl>
+                          <FormMessage />
+                        </FormItem>
+                      )}
+                    />
+                  ) : undefined}
+
+                  <FormField
+                    control={form.control}
+                    key={`${mappingField.id}-selectedColumnId`}
+                    name={`mapping.${index}.selectedColumnId`}
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormControl>
+                          <Select
+                            defaultValue={
+                              field.value
+                                ? evalObjects.find(
+                                    (evalObject) =>
+                                      evalObject.id === field.value,
+                                  )?.availableColumns[0].name ?? "N/A"
+                                : "N/A"
+                            }
+                            onValueChange={(value) => {
+                              const availableColumns = evalObjects.find(
+                                (evalObject) =>
+                                  evalObject.id ===
+                                  form.watch(`mapping.${index}.langfuseObject`),
+                              )?.availableColumns;
+                              const column = availableColumns?.find(
+                                (column) => column.name === value,
+                              );
+
+                              field.onChange(column?.id);
+                            }}
+                          >
+                            <SelectTrigger>
+                              <SelectValue placeholder="Object type" />
+                            </SelectTrigger>
+                            <SelectContent>
+                              {evalObjects
+                                .find(
+                                  (evalObject) =>
+                                    evalObject.id ===
+                                    form.watch(
+                                      `mapping.${index}.langfuseObject`,
+                                    ),
+                                )
+                                ?.availableColumns.map((column) => (
+                                  <SelectItem
+                                    value={column.name}
+                                    key={column.id}
+                                  >
+                                    {column.name}
+                                  </SelectItem>
+                                ))}
+                            </SelectContent>
+                          </Select>
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+                </div>
+              ))}
+            </div>
+            <FormDescription>Description </FormDescription>
+            <FormMessage />
+          </Card>
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="sampling"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Sampling</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="delay"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Delay (ms)</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+        </div>
+
+        <Button
+          type="submit"
+          loading={createJobMutation.isLoading}
+          className="mt-3"
+        >
+          Save
+        </Button>
+      </form>
+      {formError ? (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      ) : null}
+    </Form>
+  );
+};
@@ -0,0 +1,282 @@
+import { usePostHog } from "posthog-js/react";
+import { useEffect, useState } from "react";
+import { useForm } from "react-hook-form";
+import * as z from "zod";
+import { Input } from "@/src/components/ui/input";
+import { Button } from "@/src/components/ui/button";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { Textarea } from "@/src/components/ui/textarea";
+import { api } from "@/src/utils/api";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
+import router from "next/router";
+import { type EvalTemplate } from "@prisma/client";
+import { usePlaygroundContext } from "@/src/features/playground/client/context";
+import { ModelParameters } from "@/src/features/playground/client/components/ModelParameters";
+import { EvalModelNames, OutputSchema, evalModels } from "@langfuse/shared";
+import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
+import { AlphaNumericUnderscoreString } from "@/src/utils/zod";
+
+const formSchema = z.object({
+  name: AlphaNumericUnderscoreString,
+  prompt: z
+    .string()
+    .min(1, "Enter a prompt")
+    .refine((val) => {
+      const variables = extractVariables(val);
+      const matches = variables.map((variable) => {
+        // check regex here
+        if (variable.match(/^[A-Za-z_]+$/)) {
+          return true;
+        }
+        return false;
+      });
+      return !matches.includes(false);
+    }, "Variables must only contain letters and underscores (_)"),
+
+  variables: z.array(
+    z.string().min(1, "Variables must have at least one character"),
+  ),
+  model: EvalModelNames,
+  outputScore: z.string(),
+  outputReasoning: z.string(),
+});
+
+export const EvalTemplateForm = (props: {
+  projectId: string;
+  existingEvalTemplate?: EvalTemplate;
+  onFormSuccess?: () => void;
+  isEditing?: boolean;
+  setIsEditing?: (isEditing: boolean) => void;
+}) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const playgroundContext = usePlaygroundContext();
+
+  const posthog = usePostHog();
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    disabled: !props.isEditing,
+    defaultValues: {
+      name: props.existingEvalTemplate?.name ?? "",
+      model: EvalModelNames.parse(
+        props.existingEvalTemplate?.model ?? "gpt-3.5-turbo",
+      ),
+      prompt: props.existingEvalTemplate?.prompt ?? undefined,
+      variables: props.existingEvalTemplate?.vars ?? [],
+      outputReasoning: props.existingEvalTemplate
+        ? OutputSchema.parse(props.existingEvalTemplate?.outputSchema).reasoning
+        : undefined,
+      outputScore: props.existingEvalTemplate
+        ? OutputSchema.parse(props.existingEvalTemplate?.outputSchema).score
+        : undefined,
+    },
+  });
+
+  // reset the form if the input template changes
+  useEffect(() => {
+    if (props.existingEvalTemplate) {
+      const model = EvalModelNames.parse(props.existingEvalTemplate.model);
+
+      form.reset({
+        name: props.existingEvalTemplate.name,
+        model: model,
+        prompt: props.existingEvalTemplate.prompt,
+        variables: props.existingEvalTemplate.vars,
+        outputReasoning: OutputSchema.parse(
+          props.existingEvalTemplate.outputSchema,
+        ).reasoning,
+        outputScore: OutputSchema.parse(props.existingEvalTemplate.outputSchema)
+          .score,
+      });
+
+      // also set the context for the playground
+      playgroundContext.updateModelParam("model", model);
+      playgroundContext.updateModelParams(
+        props.existingEvalTemplate.modelParams,
+      );
+
+      const modelProvider = evalModels.find((m) => m.model === model)?.provider;
+      if (modelProvider) {
+        playgroundContext.updateModelParam("provider", modelProvider);
+      }
+    }
+  }, [props.existingEvalTemplate, form]);
+
+  const extractedVariables = form.watch("prompt")
+    ? extractVariables(form.watch("prompt")).filter(getIsCharOrUnderscore)
+    : undefined;
+
+  const utils = api.useUtils();
+  const createEvalTemplateMutation = api.evals.createTemplate.useMutation({
+    onSuccess: () => utils.models.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    console.log("submitting", values);
+    posthog.capture("models:new_template_form");
+
+    createEvalTemplateMutation
+      .mutateAsync({
+        name: values.name,
+        projectId: props.projectId,
+        prompt: values.prompt,
+        model: EvalModelNames.parse(playgroundContext.modelParams.model),
+        modelParameters: playgroundContext.modelParams,
+        variables: extractedVariables ?? [],
+        outputSchema: {
+          score: values.outputScore,
+          reasoning: values.outputReasoning,
+        },
+      })
+      .then((res) => {
+        props.onFormSuccess?.();
+        form.reset();
+        props.setIsEditing?.(false);
+        void router.push(
+          `/project/${props.projectId}/evals/templates/${res.id}`,
+        );
+      })
+      .catch((error) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        if ("message" in error && typeof error.message === "string") {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          setFormError(error.message as string);
+          return;
+        } else {
+          setFormError(JSON.stringify(error));
+          console.error(error);
+        }
+      });
+  }
+
+  return (
+    <Form {...form}>
+      <form
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="grid grid-cols-1 gap-6 gap-x-12 lg:grid-cols-3"
+      >
+        {!props.existingEvalTemplate ? (
+          <>
+            <div className="col-span-1 row-span-1 lg:col-span-2">
+              <FormField
+                control={form.control}
+                name="name"
+                render={({ field }) => (
+                  <>
+                    <FormItem>
+                      <FormLabel>Name</FormLabel>
+                      <FormControl>
+                        <Input
+                          {...field}
+                          placeholder="Select a template name"
+                        />
+                      </FormControl>
+                      <FormMessage />
+                    </FormItem>
+                  </>
+                )}
+              />
+            </div>
+            <div className="lg:col-span-0 col-span-1 row-span-1"></div>
+          </>
+        ) : undefined}
+
+        <div className="col-span-1 flex flex-col gap-6 lg:col-span-2">
+          <FormField
+            control={form.control}
+            name="prompt"
+            render={({ field }) => (
+              <>
+                <FormItem>
+                  <FormLabel>Prompt</FormLabel>
+                  <FormControl>
+                    <Textarea
+                      {...field}
+                      placeholder="{{input}} Please evaluate the input on toxicity."
+                      className="min-h-[150px] flex-1 font-mono text-xs"
+                    />
+                  </FormControl>
+                  <FormMessage />
+                  <PromptDescription
+                    currentExtractedVariables={extractedVariables ?? []}
+                  />
+                </FormItem>
+              </>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name="outputScore"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Score</FormLabel>
+                <FormControl>
+                  <Input {...field} placeholder="Score between 0 and 1" />
+                </FormControl>
+                <FormDescription>
+                  We use function calls to extract data from the LLM. Specify
+                  what the LLM should return for the score.
+                </FormDescription>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          <FormField
+            control={form.control}
+            name="outputReasoning"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Reasoning</FormLabel>
+                <FormControl>
+                  <Input
+                    placeholder="One sentence reasoning for the score"
+                    {...field}
+                  />
+                </FormControl>
+                <FormDescription>
+                  We use function calls to extract data from the LLM. Specify
+                  what the LLM should return for the reasoning.
+                </FormDescription>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+        </div>
+        <div className="col-span-1 row-span-3">
+          <ModelParameters
+            {...playgroundContext}
+            availableModels={[...evalModels]}
+            disabled={!props.isEditing}
+          />
+        </div>
+
+        {props.isEditing && (
+          <Button
+            type="submit"
+            loading={createEvalTemplateMutation.isLoading}
+            className="col-span-1 mt-3 lg:col-span-3"
+          >
+            Save
+          </Button>
+        )}
+      </form>
+      {formError ? (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      ) : null}
+    </Form>
+  );
+};
@@ -0,0 +1,290 @@
+import { z } from "zod";
+
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { DEFAULT_TRACE_JOB_DELAY, EvalTargetObject } from "@langfuse/shared";
+import {
+  EvalModelNames,
+  ZodModelConfig,
+  singleFilter,
+  variableMapping,
+} from "@langfuse/shared";
+import { AlphaNumericUnderscoreString } from "@/src/utils/zod";
+
+export const CreateEvalTemplate = z.object({
+  name: AlphaNumericUnderscoreString,
+  projectId: z.string(),
+  prompt: z.string(),
+  model: EvalModelNames,
+  modelParameters: ZodModelConfig,
+  variables: z.array(z.string()),
+  outputSchema: z.object({
+    score: z.string(),
+    reasoning: z.string(),
+  }),
+});
+
+export const evalRouter = createTRPCRouter({
+  allConfigs: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        limit: z.number(),
+        page: z.number(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "job:read",
+      });
+
+      const configs = await ctx.prisma.jobConfiguration.findMany({
+        where: {
+          projectId: input.projectId,
+          jobType: "EVAL",
+        },
+        take: input.limit,
+        skip: input.page * input.limit,
+      });
+
+      const count = await ctx.prisma.jobConfiguration.count({
+        where: {
+          projectId: input.projectId,
+          jobType: "EVAL",
+        },
+      });
+      return {
+        configs: configs,
+        totalCount: count,
+      };
+    }),
+
+  allTemplatesForName: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:read",
+      });
+
+      const templates = await ctx.prisma.evalTemplate.findMany({
+        where: {
+          projectId: input.projectId,
+          name: input.name,
+        },
+        orderBy: [{ version: "desc" }],
+      });
+
+      return {
+        templates: templates,
+      };
+    }),
+
+  templateNames: protectedProjectProcedure
+    .input(
+      z.object({ projectId: z.string(), page: z.number(), limit: z.number() }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:read",
+      });
+
+      const templates = await ctx.prisma.$queryRaw<
+        Array<{
+          name: string;
+          version: number;
+          latestCreatedAt: Date;
+          latestId: string;
+        }>
+      >`
+        SELECT
+          name,
+          MAX(version) as version,
+          MAX(created_at) as "latestCreatedAt",
+          (SELECT id FROM "eval_templates" WHERE "project_id" = ${input.projectId} AND name = et.name ORDER BY version DESC LIMIT 1) as "latestId"
+        FROM "eval_templates" as et
+        WHERE "project_id" = ${input.projectId}
+        GROUP BY name
+        ORDER BY name
+        LIMIT ${input.limit}
+        OFFSET ${input.page * input.limit}
+      `;
+      return {
+        templates: templates,
+        totalCount: templates.length,
+      };
+    }),
+
+  templateById: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        id: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:read",
+      });
+
+      const template = await ctx.prisma.evalTemplate.findUnique({
+        where: {
+          id: input.id,
+          projectId: input.projectId,
+        },
+      });
+
+      return template;
+    }),
+  allTemplates: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        id: z.string().optional(),
+        limit: z.number().optional(),
+        page: z.number().optional(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:read",
+      });
+
+      const templates = await ctx.prisma.evalTemplate.findMany({
+        where: {
+          projectId: input.projectId,
+          ...(input.id ? { id: input.id } : undefined),
+        },
+        ...(input.limit && input.page
+          ? { take: input.limit, skip: input.page * input.limit }
+          : undefined),
+      });
+
+      const count = await ctx.prisma.evalTemplate.count({
+        where: {
+          projectId: input.projectId,
+          ...(input.id ? { id: input.id } : undefined),
+        },
+      });
+      return {
+        templates: templates,
+        totalCount: count,
+      };
+    }),
+
+  createJob: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        evalTemplateId: z.string(),
+        scoreName: AlphaNumericUnderscoreString,
+        target: z.string(),
+        filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+        mapping: z.array(variableMapping),
+        sampling: z.number().gte(0).lte(1),
+        delay: z.number().gte(0).default(10_000),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "job:create",
+        });
+
+        const evalTemplate = await ctx.prisma.evalTemplate.findUnique({
+          where: {
+            id: input.evalTemplateId,
+            projectId: input.projectId,
+          },
+        });
+
+        if (!evalTemplate) {
+          console.log(
+            `Template not found for project ${input.projectId} and id ${input.evalTemplateId}`,
+          );
+          throw new Error("Template not found");
+        }
+
+        const job = await ctx.prisma.jobConfiguration.create({
+          data: {
+            projectId: input.projectId,
+            jobType: "EVAL",
+            evalTemplateId: input.evalTemplateId,
+            scoreName: input.scoreName,
+            targetObject: EvalTargetObject.Trace,
+            filter: input.filter ?? [],
+            variableMapping: input.mapping,
+            sampling: input.sampling,
+            delay: DEFAULT_TRACE_JOB_DELAY, // 10 seconds default
+          },
+        });
+        await auditLog({
+          session: ctx.session,
+          resourceType: "job",
+          resourceId: job.id,
+          action: "create",
+        });
+      } catch (e) {
+        console.log(e);
+        throw e;
+      }
+    }),
+  createTemplate: protectedProjectProcedure
+    .input(CreateEvalTemplate)
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:create",
+      });
+
+      const latestTemplate = await ctx.prisma.evalTemplate.findFirst({
+        where: {
+          projectId: input.projectId,
+          name: input.name,
+        },
+        orderBy: [{ version: "desc" }],
+      });
+
+      const evalTemplate = await ctx.prisma.evalTemplate.create({
+        data: {
+          version: latestTemplate?.version ? latestTemplate.version + 1 : 1,
+          name: input.name,
+          projectId: input.projectId,
+          prompt: input.prompt,
+          model: input.model,
+          modelParams: input.modelParameters,
+          vars: input.variables,
+          outputSchema: input.outputSchema,
+        },
+      });
+
+      await auditLog({
+        session: ctx.session,
+        resourceType: "evalTemplate",
+        resourceId: evalTemplate.id,
+        action: "create",
+      });
+      return evalTemplate;
+    }),
+});
@@ -1 +1 @@
-export const availableFlags = ["templateFlag", "playground"] as const;
+export const availableFlags = ["templateFlag", "evals", "playground"] as const;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Marc Klingen	5edf3dd3d9	chore: release v2.21.2 CI/CD / push-docker-image (push) Blocked by required conditions Details CI/CD / tests-web (20) (push) Failing after 45s Details CI/CD / tests-worker (20) (push) Failing after 51s Details CI/CD / test-docker-build (push) Failing after 1m42s Details CI/CD / e2e-tests (push) Failing after 33s Details CI/CD / lint (push) Successful in 3m48s Details CI/CD / all-ci-passed (push) Waiting to run Details release.yml / release (push) Failing after 3s Details	2024-04-11 00:27:40 +02:00
Marc KlingenandGitHub	135650382c	fix(cloud): signup blocked due to turnstile (#1667 )	2024-04-11 00:26:06 +02:00
Marc KlingenandGitHub	0cbab9966a	feat(security): add `AUTH_DISABLE_SIGNUP` env to disable all new signups (#1666 )	2024-04-11 00:16:20 +02:00
Max DeichmannandGitHub	75d020cc2c	fix: eval template ui fixes (#1662 )	2024-04-10 16:49:55 +00:00
Max DeichmannandGitHub	02361ea10c	feat: templates evals UI (#1654 )	2024-04-10 16:30:51 +00:00
Hassieb PakzadandGitHub	b75968024c	perf: increase trpc route timeout to support larger exports (#1657 )	2024-04-10 18:25:21 +02:00
Marc Klingen	3eb4974e11	chore: release v2.21.1 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests-web (20) (push) Waiting to run Details CI/CD / tests-worker (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-10 17:52:03 +02:00
Marc KlingenandGitHub	49c556d83c	feat: enable up/down nav on traces/datasetitems when coming from dataset run items (#1661 ) * feat: enable up/down nav for traces/datasetItems coming from list of runitems * fix	2024-04-10 15:43:58 +00:00
Marc KlingenandGitHub	e938b33bd4	fix: prompt var validation in form should not block (#1659 )	2024-04-10 15:25:24 +00:00
Marc Klingen	3c6b2e1ba1	chore: release v2.21.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests-web (20) (push) Waiting to run Details CI/CD / tests-worker (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-10 14:50:19 +02:00
Marc KlingenandGitHub	c66f16d6d3	feat(prompts): use codemirror to edit prompt config (#1644 )	2024-04-10 12:45:03 +00:00
Marc KlingenandGitHub	39dd4f3b3d	feat(models): use codemirror to edit tokenizer config (#1645 )	2024-04-10 14:40:20 +02:00
Marc KlingenandGitHub	245321da11	feat(datasets): add codemirror as json editor (#1643 )	2024-04-10 14:29:18 +02:00
Marc KlingenandGitHub	bb4a3ab532	perf: no `refetchOnMount` for I/O previews in tables (#1656 )	2024-04-10 11:37:39 +00:00
Marc KlingenandGitHub	25080bcf73	perf: reduce response size of runitems trpc route (#1652 )	2024-04-10 07:34:17 +00:00
Marc KlingenandGitHub	2760bf0a39	feat(datasets): link to source trace/observation in ui (#1651 )	2024-04-10 07:25:45 +00:00
Marc KlingenandGitHub	c549fd6fb9	fix(ui): sizing of dataset item forms (#1650 )	2024-04-10 07:01:37 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	d5c892a0ba	chore(deps-dev): bump tsc-watch from 6.0.4 to 6.2.0 (#1647 ) Bumps [tsc-watch](https://github.com/gilamran/tsc-watch) from 6.0.4 to 6.2.0. - [Release notes](https://github.com/gilamran/tsc-watch/releases) - [Changelog](https://github.com/gilamran/tsc-watch/blob/master/CHANGELOG.md) - [Commits](https://github.com/gilamran/tsc-watch/compare/v6.0.4...v6.2.0) --- updated-dependencies: - dependency-name: tsc-watch dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-09 23:37:56 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	fba5a68005	chore(deps): bump the patches group with 7 updates (#1646 ) Bumps the patches group with 7 updates: \| Package \| From \| To \| \| --- \| --- \| --- \| \| [@langchain/anthropic](https://github.com/langchain-ai/langchainjs) \| `0.1.10` \| `0.1.12` \| \| [@langchain/core](https://github.com/langchain-ai/langchainjs) \| `0.1.54` \| `0.1.55` \| \| [ai](https://github.com/vercel/ai) \| `3.0.19` \| `3.0.20` \| \| [langchain](https://github.com/langchain-ai/langchainjs) \| `0.1.31` \| `0.1.32` \| \| [@types/react](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/react) \| `18.2.74` \| `18.2.75` \| \| [crisp-sdk-web](https://github.com/crisp-im/crisp-sdk-web) \| `1.0.22` \| `1.0.23` \| \| [@types/pg](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/pg) \| `8.11.4` \| `8.11.5` \| Updates `@langchain/anthropic` from 0.1.10 to 0.1.12 - [Release notes](https://github.com/langchain-ai/langchainjs/releases) - [Changelog](https://github.com/langchain-ai/langchainjs/blob/main/release_workspace.js) - [Commits](https://github.com/langchain-ai/langchainjs/compare/0.1.10...0.1.12) Updates `@langchain/core` from 0.1.54 to 0.1.55 - [Release notes](https://github.com/langchain-ai/langchainjs/releases) - [Changelog](https://github.com/langchain-ai/langchainjs/blob/main/release_workspace.js) - [Commits](https://github.com/langchain-ai/langchainjs/commits) Updates `ai` from 3.0.19 to 3.0.20 - [Release notes](https://github.com/vercel/ai/releases) - [Commits](https://github.com/vercel/ai/compare/ai@3.0.19...ai@3.0.20) Updates `langchain` from 0.1.31 to 0.1.32 - [Release notes](https://github.com/langchain-ai/langchainjs/releases) - [Changelog](https://github.com/langchain-ai/langchainjs/blob/main/release_workspace.js) - [Commits](https://github.com/langchain-ai/langchainjs/compare/0.1.31...0.1.32) Updates `@types/react` from 18.2.74 to 18.2.75 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/react) Updates `crisp-sdk-web` from 1.0.22 to 1.0.23 - [Commits](https://github.com/crisp-im/crisp-sdk-web/compare/v1.0.22...v1.0.23) Updates `@types/pg` from 8.11.4 to 8.11.5 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/pg) --- updated-dependencies: - dependency-name: "@langchain/anthropic" dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: "@langchain/core" dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: ai dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: langchain dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: "@types/react" dependency-type: direct:development update-type: version-update:semver-patch dependency-group: patches - dependency-name: crisp-sdk-web dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: "@types/pg" dependency-type: direct:development update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-10 01:31:03 +02:00
Marc KlingenandGitHub	3227cca756	feat(api): add `failIfNoEventsInLastMinute` flag to `/health` (#1642 )	2024-04-09 18:13:11 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>Richard Krümmel Marc Klingen	fefaa3a63b	chore(deps): bump cmdk from 0.2.1 to 1.0.0 (#1623 ) * chore(deps): bump cmdk from 0.2.1 to 1.0.0 Bumps cmdk from 0.2.1 to 1.0.0. --- updated-dependencies: - dependency-name: cmdk dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> * fix --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Richard Krümmel <61158193+RichardKruemmel@users.noreply.github.com> Co-authored-by: Marc Klingen <git@marcklingen.com>	2024-04-09 17:51:16 +00:00
Max DeichmannandGitHub	8ee3ae91ec	feat: use playground model params for evals (#1639 )	2024-04-09 16:34:33 +00:00
Marc KlingenandGitHub	c3bdf51972	feat(datasets): show i/o and expected output on runitems (#1641 )	2024-04-09 18:28:47 +02:00
Marc KlingenandGitHub	843de9cf31	chore: update .env.prod.example	2024-04-09 18:08:16 +02:00
Hassieb PakzadandGitHub	5839cc5c87	fix(ui): prompt variables must be only letters or underscore (#1635 ) * fix(ui): remove invalid variables from prompt * feat: add note in new prompt form on variable format	2024-04-09 14:30:58 +00:00
Marc KlingenandGitHub	2fc8dcc855	feat(ui): columns visibility on all dataset tables (#1640 )	2024-04-09 14:23:38 +00:00
Marc Klingen	8c2d5262e1	chore: release v2.20.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests-web (20) (push) Waiting to run Details CI/CD / tests-worker (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-09 15:35:23 +02:00
Richard Krümmel GitHub Marc Klingen	6bc2e2aa7a	feat(prompts): add tags, filtering, sorting, pagination (#1557 ) --------- Co-authored-by: Marc Klingen <git@marcklingen.com>	2024-04-09 13:32:48 +00:00
Marc KlingenandGitHub	4c07776e20	fix: imports from shared package (#1629 )	2024-04-09 13:22:19 +00:00
Marc KlingenandGitHub	bd98d20f65	feat(api): add runName and datasetName references to all dataset apis (#1637 )	2024-04-09 13:12:21 +00:00
Max DeichmannandGitHub	fe74c14833	feat: add worker rate limits (#1636 )	2024-04-09 14:12:09 +02:00
Max DeichmannandGitHub	a89afd47c7	feat: evals queue logic (#1615 )	2024-04-09 13:48:23 +02:00
Marc KlingenandGitHub	fc453e1c5f	fix(cloud): disable sign-in until captcha passed (#1634 )	2024-04-09 09:31:26 +00:00
Richard KrümmelandGitHub	55cf92e540	feat(dashboard): tag filter (#1573 )	2024-04-09 10:52:58 +02:00
Hassieb PakzadandGitHub	de30c90b53	fix: router.query undefined on first render (#1621 )	2024-04-09 07:49:40 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	a80f7ee63d	chore(deps): bump @aws-sdk/client-s3 from 3.536.0 to 3.550.0 (#1626 ) Bumps [@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-s3) from 3.536.0 to 3.550.0. - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.550.0/clients/client-s3) --- updated-dependencies: - dependency-name: "@aws-sdk/client-s3" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-08 23:40:08 +00:00
Marc Klingen	ff3975bb31	chore: release v2.19.1 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-09 01:26:01 +02:00
Marc KlingenandGitHub	919a4659b2	fix(ui): dataset items table import (#1628 ) * fix(ui): dataset items table * push	2024-04-08 23:21:59 +00:00
Marc Klingen	9bb84878e3	chore: release v2.19.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-08 23:08:11 +02:00
Thomas CalvetandGitHub	c7aec93a21	feat(api): expose promptId via GET /observations (#1619 )	2024-04-08 23:01:25 +02:00
Richard Krümmel GitHub Marc Klingen	b0dbae98e8	chore: add docs to nav and overflow of score comment (#1620 ) * Add Docs link to bottom navigation * Update HoverCardContent styling * open docs in new tab * break score comments --------- Co-authored-by: Marc Klingen <git@marcklingen.com>	2024-04-08 20:55:10 +00:00
Richard Krümmel GitHub Max Deichmann	65e6cc819d	feat(prompts): improve type safety by using columnHelper (#1586 ) Co-authored-by: Max Deichmann <m.deichmann@tum.de>	2024-04-08 20:26:59 +00:00
Marc Klingen	f307b298c2	chore: remove codeowners	2024-04-08 22:16:01 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	78c4774b1e	chore(deps): bump @aws-sdk/lib-storage from 3.540.0 to 3.550.0 (#1622 ) Bumps [@aws-sdk/lib-storage](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/lib/lib-storage) from 3.540.0 to 3.550.0. - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/lib/lib-storage/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.550.0/lib/lib-storage) --- updated-dependencies: - dependency-name: "@aws-sdk/lib-storage" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-08 20:13:49 +00:00
Marc Klingen	b26130df64	chore: release v2.18.1 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-08 22:07:15 +02:00
Max DeichmannandGitHub	806b203d66	feat: add worker health check (#1624 )	2024-04-08 21:41:00 +02:00
Marc KlingenandGitHub	0eead0360e	fix(prompts): render all prompt types correctly and fallback on json (#1625 )	2024-04-08 21:20:15 +02:00
Marc Klingen	aa1a964c6b	chore: use ci to rebase dependabot PRs selectively	2024-04-08 20:26:34 +02:00
Max Deichmann	6f108da941	chore: release v2.18.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-08 20:12:29 +02:00
Max DeichmannandGitHub	f008a31a3d	feat: add eval basic function (#1455 )	2024-04-08 17:06:45 +02:00
Marc Klingen	0cea7a5168	feat(ui): add user.email to userNav menu	2024-04-08 16:51:16 +02:00
Marc KlingenandGitHub	1321b775ef	chore: update codeowners	2024-04-08 14:28:59 +02:00
Max Deichmann	9d698a95c5	chore: release v2.17.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests (20) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-04-08 11:31:41 +02:00
Max DeichmannandGitHub	baac382fa9	fix: fix version number (#1616 )	2024-04-08 11:31:06 +02:00
Hassieb PakzadandGitHub	c92e9984f1	feat: adds chat prompts (#1508 )	2024-04-08 10:45:37 +02:00
Max DeichmannandGitHub	7fac267135	feat: add worker to cd pipeline (#1609 )	2024-04-07 12:16:50 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	9e3653fa41	chore(deps): bump ai from 3.0.18 to 3.0.19 in the patches group (#1605 ) Bumps the patches group with 1 update: [ai](https://github.com/vercel/ai). Updates `ai` from 3.0.18 to 3.0.19 - [Release notes](https://github.com/vercel/ai/releases) - [Commits](https://github.com/vercel/ai/compare/ai@3.0.18...ai@3.0.19) --- updated-dependencies: - dependency-name: ai dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-07 09:46:00 +00:00
Max DeichmannandGitHub	575148f8de	refactor: move release to root (#1602 )	2024-04-06 13:13:27 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	06edd7b07b	chore(deps-dev): bump @playwright/test from 1.42.1 to 1.43.0 (#1607 ) Bumps [@playwright/test](https://github.com/microsoft/playwright) from 1.42.1 to 1.43.0. - [Release notes](https://github.com/microsoft/playwright/releases) - [Commits](https://github.com/microsoft/playwright/compare/v1.42.1...v1.43.0) --- updated-dependencies: - dependency-name: "@playwright/test" dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-06 13:07:04 +02:00