chore: release v2.19.0

* Add Docs link to bottom navigation

* Update HoverCardContent styling

* open docs in new tab

* break score comments

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

.env.dev.example

@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""
 
 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret

.env.local.example

@@ -16,4 +16,11 @@ SALT="salt"
 
 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword

.github/CODEOWNERS

@@ -1 +1,2 @@
-* @langfuse/founders
+# Currently inactive
+# * @langfuse/maintainers

.github/dependabot.yml

@@ -7,6 +7,7 @@ version: 2
 updates:
   - package-ecosystem: npm
     directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
     schedule:
       interval: "daily"
     versioning-strategy: "increase"

.github/workflows/pipeline.yml

@@ -193,7 +193,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
         uses: docker/metadata-action@v4
         with:
           images: |
@@ -206,11 +206,34 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
           context: .
           file: ./web/Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ github.repository }}-worker
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}

CONTRIBUTING.md

@@ -192,7 +192,7 @@ Requirements
   To run migrations, you can execute the following command.
 
   ```bash
-  pnpm --filter=shared run db:migrate
+  pnpm run db:migrate -- --name <name of the migration>
   ```
 
 > [!NOTE]

docker-compose.build.yml

@@ -18,9 +18,8 @@ services:
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   worker:
@@ -30,7 +29,6 @@ services:
     depends_on:
       - db
       - redis
-      - server
     ports:
       - "3030:3030"
     environment:
@@ -39,9 +37,10 @@ services:
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
       - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
       - REDIS_PORT=${REDIS_PORT:-6379}
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   redis:

fern/apis/server/definition/prompts.yml

@@ -25,6 +25,20 @@ service:
 
 types:
   CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
     properties:
       name: string
       isActive:
@@ -34,8 +48,29 @@ types:
       config: optional<unknown>
 
   Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
     properties:
       name: string
       version: integer
-      prompt: string
       config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>

generated/postman/collection.json

@@ -628,7 +628,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"

package.json

@@ -1,8 +1,9 @@
 {
   "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.19.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
+  "private": true,
   "engines": {
     "node": ">=20"
   },
@@ -21,11 +22,51 @@
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate"
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
   },
   "devDependencies": {
+    "@release-it/bumper": "^6.0.1",
     "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
+    "release-it": "^17.1.1",
     "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
   }
 }

packages/shared/package.json

@@ -5,12 +5,23 @@
   "private": true,
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
   "engines": {
     "node": ">=20"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    }
+  },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
@@ -37,8 +48,8 @@
     "kysely": "^0.27.3",
     "langchain": "^0.1.31",
     "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
     "@repo/eslint-config": "*",
@@ -57,9 +68,9 @@
     "prettier": "^3.2.5",
     "prisma": "^5.12.1",
     "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
     "typescript": "^5.4.4",
     "vitest": "^1.3.1"
   }

packages/shared/prisma/generated/types.ts

@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
     API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,17 @@ export const DatasetStatus = {
     ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
     id: string;
     user_id: string;
@@ -129,6 +141,19 @@ export type DatasetRuns = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -139,6 +164,33 @@ export type Events = {
     url: string | null;
     method: string | null;
 };
+export type JobConfiguration = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
+};
 export type Membership = {
     project_id: string;
     user_id: string;
@@ -253,9 +305,10 @@ export type Prompt = {
     updated_at: Generated<Timestamp>;
     project_id: string;
     created_by: string;
-    prompt: string;
+    prompt: unknown;
     name: string;
     version: number;
+    type: Generated<string>;
     is_active: boolean;
     config: Generated<unknown>;
 };
@@ -344,7 +397,10 @@ export type DB = {
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
     datasets: Dataset;
+    eval_templates: EvalTemplate;
     events: Events;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
     membership_invitations: MembershipInvitation;
     memberships: Membership;
     models: Model;

packages/shared/prisma/migrations/20240405124810_prompt_to_json/migration.sql

@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;

packages/shared/prisma/migrations/20240408133037_add_objects_for_evals/migration.sql

@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -92,22 +92,25 @@ model VerificationToken {
 }
 
 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id               String                 @id @default(cuid())
+    createdAt        DateTime               @default(now()) @map("created_at")
+    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
+    name             String
+    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members          Membership[]
+    traces           Trace[]
+    observations     Observation[]
+    apiKeys          ApiKey[]
+    dataset          Dataset[]
+    RawEvents        Events[]
+    invitations      MembershipInvitation[]
+    sessions         TraceSession[]
+    Prompt           Prompt[]
+    Model            Model[]
+    AuditLog         AuditLog[]
+    EvalTemplate     EvalTemplate[]
+    JobConfiguration JobConfiguration[]
+    JobExecution     JobExecution[]
 
     @@map("projects")
 }
@@ -209,6 +212,7 @@ model Trace {
     scores          Score[]
     DatasetRunItems DatasetRunItems[]
     DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -362,16 +366,17 @@ enum ObservationLevel {
 }
 
 model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
     name          String
     value         Float
     source        ScoreSource
     comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]
 
     @@unique([id, traceId]) // used for upsert
     @@index(timestamp)
@@ -385,6 +390,7 @@ model Score {
 enum ScoreSource {
     API
     REVIEW
+    EVAL
 }
 
 enum PricingUnit {
@@ -521,9 +527,10 @@ model Prompt {
 
     createdBy String @map("created_by")
 
-    prompt      String
+    prompt      Json
     name        String
     version     Int
+    type        String        @default("text")
     isActive    Boolean       @map("is_active")
     config      Json          @default("{}")
     Observation Observation[]
@@ -579,3 +586,89 @@ model AuditLog {
     @@index([createdAt])
     @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType       @map("job_type")
+    evalTemplateId String?       @map("eval_template_id")
+    evalTemplate   EvalTemplate? @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}

packages/shared/prisma/seed.ts

@@ -4,13 +4,14 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
 
 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
 
 const LOAD_TRACE_VOLUME = 10_000;
 

packages/shared/src/db.ts

@@ -3,6 +3,14 @@
 
 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";
 
 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
@@ -22,6 +30,23 @@ declare global {
 }
 
 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+
+export const kyselyPrisma = prisma.$extends(
+  kyselyExtension({
+    kysely: (driver) =>
+      new Kysely<DB>({
+        dialect: {
+          // This is where the magic happens!
+          createDriver: () => driver,
+          // Don't forget to customize these to match your database!
+          createAdapter: () => new PostgresAdapter(),
+          createIntrospector: (db) => new PostgresIntrospector(db),
+          createQueryCompiler: () => new PostgresQueryCompiler(),
+        },
+      }),
+  })
+);
+
 export * from "@prisma/client";
 
 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;

packages/shared/src/features/evals/types.ts

@@ -0,0 +1,31 @@
+import z from "zod";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(),
+    objectName: z.string().nullish(), // can be null as this is only required for langfuseObjects other than trace
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});

packages/shared/src/filterToPrisma.ts

@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
   type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";
 
 const operatorReplacements = {
   "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
     const col = tableColumns.find(
       (c) =>
         // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
     );
     if (!col) {
       console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
         ? Prisma.raw(
             arrayOperatorReplacements[
               filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
           )
         : filter.operator in operatorReplacements
           ? Prisma.raw(
               operatorReplacements[
                 filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
             )
           : Prisma.raw(filter.operator); //checked by zod
 
@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
         break;
       case "stringOptions":
         valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
         )})`;
         break;
       case "arrayOptions":
         valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
           filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
         )}] `;
         break;
 
@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
       filter.type === "string" || filter.type === "stringObject"
         ? [
             ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw("'%' || ")
               : Prisma.empty,
             ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw(" || '%'")
               : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
   return column.name === "type" &&
     (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
   safeColumn: string,
   operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
   if (!dateOperators.includes(operator)) {
     throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
   }
 
   return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
   )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };

packages/shared/src/index.ts

@@ -1,9 +1,19 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
-
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./features/evals/types";
+export * from "./observationsTable";
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
+export * from "./server/evals/types";
 
-export { ModelUsageUnit };
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";

packages/shared/src/observationsTable.ts

@@ -1,8 +1,8 @@
+import { ObservationLevel } from "@prisma/client";
 import {
   type OptionsDefinition,
   type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -164,7 +164,7 @@ export type ObservationOptions = {
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {

packages/shared/src/queues.ts

@@ -0,0 +1,41 @@
+import { z } from "zod";
+
+export const QueueEnvelope = z.object({
+  timestamp: z.string().datetime({ offset: true }),
+  id: z.string(),
+});
+
+export const TraceUpsertEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    traceId: z.string(),
+  }),
+});
+
+export const EvalExecutionEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    jobExecutionId: z.string(),
+  }),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};

packages/shared/src/server/evals/types.ts

@@ -0,0 +1,44 @@
+import { observationsTableCols, tracesTableCols } from "../..";
+
+export const evalObjects = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      ...tracesTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -14,13 +14,17 @@ import { ChatOpenAI } from "@langchain/openai";
 import {
   ChatMessage,
   ChatMessageRole,
+  LLMFunctionCall,
   ModelParams,
   ModelProvider,
 } from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";
 
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -30,18 +34,27 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
+    functionCall: undefined;
   }
 ): Promise<IterableReadableStream<Uint8Array>>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
+    functionCall: undefined;
   }
 ): Promise<string>;
 
+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array>> {
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   const { messages, modelParams, streaming } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
@@ -71,6 +84,20 @@ export async function fetchLLMCompletion(
     });
   }
 
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
   if (streaming) {
     return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
   }

packages/shared/src/server/llm/types.ts

@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
   role: ChatMessageRole;
   content: string;
@@ -72,3 +75,9 @@ export const supportedModels = {
   [ModelProvider.Anthropic]: anthropicModels,
   [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};

packages/shared/src/tracesTable.ts

@@ -1,8 +1,5 @@
-import {
-  type OptionsDefinition,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
 import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";
 
 export const tracesTableCols: ColumnDefinition[] = [
   { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
@@ -122,7 +119,7 @@ export type TraceOptions = {
 };
 
 export function tracesTableColsWithOptions(
-  options?: TraceOptions,
+  options?: TraceOptions
 ): ColumnDefinition[] {
   return tracesTableCols.map((col) => {
     if (col.id === "scores_avg") {

packages/shared/src/types.ts

@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";
 
 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;

packages/shared/tsconfig.json

@@ -6,7 +6,8 @@
     "lib": ["ES2020"],
     "outDir": "./dist",
     "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

web/generated/openapi-server/openapi.yml

@@ -2412,6 +2412,50 @@ components:
         - name
     CreatePromptRequest:
       title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
       type: object
       properties:
         name:
@@ -2429,20 +2473,75 @@ components:
         - prompt
     Prompt:
       title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
       type: object
       properties:
         name:
           type: string
         version:
           type: integer
-        prompt:
-          type: string
         config: {}
       required:
         - name
         - version
-        - prompt
         - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.16.2",
+  "version": "2.19.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -18,13 +18,12 @@
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
     "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.540.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
     "@aws-sdk/s3-request-presigner": "^3.540.0",
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.3",
@@ -72,7 +71,7 @@
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
-    "ai": "^3.0.18",
+    "ai": "^3.0.19",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
@@ -112,8 +111,7 @@
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.0",
     "@testing-library/jest-dom": "^6.4.2",
     "@testing-library/react": "^14.2.2",
     "@types/bcryptjs": "^2.4.6",
@@ -138,7 +136,6 @@
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.13",
-    "release-it": "^17.1.1",
     "tailwindcss": "^3.4.3",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
@@ -150,30 +147,5 @@
   },
   "optionalDependencies": {
     "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
   }
 }

web/src/__tests__/api-auth.servertest.ts

@@ -1,4 +1,7 @@
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
+import {
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server/auth";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { prisma } from "@langfuse/shared/src/db";
 

web/src/__tests__/ingestion.servertest.ts

@@ -520,8 +520,6 @@ describe("/api/public/ingestion API Endpoint", () => {
 
       expect(response.status).toBe(207);
 
-      console.log("response body", response.body);
-
       const dbGeneration = await prisma.observation.findUnique({
         where: {
           id: generationId,
@@ -851,7 +849,6 @@ describe("/api/public/ingestion API Endpoint", () => {
       ],
     });
 
-    console.log(responseOne.body);
     expect(responseOne.status).toBe(207);
 
     expect("errors" in responseOne.body).toBe(true);
@@ -901,7 +898,6 @@ describe("/api/public/ingestion API Endpoint", () => {
       ],
     });
 
-    console.log(responseOne.body);
     expect(responseOne.status).toBe(207);
 
     expect("errors" in responseOne.body).toBe(true);
@@ -1167,8 +1163,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     });
     expect(responseOne.status).toBe(207);
 
-    console.log(responseOne.body);
-
     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
     const errors = responseOne.body.errors;
 
@@ -1419,7 +1413,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
       input,
     )} ${JSON.stringify(expected)}`, () => {
       const cleanedEvent = cleanEvent(input);
-      console.log(cleanedEvent);
       expect(cleanedEvent).toStrictEqual(expected);
     });
   });

web/src/__tests__/observations.servertest.ts

@@ -41,6 +41,19 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
+    const prompt = await prisma.prompt.create({
+      data: {
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
     await prisma.observation.create({
       data: {
         id: uuidv4(),
@@ -62,6 +75,9 @@ describe("/api/public/observations API Endpoint", () => {
         },
         internalModel: "gpt-3.5-turbo",
         unit: ModelUsageUnit.Tokens,
+        prompt: {
+          connect: { id: prompt.id },
+        },
       },
     });
 
@@ -95,6 +111,7 @@ describe("/api/public/observations API Endpoint", () => {
     expect(
       fetchedObservations.body.data[0]?.calculatedTotalCost,
     ).toBeGreaterThan(0);
+    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
   });
   it("should fetch all observations, filtered by generations", async () => {
     await pruneDatabase();

web/src/__tests__/orderByToPrisma.servertest.ts

@@ -1,5 +1,5 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+import { tracesTableCols } from "@langfuse/shared";
 
 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {

web/src/__tests__/prompts.servertest.ts

@@ -4,6 +4,11 @@ import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
 import { type Prompt } from "@langfuse/shared/src/db";
+import {
+  PromptSchema,
+  PromptType,
+  type ValidatedPrompt,
+} from "@/src/features/prompts/server/validation";
 
 describe("/api/public/prompts API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -43,6 +48,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.id).toBe(promptId);
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -190,6 +196,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.id).toBe(promptIdTwo);
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(2);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -221,6 +228,7 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("API");
@@ -342,8 +350,6 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(response.status).toBe(207);
 
-    console.log("response body", response.body);
-
     const dbGeneration = await prisma.observation.findUnique({
       where: {
         id: generationId,
@@ -375,11 +381,185 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({});
   });
+
+  it("should create and fetch a chat prompt", async () => {
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(201);
+
+    const { body: fetchedPrompt } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+
+    const validatedPrompt = validatePrompt(fetchedPrompt);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+  });
+
+  it("should fail if chat prompt has string prompt", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if chat prompt has incorrect messages format", async () => {
+    const promptName = "prompt-name";
+    const incorrectChatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", message: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: incorrectChatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+  it("should fail if text prompt has message format", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: [{ role: "system", content: "You are a bot" }],
+      type: "text",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if previous versions have different prompt type", async () => {
+    // Create a chat prompt
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const postResponse1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse1.status).toBe(201);
+
+    // Try creating a text prompt with the same name
+    const postResponse2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "text",
+      isActive: true,
+      version: 2,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse2.status).toBe(400);
+    expect(postResponse2.body).toEqual({
+      error: "ValidationError",
+      message:
+        "Previous versions have different prompt type. Create a new prompt with a different name.",
+    });
+
+    // Check if the prompt is still the chat prompt
+    const getResponse1 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(getResponse1.status).toBe(200);
+
+    const validatedPrompt = validatePrompt(getResponse1.body);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+
+    // Check that the text prompt has not been created
+    const getResponse2 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}&version=2`,
+      undefined,
+    );
+    expect(getResponse2.status).toBe(404);
+    expect(getResponse2.body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
 });
 
 const isPrompt = (x: unknown): x is Prompt => {
@@ -393,6 +573,18 @@ const isPrompt = (x: unknown): x is Prompt => {
     typeof prompt.isActive === "boolean" &&
     typeof prompt.projectId === "string" &&
     typeof prompt.createdBy === "string" &&
-    typeof prompt.config === "object"
+    typeof prompt.config === "object" &&
+    Object.values(PromptType).includes(prompt.type as PromptType)
   );
 };
+
+const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+  Object.keys(obj).forEach((key) => {
+    obj[key] =
+      key === "createdAt" || key === "updatedAt"
+        ? new Date(obj[key] as string)
+        : obj[key];
+  });
+
+  return PromptSchema.parse(obj);
+};

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
                 <MessageCircle size={12} />
               </HoverCardTrigger>
-              <HoverCardContent>
+              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                 <p>{s.comment}</p>
               </HoverCardContent>
             </HoverCard>

web/src/components/layouts/layout.tsx

@@ -309,8 +309,8 @@ export default function Layout(props: PropsWithChildren) {
               </ul>
             </nav>
 
-            <Menu as="div" className="relative left-1">
-              <Menu.Button className="flex w-full items-center gap-x-4 p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+            <Menu as="div" className="relative">
+              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
                 <span className="sr-only">Open user menu</span>
                 <Avatar className="h-8 w-8">
                   <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -342,7 +342,10 @@ export default function Layout(props: PropsWithChildren) {
                 leaveFrom="transform opacity-100 scale-100"
                 leaveTo="transform opacity-0 scale-95"
               >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                    {session.data?.user?.email}
+                  </span>
                   {userNavigation.map((item) => (
                     <Menu.Item key={item.name}>
                       {({ active }) => (
@@ -403,7 +406,10 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="transform opacity-100 scale-100"
               leaveTo="transform opacity-0 scale-95"
             >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  {session.data?.user?.email}
+                </span>
                 {userNavigation.map((item) => (
                   <Menu.Item key={item.name}>
                     {({ active }) => (
@@ -496,6 +502,7 @@ const MainNavigation: React.FC<{
                   "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
                 )}
                 onClick={onNavitemClick}
+                target={item.newTab ? "_blank" : undefined}
               >
                 {item.icon && (
                   <item.icon
@@ -574,6 +581,7 @@ const MainNavigation: React.FC<{
                                 : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
                               "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
                             )}
+                            target={subItem.newTab ? "_blank" : undefined}
                           >
                             {subItem.name}
                             {subItem.label && (

web/src/components/layouts/routes.ts

@@ -9,6 +9,7 @@ import {
   Settings,
   UsersIcon,
   PenSquareIcon,
+  LibraryBig,
   TerminalIcon,
 } from "lucide-react";
 
@@ -21,6 +22,7 @@ export type Route = {
   pathname?: string; // link, ignored if children
   children?: Array<Route>; // folder
   bottom?: boolean; // bottom of the sidebar, only for first level routes
+  newTab?: boolean; // open in new tab
 };
 
 export const ROUTES: Route[] = [
@@ -53,6 +55,16 @@ export const ROUTES: Route[] = [
         name: "Models",
         pathname: `/project/[projectId]/models`,
       },
+      {
+        name: "Templates",
+        pathname: `/project/[projectId]/evals/templates`,
+        featureFlag: "evals",
+      },
+      {
+        name: "Configs",
+        pathname: `/project/[projectId]/evals/configs`,
+        featureFlag: "evals",
+      },
     ],
   },
   {
@@ -83,6 +95,13 @@ export const ROUTES: Route[] = [
     icon: Settings,
     bottom: true,
   },
+  {
+    name: "Docs",
+    pathname: "https://langfuse.com/docs",
+    icon: LibraryBig,
+    bottom: true,
+    newTab: true,
+  },
   {
     name: "Support",
     pathname: "/project/[projectId]/support",

web/src/components/projectNavigation.tsx

@@ -7,7 +7,6 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { cn } from "@/src/utils/tailwind";
-import { Code } from "lucide-react";
 import { useRouter } from "next/router";
 
 interface Project {

web/src/components/table/data-table-toolbar.tsx

@@ -2,9 +2,9 @@ import { Button } from "@/src/components/ui/button";
 import React, { type Dispatch, type SetStateAction, useState } from "react";
 import { Input } from "@/src/components/ui/input";
 import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-table-column-visibility-filter";
-import { type FilterState } from "@/src/features/filters/types";
-import { FilterBuilder } from "@/src/features/filters/components/filter-builder";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import { type FilterState } from "@langfuse/shared";
+import { PopoverFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import { type ColumnDefinition } from "@langfuse/shared";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 
@@ -64,7 +64,7 @@ export function DataTableToolbar<TData, TValue>({
             </Button>
           </div>
         )}
-        <FilterBuilder
+        <PopoverFilterBuilder
           columns={filterColumnDefinition}
           filterState={filterState}
           onChange={setFilterState}

web/src/components/table/use-cases/generations.tsx

@@ -22,7 +22,6 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
 import {
   formatIntervalSeconds,
   intervalInSeconds,
@@ -37,7 +36,8 @@ import { usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type ExportFileFormats,
-} from "@/src/server/api/interfaces/exportTypes";
+  observationsTableColsWithOptions,
+} from "@langfuse/shared";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";

web/src/components/table/use-cases/scores.tsx

@@ -6,11 +6,11 @@ import useColumnVisibility from "@/src/features/column-visibility/hooks/useColum
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
-  ScoreOptions,
+  type ScoreOptions,
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { RouterOutput, type RouterInput } from "@/src/utils/types";
+import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
 export type ScoresTableRow = {

web/src/components/table/use-cases/sessions.tsx

@@ -6,7 +6,7 @@ import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsTableColsWithOptions } from "@/src/server/api/definitions/sessionsView";

web/src/components/table/use-cases/traces.tsx

@@ -10,17 +10,9 @@ import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import { Checkbox } from "@/src/components/ui/checkbox";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
-import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
-import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
-import {
-  TraceOptions,
-  tracesTableColsWithOptions,
-} from "@/src/server/api/definitions/tracesTable";
 import { api } from "@/src/utils/api";
 import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
-import { type ObservationLevel, type Score } from "@langfuse/shared/src/db";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
 import {
@@ -37,6 +29,15 @@ import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
 import { IOCell } from "./IOCell";
 import { setSmallPaginationIfColumnsVisible } from "@/src/features/column-visibility/hooks/setSmallPaginationIfColumnsVisible";
+import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
+import {
+  type FilterState,
+  type TraceOptions,
+  tracesTableColsWithOptions,
+  type ObservationLevel,
+  type Score,
+} from "@langfuse/shared";
 
 export type TracesTableRow = {
   bookmarked: boolean;

web/src/components/trace/IOPreview.tsx

@@ -136,11 +136,12 @@ const ChatMlMessageSchema = z
     content,
     json: Object.keys(other).length === 0 ? undefined : other,
   }));
-const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
+export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
 
-const OpenAiMessageView: React.FC<{
+export const OpenAiMessageView: React.FC<{
+  title?: string;
   messages: z.infer<typeof ChatMlArraySchema>;
-}> = ({ messages }) => {
+}> = ({ title, messages }) => {
   const COLLAPSE_THRESHOLD = 3;
   const [isCollapsed, setCollapsed] = useState(
     messages.length > COLLAPSE_THRESHOLD ? true : null,
@@ -158,58 +159,65 @@ const OpenAiMessageView: React.FC<{
   // );
 
   return (
-    <div className="flex flex-col gap-2 rounded-md border p-3">
-      {transformedMessages
-        .filter(
-          (_, i) =>
-            // show all if not collapsed or null; show first and last n if collapsed
-            !isCollapsed || i == 0 || i > messages.length - COLLAPSE_THRESHOLD,
-        )
-        .map((message, index) => (
-          <Fragment key={index}>
-            <div>
-              {!!message.content && (
-                <JSONView
-                  title={message.name ?? message.role}
-                  json={message.content}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.json && "rounded-b-none",
-                  )}
-                />
-              )}
-              {!!message.json && (
-                <JSONView
-                  title={
-                    message.content ? undefined : message.name ?? message.role
-                  }
-                  json={message.json}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.content && "rounded-t-none border-t-0",
-                  )}
-                />
-              )}
-            </div>
-            {isCollapsed !== null && index === 0 ? (
-              <Button
-                variant="ghost"
-                size="xs"
-                onClick={() => setCollapsed((v) => !v)}
-              >
-                {isCollapsed
-                  ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
-                  : "Hide history"}
-              </Button>
-            ) : null}
-          </Fragment>
-        ))}
+    <div className="rounded-md border">
+      {title && (
+        <div className="border-b px-3 py-1 text-xs font-medium">{title}</div>
+      )}
+      <div className="flex flex-col gap-2 p-3">
+        {transformedMessages
+          .filter(
+            (_, i) =>
+              // show all if not collapsed or null; show first and last n if collapsed
+              !isCollapsed ||
+              i == 0 ||
+              i > messages.length - COLLAPSE_THRESHOLD,
+          )
+          .map((message, index) => (
+            <Fragment key={index}>
+              <div>
+                {!!message.content && (
+                  <JSONView
+                    title={message.name ?? message.role}
+                    json={message.content}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.json && "rounded-b-none",
+                    )}
+                  />
+                )}
+                {!!message.json && (
+                  <JSONView
+                    title={
+                      message.content ? undefined : message.name ?? message.role
+                    }
+                    json={message.json}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.content && "rounded-t-none border-t-0",
+                    )}
+                  />
+                )}
+              </div>
+              {isCollapsed !== null && index === 0 ? (
+                <Button
+                  variant="ghost"
+                  size="xs"
+                  onClick={() => setCollapsed((v) => !v)}
+                >
+                  {isCollapsed
+                    ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
+                    : "Hide history"}
+                </Button>
+              ) : null}
+            </Fragment>
+          ))}
+      </div>
     </div>
   );
 };

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,6 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, $Enums } from "@langfuse/shared/src/db";
+import { type Trace, type Score, $Enums } from "@langfuse/shared";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared/src/db";
+import { type Trace, type Score } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.16.2";
+export const VERSION = "v2.19.0";

web/src/env.mjs

@@ -72,6 +72,9 @@ export const env = createEnv({
     S3_REGION: z.string().optional(),
     // Database exports
     DB_EXPORT_PAGE_SIZE: z.number().optional(),
+    // Worker
+    LANGFUSE_WORKER_HOST: z.string().optional(),
+    LANGFUSE_WORKER_PASSWORD: z.string().optional(),
     // Prompt playground
     OPENAI_API_KEY: z.string().optional(),
     ANTHROPIC_API_KEY: z.string().optional(),
@@ -137,11 +140,13 @@ export const env = createEnv({
     AUTH_OKTA_CLIENT_ID: process.env.AUTH_OKTA_CLIENT_ID,
     AUTH_OKTA_CLIENT_SECRET: process.env.AUTH_OKTA_CLIENT_SECRET,
     AUTH_OKTA_ISSUER: process.env.AUTH_OKTA_ISSUER,
-    AUTH_OKTA_ALLOW_ACCOUNT_LINKING: process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
+    AUTH_OKTA_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
     AUTH_AUTH0_CLIENT_ID: process.env.AUTH_AUTH0_CLIENT_ID,
     AUTH_AUTH0_CLIENT_SECRET: process.env.AUTH_AUTH0_CLIENT_SECRET,
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
-    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
@@ -156,6 +161,9 @@ export const env = createEnv({
     S3_REGION: process.env.S3_REGION,
     // Database exports
     DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
+    // Worker
+    LANGFUSE_WORKER_HOST: process.env.LANGFUSE_WORKER_HOST,
+    LANGFUSE_WORKER_PASSWORD: process.env.LANGFUSE_WORKER_PASSWORD,
     // Prompt playground
     OPENAI_API_KEY: process.env.OPENAI_API_KEY,
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,

web/src/features/audit-logs/auditLog.ts

@@ -13,7 +13,9 @@ export type AuditableResource =
   | "model"
   | "prompt"
   | "session"
-  | "apiKey";
+  | "apiKey"
+  | "evalTemplate"
+  | "job";
 
 type AuditLog = {
   resourceType: AuditableResource;

web/src/features/column-visibility/hooks/setSmallPaginationIfColumnsVisible.tsx

@@ -1,7 +1,6 @@
-import { type LangfuseColumnDef } from "@/src/components/table/types";
-import React from "react";
-import { TracesTableRow } from "@/src/components/table/use-cases/traces";
-import { GenerationsTableRow } from "../../../components/table/use-cases/generations";
+import type React from "react";
+import type { TracesTableRow } from "@/src/components/table/use-cases/traces";
+import type { GenerationsTableRow } from "../../../components/table/use-cases/generations";
 
 export type TableColumn =
   | (keyof GenerationsTableRow)[]

web/src/features/dashboard/components/ChartScores.tsx

@@ -5,7 +5,7 @@ import {
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
   extractTimeSeriesData,
   fillMissingValuesAndTransform,

web/src/features/dashboard/components/LatencyChart.tsx

@@ -3,7 +3,7 @@ import {
   dateTimeAggregationSettings,
   type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
   getAllModels,
   extractTimeSeriesData,

web/src/features/dashboard/components/LatencyTables.tsx

@@ -1,7 +1,7 @@
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 
 import { type DatabaseRow } from "@/src/server/api/services/query-builder";

web/src/features/dashboard/components/MetricTable.tsx

@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -4,7 +4,7 @@ import {
   dateTimeAggregationSettings,
   type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 
 import {
   getAllModels,

web/src/features/dashboard/components/ScoresTable.tsx

@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { RightAlignedCell } from "./RightAlignedCell";

web/src/features/dashboard/components/TracesBarListChart.tsx

@@ -1,5 +1,5 @@
 import { api } from "@/src/utils/api";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { ExpandListButton } from "@/src/features/dashboard/components/cards/ChevronButton";
 import { useState } from "react";
 import DocPopup from "@/src/components/layouts/doc-popup";

web/src/features/dashboard/components/TracesTimeSeriesChart.tsx

@@ -3,7 +3,7 @@ import {
   dateTimeAggregationSettings,
   type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";

web/src/features/dashboard/components/UserChart.tsx

@@ -1,6 +1,6 @@
 import { api } from "@/src/utils/api";
 import { type DateTimeAggregationOption } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TabComponent } from "@/src/features/dashboard/components/TabsComponent";

web/src/features/dashboard/components/hooks.ts

@@ -1,5 +1,5 @@
 import { type TimeSeriesChartDataPoint } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { type DatabaseRow } from "@/src/server/api/services/query-builder";
 import { api } from "@/src/utils/api";
 

web/src/features/dashboard/lib/dashboard-utils.ts

@@ -1,4 +1,4 @@
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { usdFormatter } from "@/src/utils/numbers";
 
 // traces do not have a startTime or endTime column, so we need to map these to the timestamp column

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -3,7 +3,7 @@ import { DataTable } from "@/src/components/table/data-table";
 import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { api } from "@/src/utils/api";
-import { formatIntervalSeconds, intervalInSeconds } from "@/src/utils/dates";
+import { formatIntervalSeconds } from "@/src/utils/dates";
 import { type RouterOutput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 

web/src/features/evals/Readme.md

@@ -0,0 +1,4 @@
+NOTE: We may transition this feature from our MIT licensed repository to the
+a commercial License (ee folder) once we release a first stable version.
+Please consider this when planning long-term use and integration of this functionality into your projects.
+For more information see https://langfuse.com/docs/open-source

web/src/features/evals/components/eval-config-table.tsx

@@ -0,0 +1,120 @@
+import { DataTable } from "@/src/components/table/data-table";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type JobConfiguration } from "@prisma/client";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+
+export type EvalConfigRow = {
+  id: string;
+  createdAt: string;
+  evalTemplateId?: string;
+  scoreName: string;
+  targetObject: string;
+  filter: string;
+  variableMapping: string;
+};
+
+export default function EvalConfigTable({ projectId }: { projectId: string }) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const templates = api.evals.allConfigs.useQuery({
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+    projectId,
+  });
+  const totalCount = templates.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<EvalConfigRow>[] = [
+    {
+      accessorKey: "id",
+      id: "id",
+      header: "ID",
+      enableHiding: false,
+    },
+    {
+      accessorKey: "createdAt",
+      id: "createdAt",
+      header: "Created At",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "evalTemplateId",
+      id: "evalTemplateId",
+      header: "Eval Template",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "scoreName",
+      id: "scoreName",
+      header: "Score Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "targetObject",
+      id: "targetObject",
+      header: "Target",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "filter",
+      id: "filter",
+      header: "Filter",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "variableMapping",
+      id: "variableMapping",
+      header: "Mapping",
+      enableHiding: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<EvalConfigRow>("evalConfigColumnVisibility", columns);
+
+  const convertToTableRow = (jobConfig: JobConfiguration): EvalConfigRow => {
+    return {
+      id: jobConfig.id,
+      createdAt: jobConfig.createdAt.toLocaleString(),
+      evalTemplateId: jobConfig.evalTemplateId?.toLocaleString(),
+      scoreName: jobConfig.scoreName,
+      targetObject: jobConfig.targetObject,
+      filter: JSON.stringify(jobConfig.filter),
+      variableMapping: JSON.stringify(jobConfig.variableMapping),
+    };
+  };
+
+  return (
+    <div>
+      <DataTable
+        columns={columns}
+        data={
+          templates.isLoading
+            ? { isLoading: true, isError: false }
+            : templates.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: templates.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: templates.data.configs.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(totalCount / paginationState.pageSize),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
+  );
+}

web/src/features/evals/components/eval-templates-table.tsx

@@ -0,0 +1,161 @@
+import { DataTable } from "@/src/components/table/data-table";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { api } from "@/src/utils/api";
+import { type EvalTemplate } from "@prisma/client";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+
+export type EvalsTemplateRow = {
+  version: number;
+  name: string;
+  prompt: string;
+  model: string;
+  modelParameters: unknown;
+  variables: string[];
+  outputScore?: string;
+  outputName?: string;
+  outputReasoning?: string;
+};
+
+export default function EvalsTemplateTable({
+  projectId,
+}: {
+  projectId: string;
+}) {
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const templates = api.evals.allTemplates.useQuery({
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+    projectId,
+  });
+  const totalCount = templates.data?.totalCount ?? 0;
+
+  const columns: LangfuseColumnDef<EvalsTemplateRow>[] = [
+    {
+      accessorKey: "name",
+      header: "Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "version",
+      header: "Version",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "prompt",
+      header: "Prompt",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "model",
+      header: "Model",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "modelParameters",
+      header: "Model Parameters",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "variables",
+      header: "Variables",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "score",
+      header: "Score",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "name",
+      header: "Name",
+      enableHiding: true,
+    },
+    {
+      accessorKey: "reasoning",
+      header: "Reasoning",
+      enableHiding: true,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<EvalsTemplateRow>(
+      "evalTemplateConfigColumnVisibility",
+      columns,
+    );
+
+  const convertToTableRow = (template: EvalTemplate): EvalsTemplateRow => {
+    if (
+      typeof template.outputSchema !== "object" ||
+      template.outputSchema === null
+    ) {
+      return {
+        name: template.name,
+        version: template.version,
+        prompt: template.prompt,
+        model: template.model,
+        modelParameters: template.modelParams,
+        variables: template.vars,
+      };
+    }
+    return {
+      name: template.name,
+      version: template.version,
+      prompt: template.prompt,
+      model: template.model,
+      modelParameters: JSON.stringify(template.modelParams),
+      variables: template.vars,
+      outputScore:
+        "scores" in template.outputSchema &&
+        typeof template.outputSchema.scores === "string"
+          ? template.outputSchema.scores
+          : undefined,
+      outputName:
+        "name" in template.outputSchema &&
+        typeof template.outputSchema.name === "string"
+          ? template.outputSchema.name
+          : undefined,
+      outputReasoning:
+        "reasoning" in template.outputSchema &&
+        typeof template.outputSchema.reasoning === "string"
+          ? template.outputSchema.reasoning
+          : undefined,
+    };
+  };
+
+  return (
+    <div>
+      <DataTable
+        columns={columns}
+        data={
+          templates.isLoading
+            ? { isLoading: true, isError: false }
+            : templates.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: templates.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: templates.data.templates.map((t) =>
+                    convertToTableRow(t),
+                  ),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(totalCount / paginationState.pageSize),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
+  );
+}

web/src/features/evals/components/new-eval-config-form.tsx

@@ -0,0 +1,423 @@
+import { usePostHog } from "posthog-js/react";
+import { useFieldArray, useForm } from "react-hook-form";
+import { Input } from "@/src/components/ui/input";
+import { Button } from "@/src/components/ui/button";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import { tracesTableColsWithOptions, singleFilter } from "@langfuse/shared";
+import { type FilterState } from "@langfuse/shared";
+import * as z from "zod";
+import { Card } from "@/src/components/ui/card";
+import { useEffect, useState } from "react";
+import { api } from "@/src/utils/api";
+import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import {
+  type EvalTemplate,
+  variableMapping,
+  wipVariableMapping,
+  evalObjects,
+} from "@langfuse/shared";
+import router from "next/router";
+
+const formSchema = z.object({
+  evalTemplateId: z.string(),
+  scoreName: z.string(),
+  target: z.string(),
+  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+  mapping: z.array(wipVariableMapping),
+  sampling: z.coerce.number().gte(0).lte(1),
+  delay: z.coerce.number().optional().default(10_000),
+});
+
+export const NewEvalConfigForm = (props: {
+  projectId: string;
+  evalTemplates: EvalTemplate[];
+  onFormSuccess?: () => void;
+}) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const posthog = usePostHog();
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    defaultValues: {
+      evalTemplateId: "",
+      scoreName: undefined,
+      target: "trace",
+      filter: [] as FilterState,
+      mapping: [],
+      sampling: 1,
+      delay: 10_000,
+    },
+  });
+
+  const traceFilterOptions = api.traces.filterOptions.useQuery({
+    projectId: props.projectId,
+    ...form.getFieldState("filter"),
+  });
+
+  const getSelectedEvalTemplate = props.evalTemplates.find(
+    (template) =>
+      `${template.name}-${template.version}` ===
+      form.getValues().evalTemplateId,
+  );
+
+  useEffect(() => {
+    if (getSelectedEvalTemplate) {
+      form.setValue("mapping", []);
+      form.setValue(
+        "mapping",
+        getSelectedEvalTemplate.vars.map((v) => ({
+          templateVariable: v,
+          langfuseObject: "trace" as const,
+        })),
+      );
+    }
+  }, [form, getSelectedEvalTemplate]);
+
+  const { fields } = useFieldArray({
+    control: form.control,
+    name: "mapping",
+  });
+
+  const utils = api.useUtils();
+  const createJobMutation = api.evals.createJob.useMutation({
+    onSuccess: () => utils.models.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    posthog.capture("models:new_template_form");
+    if (!getSelectedEvalTemplate) {
+      setFormError("Please select an eval template");
+      return;
+    }
+
+    // validate wip variable mapping
+    const validatedVarMapping = z.array(variableMapping).parse(values.mapping);
+
+    createJobMutation
+      .mutateAsync({
+        projectId: props.projectId,
+        evalTemplateId: getSelectedEvalTemplate.id,
+        scoreName: values.scoreName,
+        target: values.target,
+        filter: values.filter,
+        mapping: validatedVarMapping,
+        sampling: values.sampling,
+      })
+      .then(() => {
+        props.onFormSuccess?.();
+        form.reset();
+        void router.push(`/project/${props.projectId}/evals/configs/`);
+      })
+      .catch((error) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        if ("message" in error && typeof error.message === "string") {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          setFormError(error.message as string);
+          return;
+        } else {
+          setFormError(JSON.stringify(error));
+          console.error(error);
+        }
+      });
+  }
+
+  return (
+    <Form {...form}>
+      {JSON.stringify(form.watch(), null, 2)}
+      <form
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="flex flex-col gap-4"
+      >
+        <div className="grid gap-4">
+          <FormField
+            control={form.control}
+            name="evalTemplateId"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Eval Template</FormLabel>
+                <Select
+                  defaultValue={field.value}
+                  onValueChange={(value) => {
+                    field.onChange(value);
+                  }}
+                >
+                  <FormControl>
+                    <SelectTrigger>
+                      <SelectValue placeholder="Select a model to run this eval template" />
+                    </SelectTrigger>
+                  </FormControl>
+                  <SelectContent>
+                    {props.evalTemplates.map((template) => (
+                      <SelectItem
+                        value={`${template.name}-${template.version}`}
+                        key={template.id}
+                      >
+                        {`${template.name}-${template.version}`}
+                      </SelectItem>
+                    ))}
+                  </SelectContent>
+                </Select>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          <FormField
+            control={form.control}
+            name="scoreName"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Score Name</FormLabel>
+                <FormControl>
+                  <Input {...field} />
+                </FormControl>
+                <FormDescription>
+                  Optional score name, defaults to ABCDEFG
+                </FormDescription>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="target"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Eval target object</FormLabel>
+                  <FormControl>
+                    <Tabs defaultValue="trace">
+                      <TabsList {...field}>
+                        <TabsTrigger value="trace">Trace</TabsTrigger>
+                        <TabsTrigger value="observation" disabled={true}>
+                          Observation (coming soon)
+                        </TabsTrigger>
+                      </TabsList>
+                    </Tabs>
+                  </FormControl>
+                  <FormDescription>Description</FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="filter"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Target filter</FormLabel>
+                  <FormControl>
+                    <div className="w-1/2">
+                      <InlineFilterBuilder
+                        columns={tracesTableColsWithOptions(
+                          traceFilterOptions.data,
+                        )}
+                        filterState={field.value ?? []}
+                        onChange={(value) => field.onChange(value)}
+                      />
+                    </div>
+                  </FormControl>
+                  <FormDescription>
+                    This will run on all future and XX historical traces.
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+
+          <Card className="p-4">
+            <FormLabel>Variable mapping</FormLabel>
+            <FormControl>Here will some variable mapping be added.</FormControl>
+            <div className="mt-2 flex flex-col gap-2">
+              {fields.map((mappingField, index) => (
+                <div className="flex gap-2" key={index}>
+                  <span className="whitespace-nowrap rounded-md bg-slate-200 px-2 py-1 text-xs	">
+                    {mappingField.templateVariable}
+                  </span>
+                  <FormField
+                    control={form.control}
+                    key={`${mappingField.id}-langfuseObject`}
+                    name={`mapping.${index}.langfuseObject`}
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormControl>
+                          <Select
+                            defaultValue={
+                              evalObjects.find(
+                                (evalObject) => evalObject.id === field.value,
+                              )?.display
+                            }
+                            onValueChange={(value) => {
+                              const obj = evalObjects.find(
+                                (evalObject) => evalObject.display === value,
+                              );
+                              field.onChange(obj?.id);
+                            }}
+                          >
+                            <SelectTrigger>
+                              <SelectValue placeholder="Object type" />
+                            </SelectTrigger>
+                            <SelectContent>
+                              {evalObjects.map((evalObject) => (
+                                <SelectItem
+                                  value={evalObject.display}
+                                  key={evalObject.id}
+                                >
+                                  {evalObject.display}
+                                </SelectItem>
+                              ))}
+                            </SelectContent>
+                          </Select>
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+
+                  {form.watch(`mapping.${index}.langfuseObject`) !== "trace" ? (
+                    <FormField
+                      control={form.control}
+                      key={`${mappingField.id}-objectName`}
+                      name={`mapping.${index}.objectName`}
+                      render={({ field }) => (
+                        <FormItem>
+                          <FormControl>
+                            <Input {...field} value={field.value ?? ""} />
+                          </FormControl>
+                          <FormMessage />
+                        </FormItem>
+                      )}
+                    />
+                  ) : undefined}
+
+                  <FormField
+                    control={form.control}
+                    key={`${mappingField.id}-selectedColumnId`}
+                    name={`mapping.${index}.selectedColumnId`}
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormControl>
+                          <Select
+                            defaultValue={
+                              field.value
+                                ? evalObjects.find(
+                                    (evalObject) =>
+                                      evalObject.id === field.value,
+                                  )?.availableColumns[0].name ?? "N/A"
+                                : "N/A"
+                            }
+                            onValueChange={(value) => {
+                              const availableColumns = evalObjects.find(
+                                (evalObject) =>
+                                  evalObject.id ===
+                                  form.watch(`mapping.${index}.langfuseObject`),
+                              )?.availableColumns;
+                              const column = availableColumns?.find(
+                                (column) => column.name === value,
+                              );
+
+                              field.onChange(column?.id);
+                            }}
+                          >
+                            <SelectTrigger>
+                              <SelectValue placeholder="Object type" />
+                            </SelectTrigger>
+                            <SelectContent>
+                              {evalObjects
+                                .find(
+                                  (evalObject) =>
+                                    evalObject.id ===
+                                    form.watch(
+                                      `mapping.${index}.langfuseObject`,
+                                    ),
+                                )
+                                ?.availableColumns.map((column) => (
+                                  <SelectItem
+                                    value={column.name}
+                                    key={column.id}
+                                  >
+                                    {column.name}
+                                  </SelectItem>
+                                ))}
+                            </SelectContent>
+                          </Select>
+                        </FormControl>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
+                </div>
+              ))}
+            </div>
+            <FormDescription>Description </FormDescription>
+            <FormMessage />
+          </Card>
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="sampling"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Sampling</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="delay"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Delay (ms)</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+        </div>
+
+        <Button
+          type="submit"
+          loading={createJobMutation.isLoading}
+          className="mt-3"
+        >
+          Save
+        </Button>
+      </form>
+      {formError ? (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      ) : null}
+    </Form>
+  );
+};

web/src/features/evals/components/new-template-form.tsx

@@ -0,0 +1,316 @@
+import { usePostHog } from "posthog-js/react";
+import { useState } from "react";
+import { useForm } from "react-hook-form";
+import JsonView from "react18-json-view";
+import * as z from "zod";
+import { Input } from "@/src/components/ui/input";
+import { Button } from "@/src/components/ui/button";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { Textarea } from "@/src/components/ui/textarea";
+import { api } from "@/src/utils/api";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { extractVariables } from "@/src/utils/string";
+import { evalModelList, evalModels } from "@/src/features/evals/constants";
+import { Badge } from "@/src/components/ui/badge";
+import { jsonSchema } from "@/src/utils/zod";
+import router from "next/router";
+import { AutoComplete } from "@/src/features/prompts/components/auto-complete";
+import { type EvalTemplate } from "@prisma/client";
+
+const formSchema = z.object({
+  name: z.string(),
+  prompt: z
+    .string()
+    .min(1, "Enter a prompt")
+    .refine((val) => {
+      const variables = extractVariables(val);
+      const matches = variables.map((variable) => {
+        // check regex here
+        if (variable.match(/^[A-Za-z_]+$/)) {
+          return true;
+        }
+        return false;
+      });
+      return !matches.includes(false);
+    }, "Variables must only contain letters and underscores (_)"),
+
+  variables: z.array(
+    z.string().min(1, "Variables must have at least one character"),
+  ),
+  model: evalModels,
+  modelParameters: z.string().refine(
+    (value) => {
+      try {
+        JSON.parse(value);
+        return true;
+      } catch (e) {
+        return false;
+      }
+    },
+    {
+      message: "Config needs to be valid JSON",
+    },
+  ),
+  outputScore: z.string(),
+  outputName: z.string(),
+  outputReasoning: z.string(),
+});
+
+export const NewEvalTemplateForm = (props: {
+  projectId: string;
+  existingEvalTemplates: EvalTemplate[];
+  onFormSuccess?: () => void;
+}) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const posthog = usePostHog();
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    defaultValues: {
+      name: "",
+      model: "gpt-4" as const,
+      prompt: "",
+      variables: [],
+      modelParameters: "{}",
+      outputName: "",
+      outputScore: "",
+      outputReasoning: "",
+    },
+  });
+
+  const extractedVariables = extractVariables(form.watch("prompt"));
+
+  const utils = api.useUtils();
+  const createEvalTemplateMutation = api.evals.createTemplate.useMutation({
+    onSuccess: () => utils.models.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    posthog.capture("models:new_template_form");
+    createEvalTemplateMutation
+      .mutateAsync({
+        name: values.name,
+        projectId: props.projectId,
+        prompt: values.prompt,
+        model: values.model,
+        modelParameters:
+          values.modelParameters &&
+          typeof JSON.parse(values.modelParameters) === "object"
+            ? jsonSchema.parse(JSON.parse(values.modelParameters))
+            : jsonSchema.parse({}),
+        variables: extractedVariables,
+        outputSchema: {
+          score: values.outputScore,
+          reasoning: values.outputReasoning,
+        },
+      })
+      .then(() => {
+        props.onFormSuccess?.();
+        form.reset();
+        void router.push(`/project/${props.projectId}/evals/templates/`);
+      })
+      .catch((error) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        if ("message" in error && typeof error.message === "string") {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          setFormError(error.message as string);
+          return;
+        } else {
+          setFormError(JSON.stringify(error));
+          console.error(error);
+        }
+      });
+  }
+
+  return (
+    <Form {...form}>
+      <form
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="flex flex-col gap-4"
+      >
+        <div className="grid grid-cols-4 gap-x-12">
+          <div className="col-span-3 row-span-1">
+            <FormField
+              control={form.control}
+              name="name"
+              render={({ field }) => (
+                <>
+                  <FormItem>
+                    <FormLabel>Name</FormLabel>
+                    <FormControl>
+                      <AutoComplete
+                        {...field}
+                        options={props.existingEvalTemplates.map(
+                          (template) => ({
+                            value: template.name,
+                            label: template.name,
+                          }),
+                        )}
+                        placeholder=""
+                        onValueChange={(option) => field.onChange(option.value)}
+                        value={{ value: field.value, label: field.value }}
+                        disabled={false}
+                        createLabel="New eval template name:"
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                </>
+              )}
+            />
+          </div>
+
+          <div className="col-span-3 row-span-4">
+            <FormField
+              control={form.control}
+              name="prompt"
+              render={({ field }) => (
+                <>
+                  <FormItem>
+                    <FormLabel>Prompt</FormLabel>
+                    <FormControl>
+                      <Textarea
+                        {...field}
+                        className="min-h-[150px] flex-1 font-mono text-xs"
+                      />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                  <FormDescription>
+                    <p className="text-sm text-gray-500">
+                      You can use{" "}
+                      <code className="text-xs">{"{{variable}}"}</code> to
+                      insert variables into your prompt. The following variables
+                      are available:
+                    </p>
+
+                    <div className="flex flex-wrap gap-2">
+                      {extractedVariables.map((variable) => (
+                        <Badge key={variable} variant="outline">
+                          {variable}
+                        </Badge>
+                      ))}
+                    </div>
+                  </FormDescription>
+                </>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="outputScore"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Score</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description</FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="outputReasoning"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Reasoning</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>Description</FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </div>
+          <div className="col-span-1">
+            <FormField
+              control={form.control}
+              name="model"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Model</FormLabel>
+                  <Select
+                    defaultValue={field.value}
+                    onValueChange={(value) =>
+                      field.onChange(value as (typeof evalModelList)[number])
+                    }
+                  >
+                    <FormControl>
+                      <SelectTrigger>
+                        <SelectValue placeholder="Select a model to run this eval template" />
+                      </SelectTrigger>
+                    </FormControl>
+                    <SelectContent>
+                      {evalModelList.map((model) => (
+                        <SelectItem value={model} key={model}>
+                          {model}
+                        </SelectItem>
+                      ))}
+                    </SelectContent>
+                  </Select>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </div>
+          <div className="col-span-1 row-span-2">
+            <FormField
+              control={form.control}
+              name="modelParameters"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Model Parameters</FormLabel>
+                  <JsonView
+                    src={jsonSchema.parse(JSON.parse(field.value))}
+                    onEdit={(edit) => {
+                      // need to put string back into the state
+                      field.onChange(JSON.stringify(edit.src));
+                    }}
+                    editable
+                    className="rounded-md border border-gray-200 p-2 text-sm"
+                  />
+                  <FormDescription>
+                    Set parameters to use for the LLM call.
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </div>
+        </div>
+
+        <Button
+          type="submit"
+          loading={createEvalTemplateMutation.isLoading}
+          className="mt-3"
+        >
+          Save
+        </Button>
+      </form>
+      {formError ? (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      ) : null}
+    </Form>
+  );
+};

web/src/features/evals/constants.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+export const evalModels = z.union([
+  z.literal("gpt-3.5-turbo"),
+  z.literal("gpt-4"),
+]);
+
+export const evalModelList = evalModels._def.options.map(
+  (option) => option.value,
+);
+
+export const jobTypes = ["evaluations"] as const;
+
+export enum JobTypes {
+  Evaluation = "evaluation",
+}
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;

web/src/features/evals/server/router.ts

@@ -0,0 +1,197 @@
+import { z } from "zod";
+
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import {
+  DEFAULT_TRACE_JOB_DELAY,
+  EvalTargetObject,
+  evalModels,
+} from "@/src/features/evals/constants";
+import { jsonSchema } from "@/src/utils/zod";
+import { singleFilter, variableMapping } from "@langfuse/shared";
+
+export const CreateEvalTemplate = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  prompt: z.string(),
+  model: evalModels,
+  modelParameters: jsonSchema,
+  variables: z.array(z.string()),
+  outputSchema: z.object({
+    score: z.string(),
+    reasoning: z.string(),
+  }),
+});
+
+export const evalRouter = createTRPCRouter({
+  allConfigs: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        limit: z.number(),
+        page: z.number(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "job:read",
+      });
+
+      const configs = await ctx.prisma.jobConfiguration.findMany({
+        where: {
+          projectId: input.projectId,
+          jobType: "EVAL",
+        },
+        take: input.limit,
+        skip: input.page * input.limit,
+      });
+
+      const count = await ctx.prisma.jobConfiguration.count({
+        where: {
+          projectId: input.projectId,
+          jobType: "EVAL",
+        },
+      });
+      return {
+        configs: configs,
+        totalCount: count,
+      };
+    }),
+
+  allTemplates: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        limit: z.number(),
+        page: z.number(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:read",
+      });
+
+      const templates = await ctx.prisma.evalTemplate.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        take: input.limit,
+        skip: input.page * input.limit,
+      });
+
+      const count = await ctx.prisma.evalTemplate.count({
+        where: {
+          projectId: input.projectId,
+        },
+      });
+      return {
+        templates: templates,
+        totalCount: count,
+      };
+    }),
+
+  createJob: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        evalTemplateId: z.string(),
+        scoreName: z.string(),
+        target: z.string(),
+        filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+        mapping: z.array(variableMapping),
+        sampling: z.number().gte(0).lte(1),
+        delay: z.number().lte(0).default(10_000),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "job:create",
+        });
+
+        const evalTemplate = await ctx.prisma.evalTemplate.findUnique({
+          where: {
+            id: input.evalTemplateId,
+            projectId: input.projectId,
+          },
+        });
+
+        if (!evalTemplate) {
+          console.log(
+            `Template not found for project ${input.projectId} and id ${input.evalTemplateId}`,
+          );
+          throw new Error("Template not found");
+        }
+
+        const job = await ctx.prisma.jobConfiguration.create({
+          data: {
+            projectId: input.projectId,
+            jobType: "EVAL",
+            evalTemplateId: input.evalTemplateId,
+            scoreName: input.scoreName,
+            targetObject: EvalTargetObject.Trace,
+            filter: input.filter ?? [],
+            variableMapping: input.mapping,
+            sampling: input.sampling,
+            delay: DEFAULT_TRACE_JOB_DELAY, // 10 seconds default
+          },
+        });
+        await auditLog({
+          session: ctx.session,
+          resourceType: "job",
+          resourceId: job.id,
+          action: "create",
+        });
+      } catch (e) {
+        console.log(e);
+        throw e;
+      }
+    }),
+  createTemplate: protectedProjectProcedure
+    .input(CreateEvalTemplate)
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalTemplate:create",
+      });
+
+      const latestTemplate = await ctx.prisma.evalTemplate.findFirst({
+        where: {
+          projectId: input.projectId,
+          name: input.name,
+        },
+        orderBy: [{ version: "desc" }],
+      });
+
+      const evalTemplate = await ctx.prisma.evalTemplate.create({
+        data: {
+          version: latestTemplate?.version ? latestTemplate.version + 1 : 1,
+          name: input.name,
+          projectId: input.projectId,
+          prompt: input.prompt,
+          model: input.model,
+          modelParams: input.modelParameters,
+          vars: input.variables,
+          outputSchema: input.outputSchema,
+        },
+      });
+
+      await auditLog({
+        session: ctx.session,
+        resourceType: "evalTemplate",
+        resourceId: evalTemplate.id,
+        action: "create",
+      });
+    }),
+});

web/src/features/feature-flags/available-flags.ts

@@ -1 +1 @@
-export const availableFlags = ["templateFlag", "playground"] as const;
+export const availableFlags = ["templateFlag", "evals", "playground"] as const;

web/src/features/feature-flags/hooks/useIsFeatureEnabled.ts

@@ -0,0 +1,15 @@
+import { useSession } from "next-auth/react";
+
+import { api } from "@/src/utils/api";
+
+import type { Flag } from "../types";
+
+export default function useIsFeatureEnabled(feature: Flag): boolean {
+  const session = useSession();
+  const isExperimentalFeaturesEnabled =
+    api.environment.enableExperimentalFeatures.useQuery().data ?? false;
+  const isFeatureEnabledOnUser =
+    session.data?.user?.featureFlags[feature] ?? false;
+
+  return isExperimentalFeaturesEnabled || isFeatureEnabledOnUser;
+}

web/src/features/filters/components/filter-builder.tsx

@@ -21,16 +21,14 @@ import {
   type WipFilterCondition,
   type FilterState,
   type FilterCondition,
-} from "@/src/features/filters/types";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
-import {
+  type ColumnDefinition,
   filterOperators,
   singleFilter,
-} from "@/src/server/api/interfaces/filters";
+} from "@langfuse/shared";
 import { NonEmptyString } from "@/src/utils/zod";
 
 // Has WipFilterState, passes all valid filters to parent onChange
-export function FilterBuilder({
+export function PopoverFilterBuilder({
   columns,
   filterState,
   onChange,
@@ -145,6 +143,42 @@ export function FilterBuilder({
   );
 }
 
+export function InlineFilterBuilder({
+  columns,
+  filterState,
+  onChange,
+}: {
+  columns: ColumnDefinition[];
+  filterState: FilterState;
+  onChange: Dispatch<SetStateAction<FilterState>>;
+}) {
+  const [wipFilterState, _setWipFilterState] =
+    useState<WipFilterState>(filterState);
+
+  const setWipFilterState = (
+    state: ((prev: WipFilterState) => WipFilterState) | WipFilterState,
+  ) => {
+    _setWipFilterState((prev) => {
+      const newState = state instanceof Function ? state(prev) : state;
+      const validFilters = newState.filter(
+        (f) => singleFilter.safeParse(f).success,
+      ) as FilterState;
+      onChange(validFilters);
+      return newState;
+    });
+  };
+
+  return (
+    <div className="flex flex-col">
+      <FilterBuilderForm
+        columns={columns}
+        filterState={wipFilterState}
+        onChange={setWipFilterState}
+      />
+    </div>
+  );
+}
+
 function FilterBuilderForm({
   columns,
   filterState,

web/src/features/filters/components/multi-select.tsx

@@ -19,7 +19,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { Separator } from "@/src/components/ui/separator";
-import { type FilterOption } from "@/src/features/filters/types";
+import { type FilterOption } from "@langfuse/shared";
 
 export function MultiSelect({
   title,

web/src/features/filters/hooks/useFilterState.ts

@@ -1,9 +1,12 @@
-import { type FilterState, type TableName } from "@/src/features/filters/types";
-import { observationsTableCols } from "@/src/server/api/definitions/observationsTable";
+import {
+  type FilterState,
+  type TableName,
+  observationsTableCols,
+  tracesTableCols,
+  singleFilter,
+} from "@langfuse/shared";
 import { scoresTableCols } from "@/src/server/api/definitions/scoresTable";
 import { sessionsViewCols } from "@/src/server/api/definitions/sessionsView";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
-import { singleFilter } from "@/src/server/api/interfaces/filters";
 import { useState } from "react";
 import {
   useQueryParam,

web/src/features/orderBy/server/orderByToPrisma.ts

@@ -1,6 +1,6 @@
 import { type OrderByState } from "@/src/features/orderBy/types";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+import { type ColumnDefinition } from "@langfuse/shared";
+import { Prisma } from "@langfuse/shared";
 import { z } from "zod";
 
 /**

web/src/features/orderBy/types.ts

@@ -1,4 +1,4 @@
-import { type orderBy } from "@/src/server/api/interfaces/orderBy";
+import { type orderBy } from "@langfuse/shared";
 import { type z } from "zod";
 
 // to be sent to the server

web/src/features/playground/Readme.md

@@ -0,0 +1,4 @@
+NOTE: We may transition this feature from our MIT licensed repository to the
+a commercial License (ee folder) once we release a first stable version.
+Please consider this when planning long-term use and integration of this functionality into your projects.
+For more information see https://langfuse.com/docs/open-source

web/src/features/playground/client/components/ChatMessageComponent.tsx

@@ -5,17 +5,16 @@ import { ChatMessageRole, type ChatMessageWithId } from "@langfuse/shared";
 import { Button } from "@/src/components/ui/button";
 import { Card, CardContent } from "@/src/components/ui/card";
 import { Textarea } from "@/src/components/ui/textarea";
-import { MessagesContext } from "@/src/features/playground/client/components/Messages";
+import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
 
 type ChatMessageProps = Pick<
   MessagesContext,
-  "deleteMessage" | "updateMessage" | "updatePromptVariables"
+  "deleteMessage" | "updateMessage"
 > & { message: ChatMessageWithId };
 
 export const ChatMessageComponent: React.FC<ChatMessageProps> = ({
   message,
   updateMessage,
-  updatePromptVariables,
   deleteMessage,
 }) => {
   const textAreaRef = useRef<HTMLTextAreaElement | null>(null);
@@ -49,30 +48,35 @@ export const ChatMessageComponent: React.FC<ChatMessageProps> = ({
   }, [message.content]);
 
   return (
-    <Card>
-      <CardContent className="flex flex-row space-x-1 pt-6">
-        <div className="min-w-[7rem]">
-          <Button onClick={toggleRole} variant="outline">
+    <Card className="p-3">
+      <CardContent className="flex flex-row space-x-1 p-0">
+        <div className="min-w-[6rem]">
+          <Button
+            onClick={toggleRole}
+            type="button" // prevents submitting a form if this button is inside a form
+            variant="outline"
+            className="text-xs"
+          >
             {capitalize(message.role)}
           </Button>
         </div>
 
         <Textarea
           ref={textAreaRef}
-          className="height-[auto] min-h-10 w-full font-mono focus:outline-none"
+          className="height-[auto] min-h-8 w-full pt-3  font-mono text-xs focus:outline-none"
           placeholder={placeholder}
           value={message.content}
           onChange={handleContentChange}
-          onBlur={updatePromptVariables}
           rows={textAreaRows}
         />
         <Button
           variant="ghost"
+          type="button" // prevents submitting a form if this button is inside a form
           size="icon"
           onClick={() => deleteMessage(message.id)}
           disabled={message.role === ChatMessageRole.System}
         >
-          <MinusCircleIcon />
+          <MinusCircleIcon size={16} />
         </Button>
       </CardContent>
     </Card>

web/src/features/playground/client/components/ChatMessages.tsx

@@ -2,9 +2,9 @@ import { PlusCircleIcon } from "lucide-react";
 
 import { Button } from "@/src/components/ui/button";
 import { ChatMessageComponent } from "@/src/features/playground/client/components/ChatMessageComponent";
-import { MessagesContext } from "@/src/features/playground/client/components/Messages";
+import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
 import { ChatMessageRole } from "@langfuse/shared";
-import { useRef, useCallback, useEffect, useState } from "react";
+import { useRef, useEffect } from "react";
 
 type ChatMessagesProps = MessagesContext;
 export const ChatMessages: React.FC<ChatMessagesProps> = (props) => {
@@ -48,11 +48,12 @@ const AddMessageButton: React.FC<AddMessageButtonProps> = ({
 
   return (
     <Button
+      type="button" // prevents submitting a form if this button is inside a form
       variant="outline"
       className="w-full space-x-2 py-6"
       onClick={() => addMessage(nextMessageRole)}
     >
-      <PlusCircleIcon />
+      <PlusCircleIcon size={16} />
       <p>Add message</p>
     </Button>
   );

web/src/features/playground/client/components/Messages.tsx

@@ -1,6 +1,6 @@
 import { Button } from "@/src/components/ui/button";
 import { usePlaygroundContext } from "@/src/features/playground/client/context";
-import { ChatMessageRole, ChatMessageWithId } from "@langfuse/shared";
+import type { ChatMessageRole, ChatMessageWithId } from "@langfuse/shared";
 
 import { GenerationOutput } from "./GenerationOutput";
 import { ChatMessages } from "@/src/features/playground/client/components/ChatMessages";
@@ -14,7 +14,6 @@ export type MessagesContext = {
     key: Key,
     value: ChatMessageWithId[Key],
   ) => void;
-  updatePromptVariables: () => void;
 };
 
 export const Messages: React.FC<MessagesContext> = (props) => {
@@ -34,7 +33,7 @@ const SubmitButton = () => {
   return (
     <Button
       variant="default"
-      className="w-full space-x-2 py-6"
+      className="h-[88px] w-full space-x-2 py-3"
       onClick={() => {
         handleSubmit().catch((err) => console.error(err));
       }}

web/src/features/playground/client/components/PromptVariableComponent.tsx

@@ -43,7 +43,7 @@ export const PromptVariableComponent: React.FC<{
         onClick={handleDeleteVariable}
         className="p-0"
       >
-        <Trash2Icon size={16} />
+        {!isUsed && <Trash2Icon size={16} />}
       </Button>
     </div>
   );

web/src/features/playground/client/context/index.tsx

@@ -1,25 +1,30 @@
 import React, {
   createContext,
-  PropsWithChildren,
+  type PropsWithChildren,
   useCallback,
   useContext,
   useEffect,
   useState,
 } from "react";
 
-import { v4 as uuidv4 } from "uuid";
-
-import { MessagesContext } from "@/src/features/playground/client/components/Messages";
-import { ModelParamsContext } from "@/src/features/playground/client/components/ModelParameters";
+import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
+import type { ModelParamsContext } from "@/src/features/playground/client/components/ModelParameters";
 import useCommandEnter from "@/src/features/playground/client/hooks/useCommandEnter";
 import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
-  ChatMessageWithId,
+  type ChatMessageWithId,
   ModelProvider,
-  PromptVariable,
-  UIModelParams,
+  type PromptVariable,
+  type UIModelParams,
 } from "@langfuse/shared";
+import { createEmptyMessage } from "../utils/createEmptyMessage";
+import { StringParam, useQueryParam } from "use-query-params";
+import { api } from "@/src/utils/api";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { PromptType } from "@/src/features/prompts/server/validation";
+import { ChatMessageListSchema } from "@/src/features/prompts/components/NewPromptForm/validation";
+import { v4 as uuidv4 } from "uuid";
 
 type PlaygroundContextType = {
   promptVariables: PromptVariable[];
@@ -31,6 +36,7 @@ type PlaygroundContextType = {
 
   handleSubmit: () => Promise<void>;
   isStreaming: boolean;
+  isInitializing: boolean;
 } & ModelParamsContext &
   MessagesContext;
 
@@ -51,6 +57,8 @@ export const usePlaygroundContext = () => {
 export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
   children,
 }) => {
+  const projectId = useProjectIdFromURL();
+  const [initialPromptId] = useQueryParam("promptId", StringParam);
   const [promptVariables, setPromptVariables] = useState<PromptVariable[]>([]);
   const [output, setOutput] = useState("");
   const [outputJson, setOutputJson] = useState("");
@@ -63,6 +71,36 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     getDefaultModelParams(ModelProvider.OpenAI),
   );
 
+  const { data: initialPrompt, isInitialLoading } = api.prompts.byId.useQuery(
+    {
+      projectId,
+      id: initialPromptId ?? "",
+    },
+    { enabled: Boolean(initialPromptId), staleTime: Infinity }, // do not refetch as this would overwrite the user's input
+  );
+
+  useEffect(() => {
+    if (!initialPrompt) return;
+    if (initialPrompt.type === PromptType.Chat) {
+      try {
+        const initialMessages = ChatMessageListSchema.parse(
+          initialPrompt.prompt,
+        );
+        setMessages(initialMessages.map((m) => ({ ...m, id: uuidv4() })));
+      } catch (err) {
+        console.warn("Failed to parse initial chat messages", err);
+      }
+    } else {
+      const promptString = initialPrompt.prompt?.valueOf();
+      setMessages([
+        createEmptyMessage(
+          ChatMessageRole.System,
+          typeof promptString === "string" ? promptString : "",
+        ),
+      ]);
+    }
+  }, [initialPrompt]);
+
   useEffect(() => {
     setModelParams(getDefaultModelParams(modelParams.provider));
   }, [modelParams.provider]);
@@ -130,7 +168,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
 
         const finalMessages = getFinalMessages(promptVariables, messages);
         const leftOverVariables = extractVariables(
-          finalMessages.map((m) => m.content).join(""),
+          finalMessages.map((m) => m.content).join("\n"),
         );
 
         if (leftOverVariables.length > 0) {
@@ -181,7 +219,6 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
     <PlaygroundContext.Provider
       value={{
         promptVariables,
-        updatePromptVariables,
         updatePromptVariableValue,
         deletePromptVariable,
 
@@ -197,6 +234,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
         outputJson,
         handleSubmit,
         isStreaming,
+        isInitializing: isInitialLoading,
       }}
     >
       {children}
@@ -269,17 +307,6 @@ function getFinalMessages(
   return finalMessages;
 }
 
-function createEmptyMessage(
-  role: ChatMessageRole,
-  content?: string,
-): ChatMessageWithId {
-  return {
-    role,
-    content: content ?? "",
-    id: uuidv4(),
-  };
-}
-
 function getDefaultModelParams(provider: ModelProvider): UIModelParams {
   switch (provider) {
     // Docs: https://platform.openai.com/docs/api-reference/chat/create

web/src/features/playground/client/index.tsx

@@ -6,6 +6,10 @@ import { Messages } from "./components/Messages";
 export default function Playground() {
   const playgroundContext = usePlaygroundContext();
 
+  if (playgroundContext.isInitializing) {
+    return <div>Loading...</div>;
+  }
+
   return (
     <div className="flex h-full flex-row space-x-8">
       <div className="h-full basis-3/4 overflow-auto">

web/src/features/playground/client/utils/createEmptyMessage.ts

@@ -0,0 +1,13 @@
+import { v4 as uuidv4 } from "uuid";
+import type { ChatMessageRole, ChatMessageWithId } from "@langfuse/shared";
+
+export function createEmptyMessage(
+  role: ChatMessageRole,
+  content?: string,
+): ChatMessageWithId {
+  return {
+    role,
+    content: content ?? "",
+    id: uuidv4(),
+  };
+}

web/src/features/playground/server/chatCompletionHandler.ts

@@ -2,7 +2,7 @@ import { StreamingTextResponse } from "ai";
 import { getToken } from "next-auth/jwt";
 import { type NextRequest, NextResponse } from "next/server";
 
-import { fetchLLMCompletion } from "@langfuse/shared/src/server/llm/fetchLLMCompletion";
+import { fetchLLMCompletion } from "@langfuse/shared";
 
 import {
   validateChatCompletionBody,
@@ -44,6 +44,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
     messages,
     modelParams,
     streaming: true,
+    functionCall: undefined,
   });
 
   return new StreamingTextResponse(stream);

web/src/features/prompts/components/NewPromptForm/PromptChatMessages.tsx

@@ -0,0 +1,61 @@
+import { useEffect, useState } from "react";
+import type { ControllerRenderProps } from "react-hook-form";
+import { ChatMessages } from "@/src/features/playground/client/components/ChatMessages";
+import { ChatMessageRole } from "@langfuse/shared";
+import { createEmptyMessage } from "@/src/features/playground/client/utils/createEmptyMessage";
+import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
+import {
+  ChatMessageListSchema,
+  type NewPromptFormSchemaType,
+} from "./validation";
+import { v4 as uuidv4 } from "uuid";
+
+type PromptChatMessagesProps = ControllerRenderProps<
+  NewPromptFormSchemaType,
+  "chatPrompt"
+>;
+export const PromptChatMessages: React.FC<PromptChatMessagesProps> = ({
+  onChange,
+  value,
+}) => {
+  let initialMessages;
+  try {
+    if (value.length === 0) throw Error("Empty array");
+
+    initialMessages = ChatMessageListSchema.parse(value).map((message) => ({
+      ...message,
+      id: uuidv4(),
+    }));
+  } catch (err) {
+    initialMessages = [createEmptyMessage(ChatMessageRole.System)];
+  }
+
+  const [messages, setMessages] = useState(initialMessages);
+
+  const addMessage: MessagesContext["addMessage"] = (role, content) => {
+    const message = createEmptyMessage(role, content);
+    setMessages((prev) => [...prev, message]);
+
+    return message;
+  };
+
+  const updateMessage: MessagesContext["updateMessage"] = (id, key, value) => {
+    setMessages((prev) =>
+      prev.map((message) =>
+        message.id === id ? { ...message, [key]: value } : message,
+      ),
+    );
+  };
+
+  const deleteMessage: MessagesContext["deleteMessage"] = (id) => {
+    setMessages((prev) => prev.filter((message) => message.id !== id));
+  };
+
+  useEffect(() => {
+    onChange(messages);
+  }, [messages, onChange]);
+
+  return (
+    <ChatMessages {...{ messages, addMessage, deleteMessage, updateMessage }} />
+  );
+};

web/src/features/prompts/components/NewPromptForm/index.tsx

@@ -0,0 +1,350 @@
+import { capitalize } from "lodash";
+import router from "next/router";
+import { usePostHog } from "posthog-js/react";
+import { useState, useEffect } from "react";
+import { useForm } from "react-hook-form";
+import JsonView from "react18-json-view";
+
+import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
+import { Checkbox } from "@/src/components/ui/checkbox";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Tabs,
+  TabsContent,
+  TabsList,
+  TabsTrigger,
+} from "@/src/components/ui/tabs";
+import { Textarea } from "@/src/components/ui/textarea";
+import {
+  type CreatePromptTRPCType,
+  PromptType,
+} from "@/src/features/prompts/server/validation";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { api } from "@/src/utils/api";
+import { extractVariables } from "@/src/utils/string";
+import { jsonSchema } from "@/src/utils/zod";
+import { zodResolver } from "@hookform/resolvers/zod";
+import type { Prompt } from "@langfuse/shared/src/db";
+
+import { PromptChatMessages } from "./PromptChatMessages";
+import {
+  NewPromptFormSchema,
+  type NewPromptFormSchemaType,
+  PromptContentSchema,
+  type PromptContentType,
+} from "./validation";
+import { Input } from "@/src/components/ui/input";
+import Link from "next/link";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+type NewPromptFormProps = {
+  initialPrompt?: Prompt | null;
+  onFormSuccess?: () => void;
+};
+
+export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
+  const { onFormSuccess, initialPrompt } = props;
+  const projectId = useProjectIdFromURL();
+  const [formError, setFormError] = useState<string | null>(null);
+  const utils = api.useUtils();
+  const posthog = usePostHog();
+
+  let initialPromptContent: PromptContentType | null;
+  try {
+    initialPromptContent = PromptContentSchema.parse({
+      type: initialPrompt?.type,
+      prompt: initialPrompt?.prompt?.valueOf(),
+    });
+  } catch (err) {
+    initialPromptContent = null;
+  }
+
+  const defaultValues: NewPromptFormSchemaType = {
+    type: initialPromptContent?.type ?? PromptType.Text,
+    chatPrompt:
+      initialPromptContent?.type === PromptType.Chat
+        ? initialPromptContent?.prompt
+        : [],
+    textPrompt:
+      initialPromptContent?.type === PromptType.Text
+        ? initialPromptContent?.prompt
+        : "",
+    name: initialPrompt?.name ?? "",
+    config: JSON.stringify(initialPrompt?.config?.valueOf()) || "{}",
+    isActive: false,
+  };
+
+  const form = useForm<NewPromptFormSchemaType>({
+    resolver: zodResolver(NewPromptFormSchema),
+    defaultValues,
+  });
+
+  const currentName = form.watch("name");
+  const currentType = form.watch("type");
+  const currentIsActive = form.watch("isActive");
+  const currentExtractedVariables = extractVariables(
+    currentType === PromptType.Text
+      ? form.watch("textPrompt")
+      : JSON.stringify(form.watch("chatPrompt"), null, 2),
+  );
+
+  const createPromptMutation = api.prompts.create.useMutation({
+    onSuccess: () => utils.prompts.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  const allPrompts = api.prompts.all.useQuery({ projectId }).data;
+
+  function onSubmit(values: NewPromptFormSchemaType) {
+    posthog.capture("prompts:new_prompt_form_submit");
+
+    const { type, textPrompt, chatPrompt } = values;
+
+    // TS does not narrow down type of 'prompt' property given the type of 'type' property in ternary operator
+    let newPrompt: CreatePromptTRPCType;
+    if (type === PromptType.Chat) {
+      newPrompt = {
+        ...values,
+        projectId,
+        type,
+        prompt: chatPrompt,
+        config: JSON.parse(values.config),
+      };
+    } else {
+      newPrompt = {
+        ...values,
+        projectId,
+        type,
+        prompt: textPrompt,
+        config: JSON.parse(values.config),
+      };
+    }
+
+    createPromptMutation
+      .mutateAsync(newPrompt)
+      .then((newPrompt) => {
+        onFormSuccess?.();
+        form.reset();
+        void router.push(
+          `/project/${projectId}/prompts/${encodeURIComponent(newPrompt.name)}`,
+        );
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  }
+
+  useEffect(() => {
+    const isNewPrompt = !allPrompts
+      ?.map((prompt) => prompt.name)
+      .includes(currentName);
+
+    if (!isNewPrompt) {
+      form.setError("name", { message: "Prompt name already exist." });
+    } else {
+      form.clearErrors("name");
+    }
+  }, [currentName, allPrompts, form]);
+
+  return (
+    <Form {...form}>
+      <form
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="flex flex-col gap-6"
+      >
+        {/* Prompt name field - text vs. chat only for new prompts */}
+        {!initialPrompt ? (
+          <FormField
+            control={form.control}
+            name="name"
+            render={({ field }) => {
+              const errorMessage = form.getFieldState("name").error?.message;
+
+              return (
+                <div>
+                  <FormItem>
+                    <FormLabel>Name</FormLabel>
+                    <FormControl>
+                      <Input placeholder="Select a prompt name" {...field} />
+                    </FormControl>
+                    {/* Custom form message to include a link to the already existing prompt */}
+                    {form.getFieldState("name").error ? (
+                      <div className="flex flex-row space-x-1 text-sm font-medium text-destructive">
+                        <p className="text-sm font-medium text-destructive">
+                          {errorMessage}
+                        </p>
+                        {errorMessage?.includes("already exist") ? (
+                          <Link
+                            href={`/project/${projectId}/prompts/${currentName.trim()}`}
+                            className="flex flex-row"
+                          >
+                            Create a new version for it here.{" "}
+                            <ArrowTopRightIcon />
+                          </Link>
+                        ) : null}
+                      </div>
+                    ) : null}
+                  </FormItem>
+                </div>
+              );
+            }}
+          />
+        ) : null}
+
+        {/* Prompt content field - text vs. chat */}
+        <>
+          <FormItem>
+            <FormLabel>Prompt</FormLabel>
+            <Tabs
+              value={form.watch("type")}
+              onValueChange={(e) => {
+                form.setValue("type", e as PromptType);
+              }}
+            >
+              {!initialPrompt ? (
+                <TabsList className="flex w-full">
+                  <TabsTrigger
+                    disabled={
+                      Boolean(initialPromptContent) &&
+                      initialPromptContent?.type !== PromptType.Text
+                    }
+                    className="flex-1"
+                    value={PromptType.Text}
+                  >
+                    {capitalize(PromptType.Text)}
+                  </TabsTrigger>
+                  <TabsTrigger
+                    disabled={
+                      Boolean(initialPromptContent) &&
+                      initialPromptContent?.type !== PromptType.Chat
+                    }
+                    className="flex-1"
+                    value={PromptType.Chat}
+                  >
+                    {capitalize(PromptType.Chat)}
+                  </TabsTrigger>
+                </TabsList>
+              ) : null}
+              <TabsContent value={PromptType.Text}>
+                <FormField
+                  control={form.control}
+                  name="textPrompt"
+                  render={({ field }) => (
+                    <>
+                      <FormControl>
+                        <Textarea
+                          {...field}
+                          className="min-h-[200px] flex-1 font-mono text-xs"
+                        />
+                      </FormControl>
+                      <FormMessage />
+                    </>
+                  )}
+                />
+              </TabsContent>
+              <TabsContent value={PromptType.Chat}>
+                <FormField
+                  control={form.control}
+                  name="chatPrompt"
+                  render={({ field }) => (
+                    <>
+                      <PromptChatMessages {...field} />
+                      <FormMessage />
+                    </>
+                  )}
+                />
+              </TabsContent>
+            </Tabs>
+          </FormItem>
+          <p className="text-sm text-gray-500">
+            You can use <code className="text-xs">{"{{variable}}"}</code> to
+            insert variables into your prompt.
+            {currentExtractedVariables.length > 0
+              ? " The following variables are available:"
+              : ""}
+          </p>
+          <div className="flex min-h-6 flex-wrap gap-2">
+            {currentExtractedVariables.map((variable) => (
+              <Badge key={variable} variant="outline">
+                {variable}
+              </Badge>
+            ))}
+          </div>
+        </>
+
+        {/* Prompt Config field */}
+        <FormField
+          control={form.control}
+          name="config"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Config</FormLabel>
+              <JsonView
+                src={jsonSchema.parse(JSON.parse(field.value))}
+                onEdit={(edit) => {
+                  field.onChange(JSON.stringify(edit.src));
+                }}
+                editable
+                className="rounded-md border border-gray-200 p-2 text-sm"
+              />
+              <FormDescription>
+                Track configs for LLM API calls such as function definitions or
+                LLM parameters.
+              </FormDescription>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+
+        {/* Activate prompt field */}
+        <FormField
+          control={form.control}
+          name="isActive"
+          render={({ field }) => (
+            <FormItem className="flex flex-row items-center space-x-3 space-y-0 rounded-md border p-4">
+              <FormControl>
+                <Checkbox
+                  checked={field.value}
+                  onCheckedChange={field.onChange}
+                />
+              </FormControl>
+              <div className="space-y-1 leading-none">
+                <FormLabel>Activate prompt</FormLabel>
+              </div>
+              {currentIsActive ? (
+                <div className="text-xs text-gray-500">
+                  Activating the prompt will make it available to the SDKs
+                  immediately.
+                </div>
+              ) : null}
+            </FormItem>
+          )}
+        />
+        <Button
+          type="submit"
+          loading={createPromptMutation.isLoading}
+          className="w-full"
+          disabled={Boolean(
+            !initialPrompt && form.formState.errors.name?.message,
+          )} // Disable button if prompt name already exists. Check is dynamic and not part of zod schema
+        >
+          Create prompt
+        </Button>
+      </form>
+      {formError && (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      )}
+    </Form>
+  );
+};

web/src/features/prompts/components/NewPromptForm/validation.ts

@@ -0,0 +1,79 @@
+import { z } from "zod";
+import { extractVariables } from "@/src/utils/string";
+import { PromptType } from "@/src/features/prompts/server/validation";
+import { ChatMessageRole } from "@langfuse/shared";
+
+const ChatMessageSchema = z.object({
+  role: z.nativeEnum(ChatMessageRole),
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z
+  .string()
+  .min(1, "Enter a prompt")
+  .refine(
+    validateVariables,
+    "Variables must only contain letters and underscores (_)",
+  );
+
+const NewPromptBaseSchema = z.object({
+  name: z.string().min(1, "Enter a name"),
+  isActive: z.boolean({
+    required_error: "Enter whether the prompt should go live",
+  }),
+  config: z.string().refine(validateJson, "Config needs to be valid JSON"),
+});
+
+const NewChatPromptSchema = NewPromptBaseSchema.extend({
+  type: z.literal(PromptType.Chat),
+  chatPrompt: ChatMessageListSchema.refine(
+    (messages) => messages.every((message) => message.content.length > 0),
+    "Enter a chat message or remove the empty message",
+  ).refine(
+    (messages) => validateVariables(messages.map((m) => m.content).join("\n")),
+    "Variables must only contain letters and underscores (_)",
+  ),
+  textPrompt: z.string(),
+});
+
+const NewTextPromptSchema = NewPromptBaseSchema.extend({
+  type: z.literal(PromptType.Text),
+  chatPrompt: z.array(z.any()),
+  textPrompt: TextPromptSchema,
+});
+
+export const NewPromptFormSchema = z.union([
+  NewChatPromptSchema,
+  NewTextPromptSchema,
+]);
+export type NewPromptFormSchemaType = z.infer<typeof NewPromptFormSchema>;
+
+export const PromptContentSchema = z.union([
+  z.object({
+    type: z.literal(PromptType.Chat),
+    prompt: ChatMessageListSchema,
+  }),
+  z.object({
+    type: z.literal(PromptType.Text),
+    prompt: z.string(),
+  }),
+]);
+export type PromptContentType = z.infer<typeof PromptContentSchema>;
+
+function validateVariables(content: string): boolean {
+  const variables = extractVariables(content);
+  const charOrUnderscore = /^[A-Za-z_]+$/;
+
+  return variables.every((variable) => charOrUnderscore.test(variable));
+}
+
+function validateJson(content: string): boolean {
+  try {
+    JSON.parse(content);
+
+    return true;
+  } catch (e) {
+    return false;
+  }
+}

web/src/features/prompts/components/new-prompt-button.tsx

@@ -1,318 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import {
-  Dialog,
-  DialogContent,
-  DialogHeader,
-  DialogTitle,
-} from "@/src/components/ui/dialog";
-import { useState } from "react";
-import { DialogTrigger } from "@radix-ui/react-dialog";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import {
-  FormField,
-  FormItem,
-  FormLabel,
-  FormControl,
-  FormMessage,
-  Form,
-  FormDescription,
-} from "@/src/components/ui/form";
-import { api } from "@/src/utils/api";
-import { zodResolver } from "@hookform/resolvers/zod";
-
-import { usePostHog } from "posthog-js/react";
-import { z } from "zod";
-import { useForm } from "react-hook-form";
-import { Textarea } from "@/src/components/ui/textarea";
-import { Checkbox } from "@/src/components/ui/checkbox";
-import { extractVariables } from "@/src/utils/string";
-import { Badge } from "@/src/components/ui/badge";
-import router from "next/router";
-import { AutoComplete } from "@/src/features/prompts/components/auto-complete";
-import { type AutoCompleteOption } from "@/src/features/prompts/components/auto-complete";
-import JsonView from "react18-json-view";
-import { jsonSchema } from "@/src/utils/zod";
-
-export const CreatePromptDialog = (props: {
-  projectId: string;
-  title: string;
-  promptName?: string;
-  promptText?: string;
-  subtitle?: string;
-  promptConfig?: z.infer<typeof jsonSchema>;
-  children?: React.ReactNode;
-}) => {
-  const [open, setOpen] = useState(false);
-
-  const hasAccess = useHasAccess({
-    projectId: props.projectId,
-    scope: "datasets:CUD",
-  });
-
-  return (
-    <Dialog open={hasAccess && open} onOpenChange={setOpen}>
-      <DialogTrigger asChild>{props.children}</DialogTrigger>
-      <DialogContent className="max-h-screen overflow-auto sm:max-w-3xl">
-        <DialogHeader>
-          <DialogTitle className="mb-5">
-            {props.title}
-            {props.subtitle ? (
-              <p className="mt-3 text-sm	font-normal">{props.subtitle}</p>
-            ) : null}
-          </DialogTitle>
-        </DialogHeader>
-        <NewPromptForm
-          projectId={props.projectId}
-          promptName={props.promptName}
-          promptText={props.promptText}
-          promptConfig={props.promptConfig}
-          onFormSuccess={() => setOpen(false)}
-        />
-      </DialogContent>
-    </Dialog>
-  );
-};
-
-const formSchema = z.object({
-  name: z.string().min(1, "Enter a name"),
-  prompt: z
-    .string()
-    .min(1, "Enter a prompt")
-    .refine((val) => {
-      const variables = extractVariables(val);
-      const matches = variables.map((variable) => {
-        // check regex here
-        if (variable.match(/^[A-Za-z_]+$/)) {
-          return true;
-        }
-        return false;
-      });
-      return !matches.includes(false);
-    }, "Variables must only contain letters and underscores (_)"),
-  isActive: z.boolean({
-    required_error: "Enter whether the prompt should go live",
-  }),
-  // string as we keep the state in string to avoid recursive zod parsing issues
-  config: z.string().refine(
-    (value) => {
-      try {
-        JSON.parse(value);
-        return true;
-      } catch (e) {
-        return false;
-      }
-    },
-    {
-      message: "Config needs to be valid JSON",
-    },
-  ),
-});
-
-export const NewPromptForm = (props: {
-  projectId: string;
-  onFormSuccess?: () => void;
-  promptName?: string;
-  promptText?: string;
-  promptConfig?: z.infer<typeof jsonSchema>;
-}) => {
-  const [formError, setFormError] = useState<string | null>(null);
-
-  const posthog = usePostHog();
-
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
-    defaultValues: {
-      isActive: false,
-      name: props.promptName ?? "",
-      prompt: props.promptText ?? "",
-      config: props.promptConfig ? JSON.stringify(props.promptConfig) : "{}",
-    },
-  });
-
-  const prompts = api.prompts.all.useQuery({
-    projectId: props.projectId,
-  });
-
-  const utils = api.useUtils();
-
-  const createPromptMutation = api.prompts.create.useMutation({
-    onSuccess: () => utils.prompts.invalidate(),
-    onError: (error) => setFormError(error.message),
-  });
-
-  const comboboxOptions =
-    prompts.data
-      ?.map((prompt) => {
-        return { label: prompt.name, value: prompt.name };
-      })
-      .filter(
-        (prompt, i, arr) =>
-          arr.findIndex((t) => t.label === prompt.label) === i,
-      ) ?? [];
-
-  const currentName = form.watch("name");
-
-  const matchingOptions = comboboxOptions.filter((option) =>
-    option.label.toLowerCase().includes(currentName.toLowerCase()),
-  );
-
-  const extractedVariables = extractVariables(form.watch("prompt"));
-  const promptIsActivated = form.watch("isActive");
-
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    posthog.capture("prompts:new_prompt_form_submit");
-
-    createPromptMutation
-      .mutateAsync({
-        ...values,
-        projectId: props.projectId,
-        name: values.name,
-        prompt: values.prompt,
-        isActive: values.isActive,
-        // we keep the config in state as string. need to convert it to JSON before sending it to the API
-        // zod parsing necessary to align with TRPC schema
-        config: jsonSchema.parse(JSON.parse(values.config)),
-      })
-      .then((newPrompt) => {
-        props.onFormSuccess?.();
-        form.reset();
-        // go to the following page after creating the prompt
-        void router.push(
-          `/project/${props.projectId}/prompts/${encodeURIComponent(newPrompt.name)}`,
-        );
-      })
-      .catch((error) => {
-        console.error(error);
-      });
-  }
-
-  return (
-    <Form {...form}>
-      <form
-        // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        onSubmit={form.handleSubmit(onSubmit)}
-        className="flex flex-col gap-6"
-      >
-        <FormField
-          control={form.control}
-          name="name"
-          render={({ field }) => {
-            const setNameValue = (value: AutoCompleteOption) => {
-              field.onChange(value.value);
-            };
-
-            return (
-              <div>
-                <FormItem>
-                  <FormLabel>Name</FormLabel>
-                  <FormControl>
-                    <AutoComplete
-                      {...field}
-                      options={matchingOptions}
-                      placeholder="Select a prompt name"
-                      onValueChange={setNameValue}
-                      value={{ value: field.value, label: field.value }}
-                      disabled={false}
-                      createLabel="Create a new prompt name"
-                    />
-                  </FormControl>
-                  <FormMessage />
-                </FormItem>
-              </div>
-            );
-          }}
-        />
-
-        <FormField
-          control={form.control}
-          name="prompt"
-          render={({ field }) => (
-            <>
-              <FormItem>
-                <FormLabel>Prompt</FormLabel>
-                <FormControl>
-                  <Textarea
-                    {...field}
-                    className="min-h-[150px] flex-1 font-mono text-xs"
-                  />
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-              <p className="text-sm text-gray-500">
-                You can use <code className="text-xs">{"{{variable}}"}</code> to
-                insert variables into your prompt. The following variables are
-                available:
-              </p>
-              <div className="flex flex-wrap gap-2">
-                {extractedVariables.map((variable) => (
-                  <Badge key={variable} variant="outline">
-                    {variable}
-                  </Badge>
-                ))}
-              </div>
-            </>
-          )}
-        />
-        <FormField
-          control={form.control}
-          name="config"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Config</FormLabel>
-              <JsonView
-                // need to convert string in state to JSON for the JSONView component
-                src={jsonSchema.parse(JSON.parse(field.value))}
-                onEdit={(edit) => {
-                  // need to put string back into the state
-                  field.onChange(JSON.stringify(edit.src));
-                }}
-                editable
-                className="rounded-md border border-gray-200 p-2 text-sm"
-              />
-              <FormDescription>
-                Track configs for LLM API calls such as function definitions or
-                LLM parameters.
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <FormField
-          control={form.control}
-          name="isActive"
-          render={({ field }) => (
-            <FormItem className="flex flex-row items-center space-x-3 space-y-0 rounded-md border p-4">
-              <FormControl>
-                <Checkbox
-                  checked={field.value}
-                  onCheckedChange={field.onChange}
-                />
-              </FormControl>
-              <div className="space-y-1 leading-none">
-                <FormLabel>Activate prompt</FormLabel>
-              </div>
-              {promptIsActivated ? (
-                <div className="text-xs text-gray-500">
-                  Activating the prompt will make it available to the SDKs
-                  immediately.
-                </div>
-              ) : null}
-            </FormItem>
-          )}
-        />
-        <Button
-          type="submit"
-          loading={createPromptMutation.isLoading}
-          className="w-full"
-        >
-          Create prompt
-        </Button>
-      </form>
-      {formError ? (
-        <p className="text-red text-center">
-          <span className="font-bold">Error:</span> {formError}
-        </p>
-      ) : null}
-    </Form>
-  );
-};

web/src/features/prompts/components/prompt-detail.tsx

@@ -1,43 +1,62 @@
+import { Pencil, Terminal } from "lucide-react";
+import Link from "next/link";
+import router, { useRouter } from "next/router";
+import { NumberParam, useQueryParam } from "use-query-params";
+import type { z } from "zod";
+
 import Header from "@/src/components/layouts/header";
+import {
+  ChatMlArraySchema,
+  OpenAiMessageView,
+} from "@/src/components/trace/IOPreview";
 import { Badge } from "@/src/components/ui/badge";
 import { Button } from "@/src/components/ui/button";
-import { CodeView } from "@/src/components/ui/code";
+import { CodeView, JSONView } from "@/src/components/ui/code";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
-import { CreatePromptDialog } from "@/src/features/prompts/components/new-prompt-button";
+import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
+import { PromotePrompt } from "@/src/features/prompts/components/promote-prompt";
+import { PromptType } from "@/src/features/prompts/server/validation";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
 import { extractVariables } from "@/src/utils/string";
 import { type Prompt } from "@langfuse/shared/src/db";
-import { Pencil } from "lucide-react";
-import { PromptHistoryNode } from "./prompt-history";
-import { PromotePrompt } from "@/src/features/prompts/components/promote-prompt";
 import { ScrollArea } from "@radix-ui/react-scroll-area";
-import { useQueryParam, NumberParam } from "use-query-params";
-import router from "next/router";
-import { JSONView } from "@/src/components/ui/code";
-import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
-import { jsonSchema } from "@/src/utils/zod";
 
-export type PromptDetailProps = {
-  projectId: string;
-  promptName: string;
-};
+import { PromptHistoryNode } from "./prompt-history";
+import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";
 
-export const PromptDetail = (props: PromptDetailProps) => {
+export const PromptDetail = () => {
+  const projectId = useProjectIdFromURL();
+  const isPlaygroundEnabled = useIsFeatureEnabled("playground");
+  const promptName = decodeURIComponent(useRouter().query.promptName as string);
   const [currentPromptVersion, setCurrentPromptVersion] = useQueryParam(
     "version",
     NumberParam,
   );
   const promptHistory = api.prompts.allVersions.useQuery({
-    name: props.promptName,
-    projectId: props.projectId,
+    name: promptName,
+    projectId,
   });
   const prompt = currentPromptVersion
     ? promptHistory.data?.find(
         (prompt) => prompt.version === currentPromptVersion,
       )
     : promptHistory.data?.[0];
-  const extractedVariables = prompt ? extractVariables(prompt.prompt) : [];
+
+  const extractedVariables = prompt
+    ? extractVariables(JSON.stringify(prompt.prompt))
+    : [];
+
+  let chatMessages: z.infer<typeof ChatMlArraySchema> | null = null;
+  try {
+    chatMessages = ChatMlArraySchema.parse(prompt?.prompt);
+  } catch (error) {
+    console.warn(
+      "Could not parse returned chat prompt to pretty ChatML",
+      error,
+    );
+  }
 
   if (!promptHistory.data || !prompt) {
     return <div>Loading...</div>;
@@ -52,45 +71,56 @@ export const PromptDetail = (props: PromptDetailProps) => {
             breadcrumb={[
               {
                 name: "Prompts",
-                href: `/project/${props.projectId}/prompts/`,
+                href: `/project/${projectId}/prompts/`,
               },
               {
                 name: prompt.name,
-                href: `/project/${props.projectId}/prompts/${encodeURIComponent(props.promptName)}`,
+                href: `/project/${projectId}/prompts/${encodeURIComponent(promptName)}`,
               },
               { name: `Version ${prompt.version}` },
             ]}
             actionButtons={
               <>
                 <PromotePrompt
-                  projectId={props.projectId}
+                  projectId={projectId}
                   promptId={prompt.id}
                   promptName={prompt.name}
                   disabled={prompt.isActive}
                   variant="outline"
                 />
-                <CreatePromptDialog
-                  projectId={props.projectId}
-                  title="Update Prompt"
-                  subtitle="We do not update prompts, instead we create a new version of the prompt."
-                  promptName={prompt.name}
-                  promptText={prompt.prompt}
-                  promptConfig={jsonSchema.parse(prompt.config)}
+
+                {isPlaygroundEnabled ? (
+                  <Link
+                    href={`/project/${projectId}/playground?promptId=${encodeURIComponent(prompt.id)}`}
+                  >
+                    <Button
+                      variant="outline"
+                      title="Test in prompt playground"
+                      size="icon"
+                    >
+                      <Terminal className="h-5 w-5" />
+                    </Button>
+                  </Link>
+                ) : null}
+
+                <Link
+                  href={`/project/${projectId}/prompts/new?promptId=${encodeURIComponent(prompt.id)}`}
                 >
                   <Button variant="outline" size="icon">
                     <Pencil className="h-5 w-5" />
                   </Button>
-                </CreatePromptDialog>
+                </Link>
+
                 <DeletePromptVersion
-                  projectId={props.projectId}
+                  projectId={projectId}
                   promptVersionId={prompt.id}
                   version={prompt.version}
                   countVersions={promptHistory.data.length}
                 />
                 <DetailPageNav
                   key="nav"
-                  currentId={props.promptName}
-                  path={(name) => `/project/${props.projectId}/prompts/${name}`}
+                  currentId={promptName}
+                  path={(name) => `/project/${projectId}/prompts/${name}`}
                   listKey="prompts"
                 />
               </>
@@ -98,7 +128,13 @@ export const PromptDetail = (props: PromptDetailProps) => {
           />
         </div>
         <div className="col-span-2 md:h-full">
-          <CodeView content={prompt.prompt} title="Prompt" />
+          {prompt.type === PromptType.Chat && chatMessages ? (
+            <OpenAiMessageView title="Chat prompt" messages={chatMessages} />
+          ) : typeof prompt.prompt === "string" ? (
+            <CodeView content={prompt.prompt} title="Text prompt" />
+          ) : (
+            <JSONView json={prompt.prompt} title="Prompt" />
+          )}
           <div className="mx-auto mt-5 w-full rounded-lg border text-base leading-7">
             <div className="border-b px-3 py-1 text-xs font-medium">
               Variables

web/src/features/prompts/components/prompt-new.tsx

@@ -0,0 +1,71 @@
+import { StringParam, useQueryParam } from "use-query-params";
+
+import Header from "@/src/components/layouts/header";
+import { NewPromptForm } from "@/src/features/prompts/components/NewPromptForm";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { api } from "@/src/utils/api";
+
+export const NewPrompt = () => {
+  const projectId = useProjectIdFromURL();
+  const [initialPromptId] = useQueryParam("promptId", StringParam);
+
+  const { data: initialPrompt, isInitialLoading } = api.prompts.byId.useQuery(
+    {
+      projectId,
+      id: initialPromptId ?? "",
+    },
+    { enabled: Boolean(initialPromptId) },
+  );
+
+  if (isInitialLoading) {
+    return <div>Loading...</div>;
+  }
+
+  const breadcrumb: { name: string; href?: string }[] = [
+    {
+      name: "Prompts",
+      href: `/project/${projectId}/prompts/`,
+    },
+    {
+      name: "New prompt",
+    },
+  ];
+
+  if (initialPrompt) {
+    breadcrumb.pop(); // Remove "New prompt"
+    breadcrumb.push(
+      {
+        name: initialPrompt.name,
+        href: `/project/${projectId}/prompts/${encodeURIComponent(initialPrompt.name)}`,
+      },
+      { name: "New version" },
+    );
+  }
+
+  return (
+    <div className="xl:container">
+      <Header
+        title={
+          initialPrompt
+            ? `${initialPrompt.name} \u2014 New version`
+            : "Create new prompt"
+        }
+        help={{
+          description:
+            "Manage and version your prompts in Langfuse. Edit and update them via the UI and SDK. Retrieve the production version via the SDKs. Learn more in the docs.",
+          href: "https://langfuse.com/docs/prompts",
+        }}
+        breadcrumb={breadcrumb}
+      />
+      {initialPrompt ? (
+        <p className="text-sm text-gray-500">
+          Prompts are immutable in Langfuse. To update a prompt, create a new
+          version.
+        </p>
+      ) : null}
+      <div className="my-8 max-w-screen-md">
+        <NewPromptForm {...{ initialPrompt }} />
+      </div>
+    </div>
+  );
+};