chore: release v2.37.2

Fixes #1787

.env.prod.example

@@ -76,6 +76,9 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
+# Disable logging to `events` table in db
+# ENABLE_EVENT_LOG=false
+
 
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments

CONTRIBUTING.md

@@ -53,53 +53,63 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+
 ```mermaid
 flowchart TB
-   subgraph s4["Clients"]
-      subgraph s2["langfuse/langfuse-python"]
-         Python["Python low-level SDK"]
-         Decorator["observe() decorator"] -->|extends| Python
-         OAI["OpenAI drop-in replacement"] -->|extends| Python
-         Llamaindex["LlamaIndex Integration"] -->|extends| Python
-         LCPYTHON["Langchain Python Integration"] -->|extends| Python
-         Langflow -->|uses| LCPYTHON
-         LiteLLM -->|uses| Python
-      end
-      subgraph s3["langfuse/langfuse-js"]
-         JS["JS SDK"]
-         LCJS["Langchain JS Integration"]  -->|extends| JS
-         Flowise -->|uses| LCJS
-      end
-   end
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
+    end
 
-   DB[Postgres Database]
-	subgraph s1["Application (langfuse/langfuse)"]
-      API[Public HTTP API]
-      G[TRPC API]
-      I[NextAuth]
-      H[React Frontend]
-      Prisma[Prisma ORM]
-      H --> G
-      H --> I
-      G --> I
-      G --- Prisma
-      API --- Prisma
-      I --- Prisma
-	end
-   Prisma --- DB
-   JS --- API
-   Python --- API
+    DB[Postgres Database]
+    Redis[Redis]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        Prisma[Prisma ORM]
+        H --> G
+        H --> I
+        G --> I
+        G --- Prisma
+        API --- Prisma
+        I --- Prisma
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker_API[Public HTTP API]
+    end
+
+    API --> Worker_API
+    Worker_API --- DB
+    Worker_API --- Redis
+
+    Prisma --- DB
+    JS --- API
+    Python --- API
 ```
 
-### Database Overview
+### Network Overview
 
-The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
-
-Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
-
-<img src="./packages/shared/prisma/database.svg">
-
-### Infrastructure & Network Overview
+> [!NOTE]
+> This will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
 
 ```mermaid
 flowchart LR
@@ -114,6 +124,14 @@ flowchart LR
    App --- DB
 ```
 
+### Database Overview
+
+The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
+
+Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
+
+<img src="./packages/shared/prisma/database.svg">
+
 ## Repository Structure
 
 We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https://turbo.build/repo/docs) to manage the dependencies and build process. The monorepo contains the following packages:
@@ -135,7 +153,7 @@ Requirements
 
 **Steps**
 
-1. Fork the the repository and clone it locally
+1. Fork the repository and clone it locally
 2. Run the development database
 
    ```bash

README.md

@@ -26,7 +26,7 @@
    <br/>
    <br/>
    <div>
-      <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
+      <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
       <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse" alt="langfuse npm package"></a>
@@ -37,17 +37,22 @@
 
 ## Overview
 
+_Unmute video for voice-over_
+
+https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+
 ### Develop
 
 - **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
-- **Prompts:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts))
+- **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
+- **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
 - **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
 - **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals))
+  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
   - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 

fern/apis/server/definition/api.yml

@@ -1,11 +1,17 @@
 name: langfuse
 docs: |
   ## Authentication
-  Authenticate with the API using Basic Auth, get API keys in the project settings:
+
+  Authenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:
 
   - username: Langfuse Public Key
   - password: Langfuse Secret Key
 
+  ## Exports
+
+  - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
+  - Postman collection: https://cloud.langfuse.com/postman-collection.json
+
 error-discrimination:
   strategy: status-code
 auth: basic

fern/apis/server/definition/prompts.yml

@@ -4,18 +4,42 @@ imports:
   pagination: ./utils/pagination.yml
 service:
   auth: true
-  base-path: /api/public
+  base-path: /api/public/v2
   endpoints:
     get:
       docs: Get a prompt
       method: GET
+      path: /prompts/{promptName}
+      path-parameters:
+        promptName:
+          type: string
+          docs: The name of the prompt
+      request:
+        name: GetPromptRequest
+        query-parameters:
+          version:
+            type: optional<integer>
+            docs: Version of the prompt to be retrieved.
+          label:
+            type: optional<string>
+            docs: Label of the prompt to be retrieved. Defaults to "production" if no label or version is set.
+      response: Prompt
+
+    list:
+      docs: Get a list of prompt names with versions and labels
+      method: GET
       path: /prompts
       request:
-        name: GetParameterRequest
+        name: ListPromptsMetaRequest
         query-parameters:
-          name: string
-          version: optional<integer>
-      response: Prompt
+          name: optional<string>
+          label: optional<string>
+          tag: optional<string>
+          page: optional<integer>
+          limit: optional<integer>
+
+      response: PromptMetaListResponse
+
     create:
       docs: Create a prompt
       method: POST
@@ -24,6 +48,18 @@ service:
       response: Prompt
 
 types:
+  PromptMetaListResponse:
+    properties:
+      data: list<PromptMeta>
+      meta: pagination.MetaResponse
+
+  PromptMeta:
+    properties:
+      name: string
+      versions: list<integer>
+      labels: list<string>
+      tags: list<string>
+
   CreatePromptRequest:
     union:
       chat: CreateChatPromptRequest
@@ -32,20 +68,16 @@ types:
   CreateChatPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: list<ChatMessage>
       config: optional<unknown>
+      labels: optional<list<string>>
 
   CreateTextPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: string
       config: optional<unknown>
+      labels: optional<list<string>>
 
   Prompt:
     union:
@@ -57,6 +89,7 @@ types:
       name: string
       version: integer
       config: unknown
+      labels: list<string>
 
   ChatMessage:
     properties:

fern/apis/server/generators.yml

@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../generated/postman
+          path: ../../../web/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.32.0",
+  "version": "2.37.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/prisma/generated/types.ts

@@ -334,9 +334,10 @@ export type Prompt = {
     name: string;
     version: number;
     type: Generated<string>;
-    is_active: boolean;
+    is_active: boolean | null;
     config: Generated<unknown>;
     tags: Generated<string[]>;
+    labels: Generated<string[]>;
 };
 export type Score = {
     id: string;

packages/shared/prisma/migrations/20240424150909_add_job_execution_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_status_idx" ON "job_executions"("project_id", "status");

packages/shared/prisma/migrations/20240429124411_add_prompt_version_labels/migration.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+-- Step 1: Alter the 'prompts' table to add the 'labels' column
+ALTER TABLE prompts
+ADD COLUMN labels TEXT[] DEFAULT ARRAY[]::TEXT[];
+
+-- Step 2: Update the 'labels' column to include 'production' for active prompts
+UPDATE prompts
+SET labels = array_append(labels, 'production')
+WHERE is_active = TRUE;
+
+-- Step 3: Drop the required constraint on 'is_active' column.
+ALTER TABLE prompts
+ALTER COLUMN is_active DROP NOT NULL;
+
+COMMIT;

packages/shared/prisma/migrations/20240429194411_add_latest_prompt_tag/migration.sql

@@ -0,0 +1,15 @@
+UPDATE
+	prompts
+SET
+	labels = array_append(labels, 'latest')
+WHERE
+	id = (
+		SELECT
+			id
+		FROM
+			prompts AS p2
+		WHERE
+			p2.name = prompts.name
+		ORDER BY
+			created_at DESC
+		LIMIT 1);

packages/shared/prisma/schema.prisma

@@ -553,9 +553,10 @@ model Prompt {
     name        String
     version     Int
     type        String        @default("text")
-    isActive    Boolean       @map("is_active")
+    isActive    Boolean?      @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
     config      Json          @default("{}")
     tags        String[]      @default([])
+    labels      String[]      @default([])
     Observation Observation[]
 
     @@unique([projectId, name, version])
@@ -699,6 +700,7 @@ model JobExecution {
     jobOutputScoreId String? @map("job_output_score_id")
     score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
+    @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
     @@map("job_executions")

packages/shared/prisma/seed.ts

@@ -72,7 +72,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      isActive: true,
+      labels: ["production"],
       version: 1,
       createdBy: "user-1",
     },
@@ -764,7 +764,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -773,7 +773,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -782,7 +782,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -791,7 +791,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      isActive: true,
+      labels: ["production"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -812,7 +812,7 @@ async function generatePrompts(project: Project) {
         prompt: prompt.prompt,
         name: prompt.name,
         version: prompt.version,
-        isActive: prompt.isActive,
+        labels: prompt.labels,
         tags: prompt.tags,
       },
       update: {
@@ -833,7 +833,6 @@ async function generatePrompts(project: Project) {
         temperature: 0.7,
       },
       version: 1,
-      isActive: false,
     },
     {
       id: `prompt-${v4()}`,
@@ -846,7 +845,7 @@ async function generatePrompts(project: Project) {
         topP: 0.9,
       },
       version: 2,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -860,7 +859,6 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
-      isActive: false,
     },
   ];
 
@@ -881,7 +879,7 @@ async function generatePrompts(project: Project) {
         name: version.name,
         config: version.config,
         version: version.version,
-        isActive: version.isActive,
+        labels: version.labels,
       },
       update: {
         id: version.id,
@@ -910,7 +908,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        isActive: i === 20,
+        labels: i === 20 ? ["production"] : [],
       },
       update: {
         id: promptId,

packages/shared/src/errors/ApiError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class ApiError extends BaseError {
+  constructor(description = "Api call failed") {
+    super("ApiError", 500, description, true);
+  }
+}

packages/shared/src/errors/BaseError.ts

@@ -7,7 +7,7 @@ export class BaseError extends Error {
     name: string,
     httpCode: number,
     description: string,
-    isOperational: boolean,
+    isOperational: boolean
   ) {
     super(description);
     Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain

packages/shared/src/errors/NotFoundError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class LangfuseNotFoundError extends BaseError {
+  constructor(description = "Not Found") {
+    super("LangfuseNotFoundError", 404, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -1,6 +1,7 @@
 export { BaseError } from "./BaseError";
-export { NotFoundError } from "./NotFoundError";
+export { LangfuseNotFoundError } from "./NotFoundError";
 export { ValidationError } from "./ValidationError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
+export { ApiError } from "./ApiError";

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, observationsTableCols } from "../..";
+import { ModelProvider } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -34,6 +34,17 @@ export const wipVariableMapping = z.object({
   selectedColumnId: z.string().nullish(),
 });
 
+const observationCols = [
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 'o."metadata"',
+  },
+  { name: "Input", id: "input", internal: 'o."input"' },
+  { name: "Output", id: "output", internal: 'o."output"' },
+];
+
 export const availableEvalVariables = [
   {
     id: "trace",
@@ -52,32 +63,18 @@ export const availableEvalVariables = [
   {
     id: "span",
     display: "Span",
-    availableColumns: [
-      {
-        name: "Metadata",
-        id: "metadata",
-        type: "stringObject",
-        internal: 'o."metadata"',
-      },
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
+    availableColumns: observationCols,
   },
   {
     id: "generation",
     display: "Generation",
-    availableColumns: [
-      {
-        name: "Metadata",
-        id: "metadata",
-        type: "stringObject",
-        internal: 'o."metadata"',
-      },
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
+    availableColumns: observationCols,
+  },
+  {
+    id: "event",
+    display: "Event",
+    availableColumns: observationCols,
   },
-  { id: "event", display: "Event", availableColumns: observationsTableCols },
 ];
 
 export const evalLLMModels = [

packages/shared/src/index.ts

@@ -21,3 +21,6 @@ export * from "./features/evals/types";
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
+
+// errors
+export * from "./errors/index";

packages/shared/src/tracesTable.ts

@@ -1,8 +1,13 @@
 import { ObservationLevel } from "@prisma/client";
 import { ColumnDefinition, OptionsDefinition } from ".";
 
-export const tracesTableCols: ColumnDefinition[] = [
-  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
+const tracesOnlyCols: ColumnDefinition[] = [
+  {
+    name: "⭐️",
+    id: "bookmarked",
+    type: "boolean",
+    internal: "t.bookmarked",
+  },
   { name: "ID", id: "id", type: "string", internal: "t.id" },
   {
     name: "Name",
@@ -24,6 +29,41 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "string",
     internal: 't."session_id"',
   },
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 't."metadata"',
+  },
+  {
+    name: "Version",
+    id: "version",
+    type: "string",
+    internal: 't."version"',
+  },
+  {
+    name: "Release",
+    id: "release",
+    type: "string",
+    internal: 't."release"',
+  },
+  {
+    name: "Level",
+    id: "level",
+    type: "stringOptions",
+    internal: '"level"',
+    options: Object.values(ObservationLevel).map((value) => ({ value })),
+  },
+  {
+    name: "Tags",
+    id: "tags",
+    type: "arrayOptions",
+    internal: 't."tags"',
+    options: [], // to be filled in at runtime
+  },
+];
+export const tracesTableCols: ColumnDefinition[] = [
+  ...tracesOnlyCols,
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -48,12 +88,7 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: 'tm."totalTokens"',
   },
-  {
-    name: "Metadata",
-    id: "metadata",
-    type: "stringObject",
-    internal: 't."metadata"',
-  },
+
   {
     name: "Scores",
     id: "scores_avg",
@@ -84,34 +119,10 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: '"calculatedTotalCost"',
   },
-  {
-    name: "Version",
-    id: "version",
-    type: "string",
-    internal: 't."version"',
-  },
-  {
-    name: "Release",
-    id: "release",
-    type: "string",
-    internal: 't."release"',
-  },
-  {
-    name: "Level",
-    id: "level",
-    type: "stringOptions",
-    internal: '"level"',
-    options: Object.values(ObservationLevel).map((value) => ({ value })),
-  },
-  {
-    name: "Tags",
-    id: "tags",
-    type: "arrayOptions",
-    internal: 't."tags"',
-    options: [], // to be filled in at runtime
-  },
 ];
 
+export const evalTableCols: ColumnDefinition[] = tracesOnlyCols;
+
 export type TraceOptions = {
   scores_avg: Array<string>;
   name: Array<OptionsDefinition>;
@@ -119,9 +130,10 @@ export type TraceOptions = {
 };
 
 export function tracesTableColsWithOptions(
-  options?: TraceOptions
+  options?: TraceOptions,
+  cols: ColumnDefinition[] = tracesTableCols
 ): ColumnDefinition[] {
-  return tracesTableCols.map((col) => {
+  return cols.map((col) => {
     if (col.id === "scores_avg") {
       return { ...col, keyOptions: options?.scores_avg ?? [] };
     }

packages/shared/tsconfig.json

@@ -7,7 +7,7 @@
     "outDir": "./dist",
     "types": ["node"],
     "target": "ES2020",
-    "rootDir": ".",
+    "rootDir": "."
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

web/generated/openapi-server/openapi.yml

@@ -6,13 +6,22 @@ info:
     ## Authentication
 
 
-    Authenticate with the API using Basic Auth, get API keys in the project
-    settings:
+    Authenticate with the API using [Basic
+    Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API
+    keys in the project settings:
 
 
     - username: Langfuse Public Key
 
     - password: Langfuse Secret Key
+
+
+    ## Exports
+
+
+    - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
+
+    - Postman collection: https://cloud.langfuse.com/postman-collection.json
 paths:
   /api/public/dataset-items:
     post:
@@ -717,19 +726,100 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
-  /api/public/prompts:
+  /api/public/v2/prompts/{promptName}:
     get:
       description: Get a prompt
       operationId: prompts_get
       tags:
         - Prompts
       parameters:
-        - name: name
-          in: query
+        - name: promptName
+          in: path
+          description: The name of the prompt
           required: true
           schema:
             type: string
         - name: version
+          in: query
+          description: Version of the prompt to be retrieved.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: label
+          in: query
+          description: >-
+            Label of the prompt to be retrieved. Defaults to "production" if no
+            label or version is set.
+          required: false
+          schema:
+            type: string
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Prompt'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/v2/prompts:
+    get:
+      description: Get a list of prompt names with versions and labels
+      operationId: prompts_list
+      tags:
+        - Prompts
+      parameters:
+        - name: name
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: label
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: tag
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
           in: query
           required: false
           schema:
@@ -741,7 +831,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Prompt'
+                $ref: '#/components/schemas/PromptMetaListResponse'
         '400':
           description: ''
           content:
@@ -2479,6 +2569,42 @@ components:
       required:
         - id
         - name
+    PromptMetaListResponse:
+      title: PromptMetaListResponse
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/PromptMeta'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    PromptMeta:
+      title: PromptMeta
+      type: object
+      properties:
+        name:
+          type: string
+        versions:
+          type: array
+          items:
+            type: integer
+        labels:
+          type: array
+          items:
+            type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - name
+        - versions
+        - labels
+        - tags
     CreatePromptRequest:
       title: CreatePromptRequest
       oneOf:
@@ -2510,18 +2636,19 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: array
           items:
             $ref: '#/components/schemas/ChatMessage'
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     CreateTextPromptRequest:
       title: CreateTextPromptRequest
@@ -2529,16 +2656,17 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: string
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     Prompt:
       title: Prompt
@@ -2574,10 +2702,15 @@ components:
         version:
           type: integer
         config: {}
+        labels:
+          type: array
+          items:
+            type: string
       required:
         - name
         - version
         - config
+        - labels
     ChatMessage:
       title: ChatMessage
       type: object

web/generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": "## Authentication\n\nAuthenticate with the API using Basic Auth, get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key"
+    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml\n- Postman collection: https://cloud.langfuse.com/postman-collection.json"
   },
   "variable": [
     {
@@ -585,13 +585,58 @@
           "request": {
             "description": "Get a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts?name=&version=",
+              "raw": "{{baseUrl}}/api/public/v2/prompts/:promptName?version=&label=",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
+                "prompts",
+                ":promptName"
+              ],
+              "query": [
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": "Version of the prompt to be retrieved."
+                },
+                {
+                  "key": "label",
+                  "value": "",
+                  "description": "Label of the prompt to be retrieved. Defaults to \"production\" if no label or version is set."
+                }
+              ],
+              "variable": [
+                {
+                  "key": "promptName",
+                  "value": "",
+                  "description": "The name of the prompt"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get a list of prompt names with versions and labels",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/v2/prompts?name=&label=&tag=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "v2",
                 "prompts"
               ],
               "query": [
@@ -601,7 +646,22 @@
                   "description": null
                 },
                 {
-                  "key": "version",
+                  "key": "label",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "tag",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
                   "value": "",
                   "description": null
                 }
@@ -621,13 +681,14 @@
           "request": {
             "description": "Create a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts",
+              "raw": "{{baseUrl}}/api/public/v2/prompts",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "prompts"
               ],
               "query": [],
@@ -638,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"

web/package.json

@@ -1,13 +1,13 @@
 {
   "name": "web",
-  "version": "2.32.0",
+  "version": "2.37.2",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
+    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml && cp generated/postman/collection.json public/postman-collection.json",
     "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",

web/public/.gitignore

@@ -1 +1,2 @@
-openapi*.yml
+openapi*.yml
+postman-collection.json

web/sentry.server.config.ts

@@ -15,13 +15,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     tracesSampler: (samplingContext: SamplingContext) => {
       if (
         samplingContext.request &&
-        samplingContext.request.method === "POST" &&
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.9;
+        return 0.3;
       }
-      return 0.15;
+      return 0.1;
     },
 
     profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate

web/src/__tests__/prompts.v1.servertest.ts

@@ -5,10 +5,10 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
 import { type Prompt } from "@langfuse/shared";
 import {
-  PromptSchema,
+  LegacyPromptSchema,
   PromptType,
-  type ValidatedPrompt,
-} from "@/src/features/prompts/server/validation";
+  type LegacyValidatedPrompt,
+} from "@/src/features/prompts/server/utils/validation";
 
 describe("/api/public/prompts API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -21,7 +21,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         config: {
           temperature: 0.1,
@@ -51,6 +51,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production"]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
     expect(fetchedObservations.body.tags).toEqual([]);
@@ -64,7 +65,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt + name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         config: {
           temperature: 0.1,
@@ -94,6 +95,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production"]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
     expect(fetchedObservations.body.tags).toEqual([]);
@@ -107,7 +109,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: false,
+        labels: [],
         version: 1,
         config: {
           temperature: 0.1,
@@ -137,7 +139,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt-one",
-        isActive: false,
+        labels: [],
         version: 1,
         config: {
           temperature: 0.1,
@@ -154,7 +156,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptTwoId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 2,
         config: {
           temperature: 0.2,
@@ -183,68 +185,192 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.prompt).toBe("prompt-one");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(false);
+    expect(fetchedObservations.body.labels).toEqual([]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
 
   it("should fetch active prompt when multiple exist", async () => {
-    const promptIdOne = uuidv4();
-    const promptIdTwo = uuidv4();
-
-    await prisma.prompt.create({
-      data: {
-        id: promptIdOne,
-        name: "prompt-name",
-        prompt: "prompt",
-        isActive: false,
-        version: 1,
-        config: {
-          temperature: 0.1,
-        },
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
+    // First prompt is activated
+    const prompt1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt1",
+      isActive: true,
+      version: 1,
+      config: {
+        temperature: 0.1,
       },
+      createdBy: "user-1",
     });
 
-    await prisma.prompt.create({
-      data: {
-        id: promptIdTwo,
-        name: "prompt-name",
-        prompt: "prompt",
-        isActive: true,
-        version: 2,
-        config: {
-          temperature: 0.2,
-        },
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
+    // Second prompt also activated
+    const prompt2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt2",
+      labels: ["production"],
+      isActive: true,
+      version: 2,
+      config: {
+        temperature: 0.2,
       },
+      createdBy: "user-1",
     });
 
-    const fetchedObservations = await makeAPICall(
+    // Third prompt is deactivated
+    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt3",
+      labels: [], // This should be ignored
+      isActive: false,
+      version: 3,
+      config: {
+        temperature: 0.3,
+      },
+      createdBy: "user-1",
+    });
+
+    // Expect the second prompt to be fetched
+    const fetchedProductionPrompt = await makeAPICall(
       "GET",
       "/api/public/prompts?name=prompt-name",
       undefined,
     );
 
-    expect(fetchedObservations.status).toBe(200);
+    expect(fetchedProductionPrompt.status).toBe(200);
 
-    if (!isPrompt(fetchedObservations.body)) {
+    if (!isPrompt(fetchedProductionPrompt.body)) {
       throw new Error("Expected body to be a prompt");
     }
 
-    expect(fetchedObservations.body.id).toBe(promptIdTwo);
-    expect(fetchedObservations.body.name).toBe("prompt-name");
-    expect(fetchedObservations.body.prompt).toBe("prompt");
-    expect(fetchedObservations.body.type).toBe("text");
-    expect(fetchedObservations.body.version).toBe(2);
-    expect(fetchedObservations.body.isActive).toBe(true);
-    expect(fetchedObservations.body.createdBy).toBe("user-1");
-    expect(fetchedObservations.body.config).toEqual({ temperature: 0.2 });
+    // @ts-expect-error
+    expect(fetchedProductionPrompt.body.id).toBe(prompt2.body.id);
+    expect(fetchedProductionPrompt.body.name).toBe("prompt-name");
+    expect(fetchedProductionPrompt.body.prompt).toBe("prompt2");
+    expect(fetchedProductionPrompt.body.type).toBe("text");
+    expect(fetchedProductionPrompt.body.version).toBe(2);
+    expect(fetchedProductionPrompt.body.isActive).toBe(true);
+    expect(fetchedProductionPrompt.body.labels).toEqual(["production"]);
+    expect(fetchedProductionPrompt.body.createdBy).toBe("API");
+    expect(fetchedProductionPrompt.body.config).toEqual({ temperature: 0.2 });
+
+    // Expect the first prompt to be deactivated
+    const fetchedOldProductionPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedOldProductionPrompt.status).toBe(200);
+
+    if (!isPrompt(fetchedOldProductionPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedOldProductionPrompt.body.id).toBe(prompt1.body.id);
+    expect(fetchedOldProductionPrompt.body.name).toBe("prompt-name");
+    expect(fetchedOldProductionPrompt.body.prompt).toBe("prompt1");
+    expect(fetchedOldProductionPrompt.body.type).toBe("text");
+    expect(fetchedOldProductionPrompt.body.version).toBe(1);
+    expect(fetchedOldProductionPrompt.body.isActive).toBe(false);
+    expect(fetchedOldProductionPrompt.body.labels).toEqual([]);
+    expect(fetchedOldProductionPrompt.body.createdBy).toBe("API");
+    expect(fetchedOldProductionPrompt.body.config).toEqual({
+      temperature: 0.1,
+    });
+  });
+
+  it("should correctly handle overwriting labels", async () => {
+    // First prompt has multiple labels
+    const prompt1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt1",
+      labels: ["production", "staging", "development"],
+      isActive: true,
+      version: 1,
+      config: {
+        temperature: 0.1,
+      },
+      createdBy: "user-1",
+    });
+
+    // Second prompt overwrites production and staging label
+    const prompt2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt2",
+      labels: ["production", "production", "staging"], // Should be deduped
+      isActive: true,
+      version: 2,
+      config: {
+        temperature: 0.2,
+      },
+      createdBy: "user-1",
+    });
+
+    // Third prompt overwrites staging label
+    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt3",
+      labels: ["staging"],
+      isActive: false,
+      version: 3,
+      config: {
+        temperature: 0.3,
+      },
+      createdBy: "user-1",
+    });
+
+    // Expect the second prompt to be fetched as default production prompt
+    const fetchedProductionPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name",
+      undefined,
+    );
+    expect(fetchedProductionPrompt.status).toBe(200);
+    if (!isPrompt(fetchedProductionPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+    // @ts-expect-error
+    expect(fetchedProductionPrompt.body.id).toBe(prompt2.body.id);
+    expect(fetchedProductionPrompt.body.labels).toEqual(["production"]); // Only production label should be present
+
+    // Expect the first prompt to have only development label
+    const fetchedFirstPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedFirstPrompt.status).toBe(200);
+    if (!isPrompt(fetchedFirstPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedFirstPrompt.body.id).toBe(prompt1.body.id);
+    expect(fetchedFirstPrompt.body.labels).toEqual(["development"]);
+
+    // Expect the third prompt to have only staging label
+    const fetchedThirdPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=3",
+      undefined,
+    );
+
+    expect(fetchedThirdPrompt.status).toBe(200);
+    if (!isPrompt(fetchedThirdPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedThirdPrompt.body.id).toBe(prompt3.body.id);
+    expect(fetchedThirdPrompt.body.labels).toEqual(["staging", "latest"]);
   });
 
   it("should create and fetch a prompt", async () => {
@@ -275,6 +401,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production", "latest"]);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
@@ -290,7 +417,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
@@ -353,7 +480,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
@@ -428,6 +555,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production", "latest"]);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({});
   });
@@ -461,6 +589,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(validatedPrompt.type).toBe("chat");
     expect(validatedPrompt.version).toBe(1);
     expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.labels).toEqual(["production", "latest"]);
     expect(validatedPrompt.createdBy).toBe("API");
     expect(validatedPrompt.config).toEqual({});
   });
@@ -484,7 +613,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -512,7 +641,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -535,7 +664,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -589,6 +718,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(validatedPrompt.type).toBe("chat");
     expect(validatedPrompt.version).toBe(1);
     expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.labels).toEqual(["production", "latest"]);
     expect(validatedPrompt.createdBy).toBe("API");
     expect(validatedPrompt.config).toEqual({});
 
@@ -600,15 +730,17 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(getResponse2.status).toBe(404);
     expect(getResponse2.body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
 });
 
-const isPrompt = (x: unknown): x is Prompt => {
+type PromptWithIsActive = Prompt & { isActive: boolean };
+
+const isPrompt = (x: unknown): x is PromptWithIsActive => {
   if (typeof x !== "object" || x === null) return false;
-  const prompt = x as Prompt;
+  const prompt = x as PromptWithIsActive;
   return (
     typeof prompt.id === "string" &&
     typeof prompt.name === "string" &&
@@ -622,7 +754,9 @@ const isPrompt = (x: unknown): x is Prompt => {
   );
 };
 
-const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+const validatePrompt = (
+  obj: Record<string, unknown>,
+): LegacyValidatedPrompt => {
   Object.keys(obj).forEach((key) => {
     obj[key] =
       key === "createdAt" || key === "updatedAt"
@@ -630,5 +764,5 @@ const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
         : obj[key];
   });
 
-  return PromptSchema.parse(obj);
+  return LegacyPromptSchema.parse(obj);
 };

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -23,6 +23,7 @@ describe("Traces TRPC Router", () => {
       ],
       featureFlags: {
         templateFlag: true,
+        evals: true,
       },
       admin: true,
     },

web/src/components/layouts/layout.tsx

@@ -161,7 +161,6 @@ export default function Layout(props: PropsWithChildren) {
 
   const hideNavigation =
     session.status === "unauthenticated" ||
-    projects.length === 0 ||
     pathsWithoutNavigation.includes(router.pathname) ||
     router.pathname.startsWith("/public/");
   if (hideNavigation)

web/src/components/layouts/routes.ts

@@ -63,19 +63,25 @@ export const ROUTES: Route[] = [
     name: "Evaluation",
     icon: Lightbulb,
     cloudOnly: true,
-    featureFlag: "evals",
+    label: "Beta",
     children: [
       {
         name: "Templates",
         pathname: `/project/[projectId]/evals/templates`,
         cloudOnly: true,
-        featureFlag: "evals",
+        rbacScope: "evalTemplate:read",
       },
       {
         name: "Configs",
         pathname: `/project/[projectId]/evals/configs`,
         cloudOnly: true,
-        featureFlag: "evals",
+        rbacScope: "job:read",
+      },
+      {
+        name: "Log",
+        pathname: `/project/[projectId]/evals/log`,
+        cloudOnly: true,
+        rbacScope: "jobExecution:read",
       },
     ],
   },

web/src/components/layouts/status-badge.tsx

@@ -1,42 +1,59 @@
 import { cn } from "@/src/utils/tailwind";
 
 const statusCategories = {
-  active: ["production", "live", "active"],
+  active: ["production", "live", "active", "pending"],
   inactive: ["disabled", "inactive"],
+  completed: ["completed", "done", "finished"],
+  error: ["error", "failed"],
 };
 
 export type Status =
   (typeof statusCategories)[keyof typeof statusCategories][number];
 
 export const StatusBadge = (props: { className?: string; type: Status }) => {
+  let badgeColor = "bg-gray-100 text-gray-800";
+  let dotColor = "bg-gray-500";
+  let dotPingColor = "bg-gray-600";
+  let showDot = true;
+
+  if (statusCategories.active.includes(props.type)) {
+    badgeColor = "bg-green-100 text-green-600";
+    dotColor = "animate-ping bg-green-500";
+    dotPingColor = "bg-green-600";
+  } else if (statusCategories.error.includes(props.type)) {
+    badgeColor = "bg-red-100 text-red-600";
+    dotColor = "bg-red-500";
+    dotPingColor = "bg-red-600";
+    showDot = false;
+  } else if (statusCategories.completed.includes(props.type)) {
+    badgeColor = "bg-green-100 text-green-600";
+    showDot = false;
+  }
+
   return (
     <div
       className={cn(
         "inline-flex items-center gap-2 rounded-sm px-2 py-1 text-xs",
-        statusCategories.active.includes(props.type)
-          ? " bg-green-100 text-green-600"
-          : "bg-gray-100 text-gray-800",
+        badgeColor,
         props.className,
       )}
     >
-      <span className="relative inline-flex h-2 w-2">
-        <span
-          className={cn(
-            "absolute inline-flex h-full w-full  rounded-full opacity-75",
-            statusCategories.active.includes(props.type)
-              ? "animate-ping bg-green-500"
-              : "bg-gray-500",
-          )}
-        ></span>
-        <span
-          className={cn(
-            "relative inline-flex h-2 w-2 rounded-full ",
-            statusCategories.active.includes(props.type)
-              ? "bg-green-600"
-              : "bg-gray-600",
-          )}
-        ></span>
-      </span>
+      {showDot && (
+        <span className="relative inline-flex h-2 w-2">
+          <span
+            className={cn(
+              "absolute inline-flex h-full w-full  rounded-full opacity-75",
+              dotColor,
+            )}
+          ></span>
+          <span
+            className={cn(
+              "relative inline-flex h-2 w-2 rounded-full ",
+              dotPingColor,
+            )}
+          ></span>
+        </span>
+      )}
       <span>{props.type}</span>
     </div>
   );

web/src/components/table/use-cases/scores.tsx

@@ -25,6 +25,7 @@ export type ScoresTableRow = {
   observationId?: string;
   traceName?: string;
   userId?: string;
+  jobConfigurationId?: string;
 };
 
 export type ScoreFilterInput = Omit<
@@ -169,7 +170,7 @@ export default function ScoresTable({
       id: "userId",
       headerTooltip: {
         description: "The user ID associated with the trace.",
-        href: "https://langfuse.com/docs/tracing/users",
+        href: "https://langfuse.com/docs/tracing-features/users",
       },
       enableHiding: true,
       enableSorting: true,
@@ -186,6 +187,29 @@ export default function ScoresTable({
         ) : undefined;
       },
     },
+    {
+      accessorKey: "jobConfigurationId",
+      header: "Eval Configuration ID",
+      id: "jobConfigurationId",
+      headerTooltip: {
+        description: "The Job Configuration ID associated with the trace.",
+        href: "https://langfuse.com/docs/scores/model-based-evals",
+      },
+      enableHiding: true,
+      enableSorting: true,
+      cell: ({ row }) => {
+        const value = row.getValue("jobConfigurationId");
+        return typeof value === "string" ? (
+          <>
+            <TableLink
+              path={`/project/${projectId}/evals/configs/${value}`}
+              value={value}
+              truncateAt={40}
+            />
+          </>
+        ) : undefined;
+      },
+    },
     {
       accessorKey: "comment",
       header: "Comment",
@@ -218,6 +242,7 @@ export default function ScoresTable({
       traceId: score.traceId,
       traceName: score.traceName ?? undefined,
       userId: score.userId ?? undefined,
+      jobConfigurationId: score.jobConfigurationId ?? undefined,
     };
   };
 

web/src/components/table/use-cases/sessions.tsx

@@ -369,7 +369,7 @@ export default function SessionsTable({
         help={{
           description:
             "A session is a collection of related traces, such as a conversation or thread. To begin, add a sessionId to the trace.",
-          href: "https://langfuse.com/docs/sessions",
+          href: "https://langfuse.com/docs/tracing-features/sessions",
         }}
       />
     </div>

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.32.0";
+export const VERSION = "v2.37.2";

web/src/ee/features/evals/components/eval-config-detail.tsx

@@ -2,7 +2,7 @@ import * as React from "react";
 import Header from "@/src/components/layouts/header";
 import { type RouterOutputs, api } from "@/src/utils/api";
 import { useRouter } from "next/router";
-import { EvalConfigForm } from "@/src/features/evals/components/eval-config-form";
+import { EvalConfigForm } from "@/src/ee/features/evals/components/eval-config-form";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { Button } from "@/src/components/ui/button";
 import {
@@ -12,6 +12,15 @@ import {
 } from "@/src/components/ui/popover";
 import { useState } from "react";
 import { Trash2 } from "lucide-react";
+import {
+  Tabs,
+  TabsContent,
+  TabsList,
+  TabsTrigger,
+} from "@/src/components/ui/tabs";
+import { Label } from "@/src/components/ui/label";
+import TableLink from "@/src/components/table/table-link";
+import EvalLogTable from "@/src/ee/features/evals/components/eval-log";
 
 export const EvalConfigDetail = () => {
   const router = useRouter();
@@ -48,6 +57,11 @@ export const EvalConfigDetail = () => {
     return <div>Config not found</div>;
   }
 
+  const existingEvalConfig =
+    config.data && config.data.evalTemplate
+      ? { ...config.data, evalTemplate: config.data.evalTemplate }
+      : undefined;
+
   return (
     <div className="md:container">
       <Header
@@ -61,16 +75,39 @@ export const EvalConfigDetail = () => {
           />
         }
       />
-      <EvalConfigForm
-        projectId={projectId}
-        evalTemplates={allTemplates.data?.templates}
-        existingEvalConfig={
-          config.data && config.data.evalTemplate
-            ? { ...config.data, evalTemplate: config.data.evalTemplate }
-            : undefined
-        }
-        disabled={true}
-      />
+      {existingEvalConfig && (
+        <>
+          <div className="my-5 flex items-center gap-4 rounded-md border p-2	">
+            <Label>Eval Template</Label>
+            <TableLink
+              path={`/project/${projectId}/evals/templates/${existingEvalConfig.evalTemplateId}`}
+              value={existingEvalConfig.evalTemplateId ?? ""}
+              truncateAt={40}
+            />
+          </div>
+
+          <Tabs defaultValue="logs">
+            <TabsList>
+              <TabsTrigger value="logs">Logs</TabsTrigger>
+              <TabsTrigger value="configuration">Configuration</TabsTrigger>
+            </TabsList>
+            <TabsContent value="configuration">
+              <EvalConfigForm
+                projectId={projectId}
+                evalTemplates={allTemplates.data?.templates}
+                existingEvalConfig={existingEvalConfig}
+                disabled={true}
+              />
+            </TabsContent>
+            <TabsContent value="logs">
+              <EvalLogTable
+                projectId={projectId}
+                jobConfigurationId={existingEvalConfig.id}
+              />
+            </TabsContent>
+          </Tabs>
+        </>
+      )}
     </div>
   );
 };

web/src/ee/features/evals/components/eval-config-form.tsx

@@ -0,0 +1,572 @@
+import { usePostHog } from "posthog-js/react";
+import { useFieldArray, useForm } from "react-hook-form";
+import { Input } from "@/src/components/ui/input";
+import { Button } from "@/src/components/ui/button";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
+import {
+  tracesTableColsWithOptions,
+  evalTableCols,
+  singleFilter,
+  type JobConfiguration,
+  availableEvalVariables,
+} from "@langfuse/shared";
+import * as z from "zod";
+import { useEffect, useState } from "react";
+import { api } from "@/src/utils/api";
+import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import {
+  type EvalTemplate,
+  variableMapping,
+  wipVariableMapping,
+} from "@langfuse/shared";
+import router from "next/router";
+import { Slider } from "@/src/components/ui/slider";
+import { Card } from "@/src/components/ui/card";
+import { JSONView } from "@/src/components/ui/CodeJsonViewer";
+import { Label } from "@/src/components/ui/label";
+import DocPopup from "@/src/components/layouts/doc-popup";
+
+const formSchema = z.object({
+  scoreName: z.string(),
+  target: z.string(),
+  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+  mapping: z.array(wipVariableMapping),
+  sampling: z.coerce.number().gte(0).lte(1),
+  delay: z.coerce.number().optional().default(10),
+});
+
+export const EvalConfigForm = (props: {
+  projectId: string;
+  evalTemplates: EvalTemplate[];
+  disabled?: boolean;
+  existingEvalConfig?: JobConfiguration & { evalTemplate: EvalTemplate };
+  onFormSuccess?: () => void;
+}) => {
+  const [evalTemplate, setEvalTemplate] = useState<string | undefined>(
+    props.existingEvalConfig?.evalTemplate.id,
+  );
+
+  const currentTemplate = props.evalTemplates.find(
+    (t) => t.id === evalTemplate,
+  );
+
+  return (
+    <>
+      {!props.disabled ? (
+        <Select onValueChange={setEvalTemplate} value={evalTemplate}>
+          <SelectTrigger
+            disabled={props.disabled}
+            defaultValue={
+              props.existingEvalConfig?.evalTemplate
+                ? props.existingEvalConfig?.evalTemplate.id
+                : undefined
+            }
+          >
+            <SelectValue placeholder="Select a template to run this eval config" />
+          </SelectTrigger>
+          <SelectContent>
+            {props.evalTemplates.map((template) => (
+              <SelectItem value={template.id} key={template.id}>
+                {`${template.name}-v${template.version}`}
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
+      ) : undefined}
+      {evalTemplate && currentTemplate ? (
+        <InnerEvalConfigForm
+          projectId={props.projectId}
+          disabled={props.disabled}
+          existingEvalConfig={props.existingEvalConfig}
+          evalTemplate={
+            props.existingEvalConfig?.evalTemplate ?? currentTemplate
+          }
+          onFormSuccess={props.onFormSuccess}
+        />
+      ) : null}
+    </>
+  );
+};
+
+export const InnerEvalConfigForm = (props: {
+  projectId: string;
+  evalTemplate: EvalTemplate;
+  disabled?: boolean;
+  existingEvalConfig?: JobConfiguration;
+  onFormSuccess?: () => void;
+}) => {
+  const [formError, setFormError] = useState<string | null>(null);
+  const posthog = usePostHog();
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    disabled: props.disabled,
+    defaultValues: {
+      scoreName:
+        props.existingEvalConfig?.scoreName ??
+        `${props.evalTemplate.name}-v${props.evalTemplate.version}`,
+      target: props.existingEvalConfig?.targetObject ?? "",
+      filter: props.existingEvalConfig?.filter
+        ? z.array(singleFilter).parse(props.existingEvalConfig.filter)
+        : [],
+      mapping: props.existingEvalConfig?.variableMapping
+        ? z
+            .array(variableMapping)
+            .parse(props.existingEvalConfig.variableMapping)
+        : z.array(variableMapping).parse(
+            props.evalTemplate
+              ? props.evalTemplate.vars.map((v) => ({
+                  templateVariable: v,
+                  langfuseObject: "trace" as const,
+                  selectedColumnId: "input",
+                }))
+              : [],
+          ),
+      sampling: props.existingEvalConfig?.sampling
+        ? props.existingEvalConfig.sampling.toNumber()
+        : 1,
+      delay: props.existingEvalConfig?.delay
+        ? props.existingEvalConfig.delay / 1000
+        : 10,
+    },
+  });
+
+  const traceFilterOptions = api.traces.filterOptions.useQuery({
+    projectId: props.projectId,
+    ...form.getFieldState("filter"),
+  });
+
+  useEffect(() => {
+    if (props.evalTemplate) {
+      form.setValue(
+        "mapping",
+        props.evalTemplate.vars.map((v) => ({
+          templateVariable: v,
+          langfuseObject: "trace" as const,
+          selectedColumnId: "input",
+        })),
+      );
+      form.setValue(
+        "scoreName",
+        `${props.evalTemplate.name}-v${props.evalTemplate.version}`,
+      );
+    }
+  }, [form, props.evalTemplate]);
+
+  const { fields } = useFieldArray({
+    control: form.control,
+    name: "mapping",
+  });
+
+  const utils = api.useUtils();
+  const createJobMutation = api.evals.createJob.useMutation({
+    onSuccess: () => utils.models.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    posthog.capture("evals:new_config_form");
+
+    const validatedFilter = z.array(singleFilter).safeParse(values.filter);
+
+    if (validatedFilter.success === false) {
+      form.setError("filter", {
+        type: "manual",
+        message: "Please fill out all filter fields",
+      });
+      return;
+    }
+
+    const validatedVarMapping = z
+      .array(variableMapping)
+      .safeParse(values.mapping);
+
+    if (validatedVarMapping.success === false) {
+      console.log(validatedVarMapping.error);
+      form.setError("mapping", {
+        type: "manual",
+        message: "Please fill out all variable mappings",
+      });
+      return;
+    }
+
+    createJobMutation
+      .mutateAsync({
+        projectId: props.projectId,
+        evalTemplateId: props.evalTemplate.id,
+        scoreName: values.scoreName,
+        target: values.target,
+        filter: validatedFilter.data,
+        mapping: validatedVarMapping.data,
+        sampling: values.sampling,
+        delay: values.delay * 1000, // multiply by 1k to convert to ms
+      })
+      .then(() => {
+        props.onFormSuccess?.();
+        form.reset();
+        void router.push(`/project/${props.projectId}/evals/configs/`);
+      })
+      .catch((error) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+        if ("message" in error && typeof error.message === "string") {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+          setFormError(error.message as string);
+          return;
+        } else {
+          setFormError(JSON.stringify(error));
+          console.error(error);
+        }
+      });
+  }
+
+  return (
+    <Form {...form}>
+      <form
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="flex flex-col gap-4"
+      >
+        <div className="grid gap-4">
+          <FormField
+            control={form.control}
+            name="scoreName"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Score Name</FormLabel>
+                <FormControl>
+                  <Input {...field} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <Card className="flex flex-col gap-6 p-4">
+            <FormField
+              control={form.control}
+              name="target"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Target object</FormLabel>
+                  <FormControl>
+                    <Tabs defaultValue="trace">
+                      <TabsList {...field}>
+                        <TabsTrigger value="trace">Trace</TabsTrigger>
+                        <TabsTrigger value="observation" disabled={true}>
+                          Observation (coming soon)
+                        </TabsTrigger>
+                      </TabsList>
+                    </Tabs>
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="filter"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Target filter</FormLabel>
+                  <FormControl>
+                    <InlineFilterBuilder
+                      columns={tracesTableColsWithOptions(
+                        traceFilterOptions.data,
+                        evalTableCols,
+                      )}
+                      filterState={field.value ?? []}
+                      onChange={(value) => field.onChange(value)}
+                      disabled={props.disabled}
+                    />
+                  </FormControl>
+                  <FormDescription>
+                    This will run on all future traces that match these filters
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+          <Card className="p-4">
+            <FormField
+              control={form.control}
+              name="mapping"
+              render={() => (
+                <>
+                  <FormLabel className="">Variable mapping</FormLabel>
+                  <FormControl>
+                    Here will some variable mapping be added.
+                  </FormControl>
+                  <div className="my-2 flex flex-col gap-2 lg:flex-row">
+                    <JSONView
+                      title={"Eval Template"}
+                      json={props.evalTemplate.prompt ?? null}
+                      className={"min-h-48 bg-gray-100 lg:w-1/2"}
+                    />
+                    <div className=" flex flex-col gap-2 lg:w-1/3">
+                      {fields.map((mappingField, index) => (
+                        <Card className="flex flex-col gap-2 p-4" key={index}>
+                          <div className="text-sm font-semibold	">
+                            {"{{"}
+                            {mappingField.templateVariable}
+                            {"}}"}
+                            <DocPopup
+                              description={
+                                "Variable in the template to be replaced with the trace data."
+                              }
+                              href={
+                                "https://langfuse.com/docs/scores/model-based-evals"
+                              }
+                              size="xs"
+                            />
+                          </div>
+                          <FormField
+                            control={form.control}
+                            key={`${mappingField.id}-langfuseObject`}
+                            name={`mapping.${index}.langfuseObject`}
+                            render={({ field }) => (
+                              <div className="flex  items-center gap-2">
+                                <VariableMappingDescription
+                                  title={"Trace object"}
+                                  description={
+                                    "Langfuse object to retrieve the data from."
+                                  }
+                                  href={
+                                    "https://langfuse.com/docs/scores/model-based-evals"
+                                  }
+                                />
+                                <FormItem className="w-2/3">
+                                  <FormControl>
+                                    <Select
+                                      disabled={props.disabled}
+                                      defaultValue={field.value}
+                                      onValueChange={field.onChange}
+                                    >
+                                      <SelectTrigger>
+                                        <SelectValue />
+                                      </SelectTrigger>
+                                      <SelectContent>
+                                        {availableEvalVariables.map(
+                                          (evalObject) => (
+                                            <SelectItem
+                                              value={evalObject.id}
+                                              key={evalObject.id}
+                                            >
+                                              {evalObject.display}
+                                            </SelectItem>
+                                          ),
+                                        )}
+                                      </SelectContent>
+                                    </Select>
+                                  </FormControl>
+                                  <FormMessage />
+                                </FormItem>
+                              </div>
+                            )}
+                          />
+
+                          {form.watch(`mapping.${index}.langfuseObject`) !==
+                          "trace" ? (
+                            <FormField
+                              control={form.control}
+                              key={`${mappingField.id}-objectName`}
+                              name={`mapping.${index}.objectName`}
+                              render={({ field }) => (
+                                <div className="flex items-center gap-2">
+                                  <VariableMappingDescription
+                                    title={"Object Name"}
+                                    description={
+                                      "Name of the Langfuse object to retrieve the data from."
+                                    }
+                                    href={
+                                      "https://langfuse.com/docs/scores/model-based-evals"
+                                    }
+                                  />
+                                  <FormItem className="w-2/3">
+                                    <FormControl>
+                                      <Input
+                                        {...field}
+                                        value={field.value ?? ""}
+                                        disabled={props.disabled}
+                                      />
+                                    </FormControl>
+                                    <FormMessage />
+                                  </FormItem>
+                                </div>
+                              )}
+                            />
+                          ) : undefined}
+
+                          <FormField
+                            control={form.control}
+                            key={`${mappingField.id}-selectedColumnId`}
+                            name={`mapping.${index}.selectedColumnId`}
+                            render={({ field }) => (
+                              <div className="flex items-center gap-2">
+                                <VariableMappingDescription
+                                  title={"Object Variable"}
+                                  description={
+                                    "Variable on the Langfuse object to insert into the template."
+                                  }
+                                  href={
+                                    "https://langfuse.com/docs/scores/model-based-evals"
+                                  }
+                                />
+                                <FormItem className="w-2/3">
+                                  <FormControl>
+                                    <Select
+                                      disabled={props.disabled}
+                                      defaultValue={field.value ?? undefined}
+                                      onValueChange={(value) => {
+                                        const availableColumns =
+                                          availableEvalVariables.find(
+                                            (evalObject) =>
+                                              evalObject.id ===
+                                              form.watch(
+                                                `mapping.${index}.langfuseObject`,
+                                              ),
+                                          )?.availableColumns;
+                                        const column = availableColumns?.find(
+                                          (column) => column.id === value,
+                                        );
+
+                                        field.onChange(column?.id);
+                                      }}
+                                    >
+                                      <SelectTrigger>
+                                        <SelectValue placeholder="Object type" />
+                                      </SelectTrigger>
+                                      <SelectContent>
+                                        {availableEvalVariables
+                                          .find(
+                                            (evalObject) =>
+                                              evalObject.id ===
+                                              form.watch(
+                                                `mapping.${index}.langfuseObject`,
+                                              ),
+                                          )
+                                          ?.availableColumns.map((column) => (
+                                            <SelectItem
+                                              value={column.id}
+                                              key={column.id}
+                                            >
+                                              {column.name}
+                                            </SelectItem>
+                                          ))}
+                                      </SelectContent>
+                                    </Select>
+                                  </FormControl>
+                                  <FormMessage />
+                                </FormItem>
+                              </div>
+                            )}
+                          />
+                        </Card>
+                      ))}
+                    </div>
+                  </div>
+                  <FormDescription>
+                    Insert trace data into the prompt template.
+                  </FormDescription>
+                  <FormMessage />
+                </>
+              )}
+            />
+          </Card>
+          <Card className="flex flex-col gap-6 p-4">
+            <FormField
+              control={form.control}
+              name="sampling"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Sampling</FormLabel>
+                  <FormControl>
+                    <Slider
+                      disabled={props.disabled}
+                      min={0}
+                      max={1}
+                      step={0.01}
+                      value={[field.value]}
+                      onValueChange={(value) => field.onChange(value[0])}
+                    />
+                  </FormControl>
+                  <div className="flex flex-col">
+                    <FormDescription className="flex justify-between">
+                      <span>0%</span>
+                      <span>100%</span>
+                    </FormDescription>
+                    <FormDescription>
+                      Percentage of traces to evaluate.
+                    </FormDescription>
+                  </div>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="delay"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Delay (seconds)</FormLabel>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormDescription>
+                    Time between first Trace event and evaluation execution to
+                    ensure all Trace data is available
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          </Card>
+        </div>
+
+        {!props.disabled ? (
+          <Button
+            type="submit"
+            loading={createJobMutation.isLoading}
+            className="mt-3"
+          >
+            Save
+          </Button>
+        ) : null}
+      </form>
+      {formError ? (
+        <p className="text-red text-center">
+          <span className="font-bold">Error:</span> {formError}
+        </p>
+      ) : null}
+    </Form>
+  );
+};
+function VariableMappingDescription(p: {
+  title: string;
+  description: string;
+  href: string;
+}) {
+  return (
+    <div className="flex w-1/2 items-center">
+      <Label className="muted-foreground text-sm font-light">{p.title}</Label>
+      <DocPopup description={p.description} href={p.href} size="xs" />
+    </div>
+  );
+}

web/src/ee/features/evals/components/eval-log.tsx

@@ -0,0 +1,215 @@
+import { StatusBadge } from "@/src/components/layouts/status-badge";
+import { DataTable } from "@/src/components/table/data-table";
+import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import TableLink from "@/src/components/table/table-link";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { type RouterOutputs, api } from "@/src/utils/api";
+import { createColumnHelper } from "@tanstack/react-table";
+import { useQueryParams, withDefault, NumberParam } from "use-query-params";
+
+export type JobExecutionRow = {
+  status: string;
+  scoreName?: string;
+  scoreValue?: number;
+  scoreComment?: string;
+  startTime?: string;
+  endTime?: string;
+  traceId?: string;
+  templateId: string;
+  configId: string;
+  error?: string;
+};
+
+
+export default function EvalLogTable({
+  projectId,
+  jobConfigurationId,
+}: {
+  projectId: string;
+  jobConfigurationId?: string;
+}) {
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage("evalLogs", "s");
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+  const logs = api.evals.getLogs.useQuery({
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+    jobConfigurationId,
+    projectId,
+  });
+  const totalCount = logs.data?.totalCount ?? 0;
+
+  const columnHelper = createColumnHelper<JobExecutionRow>();
+  const columns = [
+    columnHelper.accessor("status", {
+      header: "Status",
+      id: "status",
+      cell: (row) => {
+        const status = row.getValue();
+        return <StatusBadge type={status.toLowerCase()} />;
+      },
+    }),
+    columnHelper.accessor("startTime", {
+      id: "startTime",
+      header: "Start Time",
+      enableHiding: true,
+    }),
+    columnHelper.accessor("endTime", {
+      id: "endTime",
+      header: "End Time",
+      enableHiding: true,
+    }),
+    columnHelper.accessor("scoreName", {
+      header: "Score Name",
+      id: "scoreName",
+      enableHiding: true,
+    }),
+    columnHelper.accessor("scoreValue", {
+      header: "Score Value",
+      id: "scoreValue",
+      enableHiding: true,
+      cell: (row) => {
+        const value = row.getValue();
+        if (value === undefined) {
+          return undefined;
+        }
+        return value % 1 === 0 ? value : value.toFixed(4);
+      },
+    }),
+    columnHelper.accessor("scoreComment", {
+      header: "Score Comment",
+      id: "scoreComment",
+      enableHiding: true,
+      cell: (row) => {
+        const value = row.getValue();
+        return (
+          value !== undefined && (
+            <IOTableCell data={value} singleLine={rowHeight === "s"} />
+          )
+        );
+      },
+    }),
+    columnHelper.accessor("error", {
+      id: "error",
+      header: "Error",
+      enableHiding: true,
+      cell: (row) => {
+        const value = row.getValue();
+        return (
+          value !== undefined && (
+            <IOTableCell data={value} singleLine={rowHeight === "s"} />
+          )
+        );
+      },
+    }),
+    columnHelper.accessor("traceId", {
+      id: "traceId",
+      header: "Trace",
+      cell: (row) => {
+        const traceId = row.getValue();
+        return traceId ? (
+          <TableLink
+            path={`/project/${projectId}/traces/${encodeURIComponent(traceId)}`}
+            value={traceId}
+            truncateAt={10}
+          />
+        ) : undefined;
+      },
+    }),
+    columnHelper.accessor("templateId", {
+      id: "templateId",
+      header: "Template",
+      cell: (row) => {
+        const templateId = row.getValue();
+        return templateId ? (
+          <TableLink
+            path={`/project/${projectId}/evals/templates/${encodeURIComponent(templateId)}`}
+            value={templateId}
+            truncateAt={10}
+          />
+        ) : undefined;
+      },
+    }),
+  ] as LangfuseColumnDef<JobExecutionRow>[];
+
+  if (!jobConfigurationId) {
+    columns.push(
+      columnHelper.accessor("configId", {
+        id: "configId",
+        header: "Config",
+        cell: (row) => {
+          const configId = row.getValue();
+          return configId ? (
+            <TableLink
+              path={`/project/${projectId}/evals/configs/${encodeURIComponent(configId)}`}
+              value={configId}
+              truncateAt={10}
+            />
+          ) : undefined;
+        },
+      }) as LangfuseColumnDef<JobExecutionRow>,
+    );
+  }
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<JobExecutionRow>("evalLogColumnVisibility", columns);
+
+  const convertToTableRow = (
+    jobConfig: RouterOutputs["evals"]["getLogs"]["data"][number],
+  ): JobExecutionRow => {
+    return {
+      status: jobConfig.status,
+      scoreName: jobConfig.score?.name ?? undefined,
+      scoreValue: jobConfig.score?.value ?? undefined,
+      scoreComment: jobConfig.score?.comment ?? undefined,
+      startTime: jobConfig.startTime?.toLocaleString() ?? undefined,
+      endTime: jobConfig.endTime?.toLocaleString() ?? undefined,
+      traceId: jobConfig.jobInputTraceId ?? undefined,
+      templateId: jobConfig.jobConfiguration.evalTemplateId ?? "",
+      configId: jobConfig.jobConfigurationId,
+      error: jobConfig.error ?? undefined,
+    };
+  };
+
+  return (
+    <div>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        setRowHeight={setRowHeight}
+      />
+      <DataTable
+        columns={columns}
+        data={
+          logs.isLoading
+            ? { isLoading: true, isError: false }
+            : logs.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: logs.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: logs.data.data.map((t) => convertToTableRow(t)),
+                }
+        }
+        pagination={{
+          pageCount: Math.ceil(totalCount / paginationState.pageSize),
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+      />
+    </div>
+  );
+}

web/src/ee/features/evals/components/eval-template-detail.tsx

@@ -1,6 +1,6 @@
 import * as React from "react";
 import Header from "@/src/components/layouts/header";
-import { EvalTemplateForm } from "@/src/features/evals/components/template-form";
+import { EvalTemplateForm } from "@/src/ee/features/evals/components/template-form";
 import { api } from "@/src/utils/api";
 import { type EvalTemplate } from "@langfuse/shared";
 import { useRouter } from "next/router";
@@ -44,10 +44,6 @@ export const EvalTemplateDetail = () => {
     },
   );
 
-  const llmApiKeys = api.llmApiKey.all.useQuery({
-    projectId: projectId,
-  });
-
   return (
     <div className="md:container">
       <Header
@@ -76,16 +72,12 @@ export const EvalTemplateDetail = () => {
           )
         }
       />
-      {allTemplates.isLoading ||
-      !allTemplates.data ||
-      llmApiKeys.isLoading ||
-      !llmApiKeys.data ? (
+      {allTemplates.isLoading || !allTemplates.data ? (
         <div>Loading...</div>
       ) : (
         <EvalTemplateForm
           projectId={projectId}
           existingEvalTemplate={template.data ?? undefined}
-          existingLlmApiKeys={llmApiKeys.data?.data ?? []}
           isEditing={isEditing}
           setIsEditing={setIsEditing}
         />

web/src/ee/features/evals/components/template-form.tsx

@@ -1,5 +1,5 @@
 import { usePostHog } from "posthog-js/react";
-import { useCallback, useEffect, useState } from "react";
+import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import * as z from "zod";
 import { Input } from "@/src/components/ui/input";
@@ -14,7 +14,7 @@ import {
   FormMessage,
 } from "@/src/components/ui/form";
 import { Textarea } from "@/src/components/ui/textarea";
-import { type RouterOutputs, api } from "@/src/utils/api";
+import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
 import router from "next/router";
@@ -42,38 +42,13 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/src/components/ui/select";
-import { TEMPLATES } from "@/src/features/evals/components/templates";
-
-const formSchema = z.object({
-  name: z.string().min(1, "Enter a name"),
-  prompt: z
-    .string()
-    .min(1, "Enter a prompt")
-    .refine((val) => {
-      const variables = extractVariables(val);
-      const matches = variables.map((variable) => {
-        // check regex here
-        if (variable.match(/^[A-Za-z_]+$/)) {
-          return true;
-        }
-        return false;
-      });
-      return !matches.includes(false);
-    }, "Variables must only contain letters and underscores (_)"),
-
-  variables: z.array(
-    z.string().min(1, "Variables must have at least one character"),
-  ),
-  model: EvalModelNames,
-  outputScore: z.string(),
-  outputReasoning: z.string(),
-  apiKey: z.string({ required_error: "No LLM API key found." }),
-});
+import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
+import { Label } from "@/src/components/ui/label";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 
 export const EvalTemplateForm = (props: {
   projectId: string;
   existingEvalTemplate?: EvalTemplate;
-  existingLlmApiKeys: RouterOutputs["llmApiKey"]["all"]["data"];
   onFormSuccess?: () => void;
   isEditing?: boolean;
   setIsEditing?: (isEditing: boolean) => void;
@@ -118,9 +93,11 @@ export const EvalTemplateForm = (props: {
           existingEvalTemplateId={props.existingEvalTemplate?.id}
           existingEvalTemplateName={props.existingEvalTemplate?.name}
           preFilledFormValues={
+            // if a langfuse template is selected, use that, else use the existing template
+            // no langfuse template is selected if there is already an existing template
             langfuseTemplate
               ? {
-                  name: langfuseTemplate ?? "",
+                  name: langfuseTemplate.toLocaleLowerCase() ?? "",
                   prompt: currentTemplate?.prompt.trim() ?? "",
                   vars: [],
                   outputSchema: {
@@ -160,6 +137,30 @@ export const EvalTemplateForm = (props: {
   );
 };
 
+const formSchema = z.object({
+  name: z.string().min(1, "Enter a name"),
+  prompt: z
+    .string()
+    .min(1, "Enter a prompt")
+    .refine((val) => {
+      const variables = extractVariables(val);
+      const matches = variables.map((variable) => {
+        // check regex here
+        if (variable.match(/^[A-Za-z_]+$/)) {
+          return true;
+        }
+        return false;
+      });
+      return !matches.includes(false);
+    }, "Variables must only contain letters and underscores (_)"),
+
+  variables: z.array(
+    z.string().min(1, "Variables must have at least one character"),
+  ),
+  outputScore: z.string().min(1, "Enter a score function"),
+  outputReasoning: z.string().min(1, "Enter a reasoning function"),
+});
+
 export type EvalTemplateFormPreFill = {
   name: string;
   prompt: string;
@@ -176,8 +177,9 @@ export type EvalTemplateFormPreFill = {
 
 export const InnerEvalTemplateForm = (props: {
   projectId: string;
+  // pre-filled values from langfuse-defined template or template from db
   preFilledFormValues?: EvalTemplateFormPreFill;
-  existingLlmApiKeys: RouterOutputs["llmApiKey"]["all"]["data"];
+  // template to be updated
   existingEvalTemplateId?: string;
   existingEvalTemplateName?: string;
   onFormSuccess?: () => void;
@@ -187,15 +189,19 @@ export const InnerEvalTemplateForm = (props: {
   const posthog = usePostHog();
   const [formError, setFormError] = useState<string | null>(null);
 
+  // updates the model params based on the pre-filled data
+  // either form update or from langfuse-generated template
   const [modelParams, setModelParams] = useState<UIModelParams>({
     model: props.preFilledFormValues?.model ?? "gpt-3.5-turbo",
     provider:
       props.preFilledFormValues?.modelParams.provider ?? ModelProvider.OpenAI,
     max_tokens: props.preFilledFormValues?.modelParams.max_tokens ?? 100,
     maxTemperature:
-      props.preFilledFormValues?.modelParams.maxTemperature ?? 0.5,
+      props.preFilledFormValues?.modelParams.provider === ModelProvider.OpenAI
+        ? 2
+        : 1,
     top_p: props.preFilledFormValues?.modelParams.top_p ?? 1,
-    temperature: props.preFilledFormValues?.modelParams.temperature ?? 0.5,
+    temperature: props.preFilledFormValues?.modelParams.temperature ?? 1,
   });
 
   const updateModelParam: ModelParamsContext["updateModelParam"] = (
@@ -205,23 +211,13 @@ export const InnerEvalTemplateForm = (props: {
     setModelParams((prev) => ({ ...prev, [key]: value }));
   };
 
-  const getModelProvider = useCallback((model: string) => {
-    return evalLLMModels.find((m) => m.model === model)?.provider;
-  }, []);
-
-  const getApiKeyForModel = useCallback(
-    (model: string) => {
-      const modelProvider = getModelProvider(model);
-      return props.existingLlmApiKeys.find((k) => k.provider === modelProvider);
-    },
-    [getModelProvider, props.existingLlmApiKeys],
-  );
-
-  const defaultModel = props.preFilledFormValues?.model ?? "gpt-3.5-turbo";
+  // updates the form based on the pre-filled data
+  // either form update or from langfuse-generated template
   const form = useForm<z.infer<typeof formSchema>>({
     resolver: zodResolver(formSchema),
     disabled: !props.isEditing,
     defaultValues: {
+      // when updating, the name has to remain the same and should not be updated
       name:
         props.existingEvalTemplateName ?? props.preFilledFormValues?.name ?? "",
       prompt: props.preFilledFormValues?.prompt ?? undefined,
@@ -232,18 +228,16 @@ export const InnerEvalTemplateForm = (props: {
       outputScore: props.preFilledFormValues
         ? OutputSchema.parse(props.preFilledFormValues?.outputSchema).score
         : undefined,
-      apiKey: defaultModel ? getApiKeyForModel(defaultModel)?.id : undefined,
     },
   });
 
   // reset the form if the input template changes
   useEffect(() => {
     if (props.preFilledFormValues) {
-      const model = EvalModelNames.parse(props.preFilledFormValues.model);
-
       form.reset({
+        // taking the existing template over the pre-filled value.
+        // Existing is for editing, pre-filled is for creating off a template
         name: props.existingEvalTemplateName ?? props.preFilledFormValues.name,
-        model: model,
         prompt: props.preFilledFormValues.prompt,
         variables: props.preFilledFormValues.vars,
         outputReasoning: OutputSchema.parse(
@@ -253,7 +247,9 @@ export const InnerEvalTemplateForm = (props: {
           .score,
       });
 
+      // state for the model params is outside of the form, hence needs to be handled individually
       // also set the context for the playground
+      const model = EvalModelNames.parse(props.preFilledFormValues.model);
       updateModelParam("model", model);
       setModelParams((prev) => ({
         ...prev,
@@ -263,8 +259,13 @@ export const InnerEvalTemplateForm = (props: {
       const modelProvider = evalLLMModels.find(
         (m) => m.model === model,
       )?.provider;
+
       if (modelProvider) {
-        updateModelParam("provider", modelProvider);
+        updateModelParam("provider", modelProvider); // updating the provider based on the model
+        updateModelParam(
+          "maxTemperature",
+          modelProvider === ModelProvider.OpenAI ? 2 : 1,
+        ); // setting the max value of the slider based on the provider
       }
     }
   }, [props.preFilledFormValues, form, props.existingEvalTemplateName]);
@@ -282,12 +283,19 @@ export const InnerEvalTemplateForm = (props: {
   function onSubmit(values: z.infer<typeof formSchema>) {
     posthog.capture("evals:new_template_form");
 
+    const model = EvalModelNames.safeParse(modelParams.model);
+
+    if (!model.success) {
+      setFormError("Please select a model.");
+      return;
+    }
+
     createEvalTemplateMutation
       .mutateAsync({
         name: values.name,
         projectId: props.projectId,
         prompt: values.prompt,
-        model: EvalModelNames.parse(modelParams.model),
+        model: model.data,
         modelParams: modelParams,
         vars: extractedVariables ?? [],
         outputSchema: {
@@ -418,49 +426,9 @@ export const InnerEvalTemplateForm = (props: {
               availableModels={[...evalLLMModels]}
               disabled={!props.isEditing}
             />
-            <FormField
-              control={form.control}
-              name="apiKey"
-              render={({ field }) => {
-                const errorMessage =
-                  form.getFieldState("apiKey").error?.message;
-
-                return (
-                  <FormItem>
-                    <FormLabel>API key</FormLabel>
-                    <Input
-                      {...field}
-                      value={
-                        getApiKeyForModel(form.getValues("model"))
-                          ?.displaySecretKey
-                      }
-                      type="text"
-                      disabled
-                    />
-                    {/* Custom form message to include a link to the already existing prompt */}
-
-                    {form.getFieldState("apiKey").error ? (
-                      <div className="flex flex-col text-sm font-medium text-destructive">
-                        <p className="text-sm font-medium text-destructive">
-                          {errorMessage}
-                        </p>
-                        {errorMessage?.includes("No LLM API key found.") ? (
-                          <Link
-                            href={`/project/${props.projectId}/settings`}
-                            className="flex flex-row"
-                          >
-                            Create a new API key here. <ArrowTopRightIcon />
-                          </Link>
-                        ) : null}
-                      </div>
-                    ) : null}
-                    <FormDescription>
-                      The API key is used for each evaluation and will incur
-                      costs.
-                    </FormDescription>
-                  </FormItem>
-                );
-              }}
+            <LLMApiKeyComponent
+              projectId={props.projectId}
+              modelParams={modelParams}
             />
           </div>
         </div>
@@ -483,3 +451,75 @@ export const InnerEvalTemplateForm = (props: {
     </Form>
   );
 };
+
+export const LLMApiKeyComponent = (p: {
+  projectId: string;
+  modelParams: UIModelParams;
+}) => {
+  const hasAccess = useHasAccess({
+    projectId: p.projectId,
+    scope: "llmApiKeys:read",
+  });
+
+  if (!hasAccess) {
+    return (
+      <div>
+        <Label>API key</Label>
+        <p className="text-sm text-muted-foreground">
+          LLM API Key only visible to Owner and Admin roles.
+        </p>
+      </div>
+    );
+  }
+
+  const apiKeys = api.llmApiKey.all.useQuery({
+    projectId: p.projectId,
+  });
+
+  if (apiKeys.isLoading) {
+    return (
+      <div>
+        <Label>API key</Label>
+        <p className="text-sm text-muted-foreground">Loading...</p>
+      </div>
+    );
+  }
+
+  const getModelProvider = (model: string) => {
+    return evalLLMModels.find((m) => m.model === model)?.provider;
+  };
+
+  const getApiKeyForModel = (model: string) => {
+    const modelProvider = getModelProvider(model);
+    return apiKeys.data?.data.find((k) => k.provider === modelProvider);
+  };
+
+  return (
+    <div>
+      <Label>API key</Label>
+      <div>
+        {getApiKeyForModel(p.modelParams.model) ? (
+          <span className="mr-2 rounded-sm bg-gray-200 p-1 text-xs">
+            {getApiKeyForModel(p.modelParams.model)?.displaySecretKey}
+          </span>
+        ) : undefined}
+      </div>
+      {/* Custom form message to include a link to the already existing prompt */}
+      {!getApiKeyForModel(p.modelParams.model) ? (
+        <div className="flex flex-col text-sm font-medium text-destructive">
+          {"No LLM API key found."}
+
+          <Link
+            href={`/project/${p.projectId}/settings`}
+            className="flex flex-row"
+          >
+            Create a new API key here. <ArrowTopRightIcon />
+          </Link>
+        </div>
+      ) : undefined}
+      <p className="text-sm text-muted-foreground">
+        The API key is used for each evaluation and will incur costs.
+      </p>
+    </div>
+  );
+};

web/src/ee/features/evals/pages/configs.tsx

@@ -0,0 +1,42 @@
+import Header from "@/src/components/layouts/header";
+import { useRouter } from "next/router";
+import { Button } from "@/src/components/ui/button";
+import Link from "next/link";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { Lock } from "lucide-react";
+import EvalConfigTable from "@/src/ee/features/evals/components/eval-config-table";
+
+export default function ConfigsPage() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+
+  const hasWriteAccess = useHasAccess({
+    projectId,
+    scope: "job:CUD",
+  });
+
+  return (
+    <div>
+      <Header
+        title="Eval configs"
+        help={{
+          description: "XXX",
+          href: "https://langfuse.com/docs/evals",
+        }}
+        actionButtons={
+          <Button disabled={!hasWriteAccess} asChild>
+            <Link
+              href={
+                hasWriteAccess ? `/project/${projectId}/evals/configs/new` : "#"
+              }
+            >
+              {!hasWriteAccess && <Lock size={16} className="mr-2" />}
+              Add eval config
+            </Link>
+          </Button>
+        }
+      />
+      <EvalConfigTable projectId={projectId} />
+    </div>
+  );
+}

web/src/ee/features/evals/pages/new-configs.tsx

@@ -0,0 +1,26 @@
+import Header from "@/src/components/layouts/header";
+import { EvalConfigForm } from "@/src/ee/features/evals/components/eval-config-form";
+import { api } from "@/src/utils/api";
+
+import { useRouter } from "next/router";
+
+export default function NewConfigsPage() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+
+  const evalTemplates = api.evals.allTemplates.useQuery({
+    projectId,
+    limit: 500,
+    page: 0,
+  });
+
+  return (
+    <div>
+      <Header title="Create eval config" />
+      <EvalConfigForm
+        projectId={projectId}
+        evalTemplates={evalTemplates.data?.templates ?? []}
+      />
+    </div>
+  );
+}

web/src/ee/features/evals/pages/new-template.tsx

@@ -0,0 +1,23 @@
+import Header from "@/src/components/layouts/header";
+import { EvalTemplateForm } from "@/src/ee/features/evals/components/template-form";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+
+import { useRouter } from "next/router";
+
+export default function NewTemplatesPage() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+
+  const hasAccess = useHasAccess({ projectId, scope: "evalTemplate:read" });
+
+  if (!hasAccess) {
+    return null;
+  }
+
+  return (
+    <div className="md:container">
+      <Header title="Create eval template" />
+      <EvalTemplateForm projectId={projectId} isEditing={true} />
+    </div>
+  );
+}

web/src/ee/features/evals/pages/templates.tsx

@@ -0,0 +1,44 @@
+import Header from "@/src/components/layouts/header";
+import { useRouter } from "next/router";
+import { Button } from "@/src/components/ui/button";
+import Link from "next/link";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { Lock } from "lucide-react";
+import EvalsTemplateTable from "@/src/ee/features/evals/components/eval-templates-table";
+
+export default function TemplatesPage() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+
+  const hasWriteAccess = useHasAccess({
+    projectId,
+    scope: "evalTemplate:create",
+  });
+
+  return (
+    <div>
+      <Header
+        title="Eval Templates"
+        help={{
+          description: "XXX",
+          href: "https://langfuse.com/docs/evals",
+        }}
+        actionButtons={
+          <Button disabled={!hasWriteAccess} asChild>
+            <Link
+              href={
+                hasWriteAccess
+                  ? `/project/${projectId}/evals/templates/new`
+                  : "#"
+              }
+            >
+              {!hasWriteAccess && <Lock size={16} className="mr-2" />}
+              Add eval template
+            </Link>
+          </Button>
+        }
+      />
+      <EvalsTemplateTable projectId={projectId} />
+    </div>
+  );
+}

web/src/ee/features/evals/server/router.ts

@@ -379,4 +379,75 @@ export const evalRouter = createTRPCRouter({
         action: "update",
       });
     }),
+
+  getLogs: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        limit: z.number().optional(),
+        page: z.number().optional(),
+        jobConfigurationId: z.string().optional(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === undefined) {
+        throw new Error("Evals available in cloud only");
+      }
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "job:read",
+      });
+
+      const jobExecutions = await ctx.prisma.jobExecution.findMany({
+        where: {
+          projectId: input.projectId,
+          status: {
+            not: "CANCELLED",
+          },
+          ...(input.jobConfigurationId
+            ? { jobConfigurationId: input.jobConfigurationId }
+            : undefined),
+        },
+        select: {
+          id: true,
+          createdAt: true,
+          updatedAt: true,
+          projectId: true,
+          jobConfigurationId: true,
+          status: true,
+          startTime: true,
+          endTime: true,
+          error: true,
+          jobInputTraceId: true,
+          score: true,
+          jobConfiguration: {
+            select: {
+              evalTemplateId: true,
+            },
+          },
+        },
+        ...(input.limit !== undefined && input.page !== undefined
+          ? { take: input.limit, skip: input.page * input.limit }
+          : undefined),
+        orderBy: {
+          createdAt: "desc",
+        },
+      });
+      const count = await ctx.prisma.jobExecution.count({
+        where: {
+          projectId: input.projectId,
+          status: {
+            not: "CANCELLED",
+          },
+          ...(input.jobConfigurationId
+            ? { jobConfigurationId: input.jobConfigurationId }
+            : undefined),
+        },
+      });
+      return {
+        data: jobExecutions,
+        totalCount: count,
+      };
+    }),
 });

web/src/ee/features/playground/page/context/index.tsx

@@ -13,7 +13,7 @@ import { v4 as uuidv4 } from "uuid";
 import { createEmptyMessage } from "@/src/components/ChatMessages/utils/createEmptyMessage";
 import useCommandEnter from "@/src/ee/features/playground/page/hooks/useCommandEnter";
 import { ChatMessageListSchema } from "@/src/features/prompts/components/NewPromptForm/validation";
-import { PromptType } from "@/src/features/prompts/server/validation";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
 import { extractVariables } from "@/src/utils/string";

web/src/ee/features/playground/page/index.tsx

@@ -10,7 +10,7 @@ export default function PlaygroundPage() {
         title="Playground"
         help={{
           description: "A sandbox to test and iterate your prompts",
-          href: "https://docs.langfuse.com/docs/playground",
+          href: "https://langfuse.com/docs/playground",
         }}
         featureBetaURL="https://github.com/orgs/langfuse/discussions/1170"
       />

web/src/env.mjs

@@ -80,6 +80,8 @@ export const env = createEnv({
     OPENAI_API_KEY: z.string().optional(),
     ANTHROPIC_API_KEY: z.string().optional(),
     TURNSTILE_SECRET_KEY: z.string().optional(),
+    // DB event log
+    ENABLE_EVENT_LOG: z.enum(["true", "false"]).optional().default("true"),
   },
 
   /**
@@ -180,6 +182,8 @@ export const env = createEnv({
     NEXT_PUBLIC_POSTHOG_HOST: process.env.NEXT_PUBLIC_POSTHOG_HOST,
     // Other
     NEXT_PUBLIC_CRISP_WEBSITE_ID: process.env.NEXT_PUBLIC_CRISP_WEBSITE_ID,
+    // db event log
+    ENABLE_EVENT_LOG: process.env.ENABLE_EVENT_LOG,
   },
   // Skip validation in Docker builds
   // DOCKER_BUILD is set in Dockerfile

web/src/features/dashboard/components/TracesBarListChart.tsx

@@ -102,7 +102,7 @@ export const TracesBarListChart = ({
           <NoData noDataText="No data">
             <DocPopup
               description="Traces contain details about LLM applications and can be created using the SDK."
-              href="https://langfuse.com/docs/integrations/sdk#1-backend-tracing"
+              href="https://langfuse.com/docs/get-started"
             />
           </NoData>
         )}

web/src/features/dashboard/components/TracesTimeSeriesChart.tsx

@@ -94,7 +94,7 @@ export const TracesTimeSeriesChart = ({
         <NoData noDataText="No data available">
           <DocPopup
             description="Traces contain details about LLM applications and can be created using the SDK."
-            href="https://langfuse.com/docs/integrations/sdk#1-backend-tracing"
+            href="https://langfuse.com/docs/tracing"
           />
         </NoData>
       )}

web/src/features/dashboard/components/UserChart.tsx

@@ -182,7 +182,7 @@ export const UserChart = ({
                   <NoData noDataText="No data">
                     <DocPopup
                       description="Consumption per user is tracked by passing their ids on traces."
-                      href="https://langfuse.com/docs/user-explorer"
+                      href="https://langfuse.com/docs/tracing-features/users"
                     />
                   </NoData>
                 )}

web/src/features/datasets/components/NewDatasetItemButton.tsx

@@ -4,7 +4,6 @@ import {
   Dialog,
   DialogContent,
   DialogHeader,
-  DialogTitle,
 } from "@/src/components/ui/dialog";
 import { useState } from "react";
 import { NewDatasetItemForm } from "@/src/features/datasets/components/NewDatasetItemForm";

web/src/features/evals/Readme.md

@@ -1,4 +0,0 @@
-NOTE: We may transition this feature from our MIT licensed repository to the
-a commercial License (ee folder) once we release a first stable version.
-Please consider this when planning long-term use and integration of this functionality into your projects.
-For more information see https://langfuse.com/docs/open-source

web/src/features/evals/components/eval-config-form.tsx

@@ -1,492 +0,0 @@
-import { usePostHog } from "posthog-js/react";
-import { useFieldArray, useForm } from "react-hook-form";
-import { Input } from "@/src/components/ui/input";
-import { Button } from "@/src/components/ui/button";
-import {
-  Form,
-  FormControl,
-  FormDescription,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from "@/src/components/ui/form";
-import {
-  Select,
-  SelectContent,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "@/src/components/ui/select";
-import { zodResolver } from "@hookform/resolvers/zod";
-import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
-import {
-  tracesTableColsWithOptions,
-  singleFilter,
-  type JobConfiguration,
-  availableEvalVariables,
-} from "@langfuse/shared";
-import * as z from "zod";
-import { useEffect, useState } from "react";
-import { api } from "@/src/utils/api";
-import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";
-import {
-  type EvalTemplate,
-  variableMapping,
-  wipVariableMapping,
-} from "@langfuse/shared";
-import router from "next/router";
-import { Slider } from "@/src/components/ui/slider";
-import { Card } from "@/src/components/ui/card";
-
-const formSchema = z.object({
-  evalTemplateId: z.string(),
-  scoreName: z.string(),
-  target: z.string(),
-  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
-  mapping: z.array(wipVariableMapping),
-  sampling: z.coerce.number().gte(0).lte(1),
-  delay: z.coerce.number().optional().default(10),
-});
-
-export const EvalConfigForm = (props: {
-  projectId: string;
-  evalTemplates: EvalTemplate[];
-  disabled?: boolean;
-  existingEvalConfig?: JobConfiguration & { evalTemplate: EvalTemplate };
-  onFormSuccess?: () => void;
-}) => {
-  const [formError, setFormError] = useState<string | null>(null);
-  const posthog = usePostHog();
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
-    disabled: props.disabled,
-    defaultValues: {
-      evalTemplateId: props.existingEvalConfig?.evalTemplate.id ?? "",
-      scoreName: props.existingEvalConfig?.scoreName ?? "",
-      target: props.existingEvalConfig?.targetObject ?? "",
-      filter: props.existingEvalConfig?.filter
-        ? z.array(singleFilter).parse(props.existingEvalConfig.filter)
-        : [],
-      mapping: props.existingEvalConfig?.variableMapping
-        ? z
-            .array(variableMapping)
-            .parse(props.existingEvalConfig.variableMapping)
-        : z.array(variableMapping).parse([]),
-      sampling: props.existingEvalConfig?.sampling
-        ? props.existingEvalConfig.sampling.toNumber()
-        : 1,
-      delay: props.existingEvalConfig?.delay
-        ? props.existingEvalConfig.delay
-        : 10,
-    },
-  });
-
-  const traceFilterOptions = api.traces.filterOptions.useQuery({
-    projectId: props.projectId,
-    ...form.getFieldState("filter"),
-  });
-
-  const getSelectedEvalTemplate = props.evalTemplates.find(
-    (template) => template.id === form.watch("evalTemplateId"),
-  );
-
-  useEffect(() => {
-    if (getSelectedEvalTemplate) {
-      form.setValue("mapping", []);
-      form.setValue(
-        "mapping",
-        getSelectedEvalTemplate.vars.map((v) => ({
-          templateVariable: v,
-          langfuseObject: "trace" as const,
-        })),
-      );
-      form.setValue(
-        "scoreName",
-        `${getSelectedEvalTemplate?.name}-${getSelectedEvalTemplate?.version}`,
-      );
-    }
-  }, [form, getSelectedEvalTemplate]);
-
-  const { fields } = useFieldArray({
-    control: form.control,
-    name: "mapping",
-  });
-
-  const utils = api.useUtils();
-  const createJobMutation = api.evals.createJob.useMutation({
-    onSuccess: () => utils.models.invalidate(),
-    onError: (error) => setFormError(error.message),
-  });
-
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    posthog.capture("evals:new_config_form");
-
-    if (!getSelectedEvalTemplate) {
-      form.setError("evalTemplateId", {
-        type: "manual",
-        message: "Please select an eval template",
-      });
-      return;
-    }
-
-    const validatedFilter = z.array(singleFilter).safeParse(values.filter);
-
-    if (validatedFilter.success === false) {
-      form.setError("filter", {
-        type: "manual",
-        message: "Please fill out all filter fields",
-      });
-      return;
-    }
-
-    const validatedVarMapping = z
-      .array(variableMapping)
-      .safeParse(values.mapping);
-
-    if (validatedVarMapping.success === false) {
-      form.setError("mapping", {
-        type: "manual",
-        message: "Please fill out all variable mappings",
-      });
-      return;
-    }
-
-    createJobMutation
-      .mutateAsync({
-        projectId: props.projectId,
-        evalTemplateId: getSelectedEvalTemplate.id,
-        scoreName: values.scoreName,
-        target: values.target,
-        filter: validatedFilter.data,
-        mapping: validatedVarMapping.data,
-        sampling: values.sampling,
-        delay: values.delay * 1000, // multiply by 1k to convert to ms
-      })
-      .then(() => {
-        props.onFormSuccess?.();
-        form.reset();
-        void router.push(`/project/${props.projectId}/evals/configs/`);
-      })
-      .catch((error) => {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-        if ("message" in error && typeof error.message === "string") {
-          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-          setFormError(error.message as string);
-          return;
-        } else {
-          setFormError(JSON.stringify(error));
-          console.error(error);
-        }
-      });
-  }
-
-  return (
-    <Form {...form}>
-      <form
-        // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        onSubmit={form.handleSubmit(onSubmit)}
-        className="flex flex-col gap-4"
-      >
-        <div className="grid gap-4">
-          <FormField
-            control={form.control}
-            name="evalTemplateId"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Eval Template</FormLabel>
-                <Select
-                  defaultValue={field.value}
-                  onValueChange={(value) => {
-                    field.onChange(value);
-                  }}
-                >
-                  <FormControl>
-                    <SelectTrigger
-                      disabled={props.disabled}
-                      defaultValue={
-                        props.existingEvalConfig?.evalTemplate
-                          ? props.existingEvalConfig?.evalTemplate.id
-                          : undefined
-                      }
-                    >
-                      <SelectValue placeholder="Select a template to run this eval config" />
-                    </SelectTrigger>
-                  </FormControl>
-                  <SelectContent>
-                    {props.evalTemplates.map((template) => (
-                      <SelectItem value={template.id} key={template.id}>
-                        {`${template.name}-${template.version}`}
-                      </SelectItem>
-                    ))}
-                  </SelectContent>
-                </Select>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-
-          <FormField
-            control={form.control}
-            name="scoreName"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Score Name</FormLabel>
-                <FormControl>
-                  <Input {...field} />
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <Card className="flex flex-col gap-6 p-4">
-            <FormField
-              control={form.control}
-              name="target"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Target object</FormLabel>
-                  <FormControl>
-                    <Tabs defaultValue="trace">
-                      <TabsList {...field}>
-                        <TabsTrigger value="trace">Trace</TabsTrigger>
-                        <TabsTrigger value="observation" disabled={true}>
-                          Observation (coming soon)
-                        </TabsTrigger>
-                      </TabsList>
-                    </Tabs>
-                  </FormControl>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-
-            <FormField
-              control={form.control}
-              name="filter"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Target filter</FormLabel>
-                  <FormControl>
-                    <InlineFilterBuilder
-                      columns={tracesTableColsWithOptions(
-                        traceFilterOptions.data,
-                      )}
-                      filterState={field.value ?? []}
-                      onChange={(value) => field.onChange(value)}
-                      disabled={props.disabled}
-                    />
-                  </FormControl>
-                  <FormDescription>
-                    This will run on all future traces that match these filters
-                  </FormDescription>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-          </Card>
-          <Card className=" p-4">
-            <FormField
-              control={form.control}
-              name="mapping"
-              render={() => (
-                <>
-                  <FormLabel>Variable mapping</FormLabel>
-                  <FormControl>
-                    Here will some variable mapping be added.
-                  </FormControl>
-                  <div className="mt-2 flex flex-col gap-2">
-                    {fields.map((mappingField, index) => (
-                      <div className="flex gap-2" key={index}>
-                        <span className="whitespace-nowrap rounded-md bg-slate-200 px-2 py-1 text-xs	">
-                          {mappingField.templateVariable}
-                        </span>
-                        <FormField
-                          control={form.control}
-                          key={`${mappingField.id}-langfuseObject`}
-                          name={`mapping.${index}.langfuseObject`}
-                          render={({ field }) => (
-                            <FormItem>
-                              <FormControl>
-                                <Select
-                                  disabled={props.disabled}
-                                  defaultValue={field.value}
-                                  onValueChange={(value) => {
-                                    field.onChange(value);
-                                  }}
-                                >
-                                  <SelectTrigger>
-                                    <SelectValue placeholder="Object type" />
-                                  </SelectTrigger>
-                                  <SelectContent>
-                                    {availableEvalVariables.map(
-                                      (evalObject) => (
-                                        <SelectItem
-                                          value={evalObject.id}
-                                          key={evalObject.id}
-                                        >
-                                          {evalObject.display}
-                                        </SelectItem>
-                                      ),
-                                    )}
-                                  </SelectContent>
-                                </Select>
-                              </FormControl>
-                              <FormMessage />
-                            </FormItem>
-                          )}
-                        />
-
-                        {form.watch(`mapping.${index}.langfuseObject`) !==
-                        "trace" ? (
-                          <FormField
-                            control={form.control}
-                            key={`${mappingField.id}-objectName`}
-                            name={`mapping.${index}.objectName`}
-                            render={({ field }) => (
-                              <FormItem>
-                                <FormControl>
-                                  <Input
-                                    {...field}
-                                    value={field.value ?? ""}
-                                    disabled={props.disabled}
-                                  />
-                                </FormControl>
-                                <FormMessage />
-                              </FormItem>
-                            )}
-                          />
-                        ) : undefined}
-
-                        <FormField
-                          control={form.control}
-                          key={`${mappingField.id}-selectedColumnId`}
-                          name={`mapping.${index}.selectedColumnId`}
-                          render={({ field }) => (
-                            <FormItem>
-                              <FormControl>
-                                <Select
-                                  disabled={props.disabled}
-                                  defaultValue={field.value ?? undefined}
-                                  onValueChange={(value) => {
-                                    const availableColumns =
-                                      availableEvalVariables.find(
-                                        (evalObject) =>
-                                          evalObject.id ===
-                                          form.watch(
-                                            `mapping.${index}.langfuseObject`,
-                                          ),
-                                      )?.availableColumns;
-                                    const column = availableColumns?.find(
-                                      (column) => column.id === value,
-                                    );
-
-                                    field.onChange(column?.id);
-                                  }}
-                                >
-                                  <SelectTrigger>
-                                    <SelectValue placeholder="Object type" />
-                                  </SelectTrigger>
-                                  <SelectContent>
-                                    {availableEvalVariables
-                                      .find(
-                                        (evalObject) =>
-                                          evalObject.id ===
-                                          form.watch(
-                                            `mapping.${index}.langfuseObject`,
-                                          ),
-                                      )
-                                      ?.availableColumns.map((column) => (
-                                        <SelectItem
-                                          value={column.id}
-                                          key={column.id}
-                                        >
-                                          {column.name}
-                                        </SelectItem>
-                                      ))}
-                                  </SelectContent>
-                                </Select>
-                              </FormControl>
-                              <FormMessage />
-                            </FormItem>
-                          )}
-                        />
-                      </div>
-                    ))}
-                  </div>
-                  <FormDescription>
-                    Insert trace data into the prompt template.
-                  </FormDescription>
-                  <FormMessage />
-                </>
-              )}
-            />
-          </Card>
-          <Card className="flex flex-col gap-6 p-4">
-            <FormField
-              control={form.control}
-              name="sampling"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Sampling</FormLabel>
-                  <FormControl>
-                    <Slider
-                      disabled={props.disabled}
-                      min={0}
-                      max={1}
-                      step={0.01}
-                      value={[field.value]}
-                      onValueChange={(value) => field.onChange(value[0])}
-                    />
-                  </FormControl>
-                  <div className="flex flex-col">
-                    <FormDescription className="flex justify-between">
-                      <span>0%</span>
-                      <span>100%</span>
-                    </FormDescription>
-                    <FormDescription>
-                      Percentage of traces to evaluate.
-                    </FormDescription>
-                  </div>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-            <FormField
-              control={form.control}
-              name="delay"
-              render={({ field }) => (
-                <FormItem>
-                  <FormLabel>Delay (seconds)</FormLabel>
-                  <FormControl>
-                    <Input {...field} />
-                  </FormControl>
-                  <FormDescription>
-                    Time between first Trace event and evaluation execution to
-                    ensure all Trace data is available
-                  </FormDescription>
-                  <FormMessage />
-                </FormItem>
-              )}
-            />
-          </Card>
-        </div>
-
-        {!props.disabled ? (
-          <Button
-            type="submit"
-            loading={createJobMutation.isLoading}
-            className="mt-3"
-          >
-            Save
-          </Button>
-        ) : null}
-      </form>
-      {formError ? (
-        <p className="text-red text-center">
-          <span className="font-bold">Error:</span> {formError}
-        </p>
-      ) : null}
-    </Form>
-  );
-};

web/src/features/feature-flags/available-flags.ts

@@ -1 +1 @@
-export const availableFlags = ["templateFlag", "evals"] as const;
+export const availableFlags = ["templateFlag"] as const;

web/src/features/notifications/checkNotifications.tsx

@@ -9,6 +9,34 @@ import { Button } from "@/src/components/ui/button";
 import { env } from "@/src/env.mjs";
 
 export const NOTIFICATIONS: TNotification[] = [
+  // {
+  //   id: 3,
+  //   releaseDate: new Date("2024-04-26"),
+  //   message: "Langfuse 2.0 is live on ProductHunt",
+  // },
+  {
+    id: 2,
+    releaseDate: new Date("2024-04-24"),
+    message: "Langfuse Launch Week #1 is in Full Swing",
+    description: (
+      <div>
+        <p>
+          {
+            "We're launching a new feature every day this week. We already launched:"
+          }
+        </p>
+        <ul className="ms-4 mt-2 list-outside list-disc">
+          <li>Langfuse x Posthog integration</li>
+          <li>Python decorator integration</li>
+          <li>LLM Playground</li>
+          <li>More to come every day until Friday</li>
+        </ul>
+        <Button size="sm" variant="secondary" className="mt-3">
+          <Link href="https://langfuse.com/launch">Follow along</Link>
+        </Button>
+      </div>
+    ),
+  },
   {
     id: 1,
     releaseDate: new Date("2024-01-29"),

web/src/features/prompts/components/NewPromptForm/index.tsx

@@ -24,7 +24,7 @@ import { Textarea } from "@/src/components/ui/textarea";
 import {
   type CreatePromptTRPCType,
   PromptType,
-} from "@/src/features/prompts/server/validation";
+} from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
 import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
@@ -42,6 +42,7 @@ import Link from "next/link";
 import { ArrowTopRightIcon } from "@radix-ui/react-icons";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
 import { JsonEditor } from "@/src/components/json-editor";
+import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
 
 type NewPromptFormProps = {
   initialPrompt?: Prompt | null;
@@ -122,6 +123,7 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
         type,
         prompt: chatPrompt,
         config: JSON.parse(values.config),
+        labels: values.isActive ? [PRODUCTION_LABEL] : [],
       };
     } else {
       newPrompt = {
@@ -130,6 +132,7 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
         type,
         prompt: textPrompt,
         config: JSON.parse(values.config),
+        labels: values.isActive ? [PRODUCTION_LABEL] : [],
       };
     }
 
@@ -154,6 +157,13 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
 
     if (!isNewPrompt) {
       form.setError("name", { message: "Prompt name already exist." });
+    } else if (currentName === "new") {
+      form.setError("name", { message: "Prompt name cannot be 'new'" });
+    } else if (currentName && !/^[a-zA-Z0-9_\-.]+$/.test(currentName)) {
+      form.setError("name", {
+        message:
+          "Name must be alphanumeric with optional underscores, hyphens, or periods",
+      });
     } else {
       form.clearErrors("name");
     }
@@ -308,12 +318,11 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
                 />
               </FormControl>
               <div className="space-y-1 leading-none">
-                <FormLabel>Activate prompt</FormLabel>
+                <FormLabel>Serve prompt as default to SDKs</FormLabel>
               </div>
               {currentIsActive ? (
                 <div className="text-xs text-gray-500">
-                  Activating the prompt will make it available to the SDKs
-                  immediately.
+                  This makes the prompt available to the SDKs immediately.
                 </div>
               ) : null}
             </FormItem>

web/src/features/prompts/components/NewPromptForm/validation.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { PromptType } from "@/src/features/prompts/server/validation";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import { ChatMessageRole } from "@langfuse/shared";
 
 const ChatMessageSchema = z.object({

web/src/features/prompts/components/SetPromptVersionLabels/AddLabelForm.tsx

@@ -0,0 +1,69 @@
+import React from "react";
+import { PlusIcon } from "lucide-react";
+import { useForm } from "react-hook-form";
+import { z } from "zod";
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { Button } from "@/src/components/ui/button";
+import { Input } from "@/src/components/ui/input";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { PromptLabelSchema } from "@/src/features/prompts/server/utils/validation";
+
+const AddLabelFormSchema = z.object({
+  newLabel: PromptLabelSchema,
+});
+
+type AddLabelFromSchemaType = z.infer<typeof AddLabelFormSchema>;
+
+export const AddLabelForm = (props: {
+  setLabels: React.Dispatch<React.SetStateAction<string[]>>;
+  setSelectedLabels: React.Dispatch<React.SetStateAction<string[]>>;
+  onAddLabel: () => void;
+}) => {
+  const form = useForm<AddLabelFromSchemaType>({
+    resolver: zodResolver(AddLabelFormSchema),
+    defaultValues: {
+      newLabel: "",
+    },
+  });
+
+  const onSubmit = () => {
+    const newLabel = form.getValues().newLabel;
+
+    props.setLabels((prev) => [...prev, newLabel]);
+    props.setSelectedLabels((prev) => [...new Set([...prev, newLabel])]);
+
+    props.onAddLabel();
+    form.reset();
+  };
+
+  return (
+    <Form {...form}>
+      <form
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="my-3 flex flex-row space-x-2 align-top"
+      >
+        <FormField
+          control={form.control}
+          name="newLabel"
+          render={({ field }) => (
+            <FormItem className="flex-1">
+              <FormControl>
+                <Input placeholder="New label" {...field} />
+              </FormControl>
+              <FormMessage className="text-xs" />
+            </FormItem>
+          )}
+        />
+        <Button type="submit" size="icon" variant="outline">
+          <PlusIcon className="h-5 w-5" />
+        </Button>
+      </form>
+    </Form>
+  );
+};

web/src/features/prompts/components/SetPromptVersionLabels/LabelCommandItem.tsx

@@ -0,0 +1,41 @@
+import React from "react";
+import { CircleCheckIcon, CircleIcon } from "lucide-react";
+import { Button } from "@/src/components/ui/button";
+import { cn } from "@/src/utils/tailwind";
+
+export const LabelCommandItem = (props: {
+  label: string;
+  selectedLabels: string[];
+  setSelectedLabels: React.Dispatch<React.SetStateAction<string[]>>;
+}) => {
+  const { label, selectedLabels, setSelectedLabels } = props;
+  const handleLabelChange = () => {
+    setSelectedLabels((prev) => {
+      return prev.includes(label)
+        ? prev.filter((l) => l !== label)
+        : [...prev, label];
+    });
+  };
+
+  const isSelected = selectedLabels.includes(label);
+
+  return (
+    <Button
+      key={label}
+      type="button"
+      variant="ghost"
+      onClick={handleLabelChange}
+      className={cn(
+        "w-full justify-start px-2 py-1 text-sm",
+        isSelected && "font-bold",
+      )}
+    >
+      {isSelected ? (
+        <CircleCheckIcon className="mr-2 h-4 w-4" />
+      ) : (
+        <CircleIcon className="mr-2 h-4 w-4 opacity-20" />
+      )}
+      {label}
+    </Button>
+  );
+};

web/src/features/prompts/components/SetPromptVersionLabels/index.tsx

@@ -0,0 +1,191 @@
+import React, { useEffect, useState, useRef } from "react";
+
+import { FlagIcon, PlusIcon } from "lucide-react";
+import { usePostHog } from "posthog-js/react";
+
+import { Button } from "@/src/components/ui/button";
+import {
+  Command,
+  CommandGroup,
+  CommandList,
+  CommandSeparator,
+} from "@/src/components/ui/command";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { api } from "@/src/utils/api";
+import { type Prompt } from "@langfuse/shared";
+import { AddLabelForm } from "./AddLabelForm";
+import { LabelCommandItem } from "./LabelCommandItem";
+import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
+
+export function SetPromptVersionLabels({ prompt }: { prompt: Prompt }) {
+  const projectId = useProjectIdFromURL();
+  const utils = api.useUtils();
+  const posthog = usePostHog();
+  const hasAccess = useHasAccess({ projectId, scope: "prompts:CUD" });
+
+  const [selectedLabels, setSelectedLabels] = useState<string[]>([]);
+  const [labels, setLabels] = useState<string[]>([]);
+  const [isOpen, setIsOpen] = useState(false);
+  const [isAddingLabel, setIsAddingLabel] = useState(false);
+  const customLabelScrollRef = useRef<HTMLDivElement | null>(null);
+
+  const usedLabelsInProject = api.prompts.allLabels.useQuery(
+    {
+      projectId: projectId as string, // Typecast as query is enabled only when projectId is present
+    },
+    { enabled: Boolean(projectId) },
+  );
+
+  // Set initial labels and selected labels
+  useEffect(() => {
+    if (isOpen) {
+      setLabels([
+        ...new Set([...prompt.labels, ...(usedLabelsInProject.data ?? [])]),
+      ]);
+      setSelectedLabels(prompt.labels);
+    }
+  }, [isOpen, prompt.labels, usedLabelsInProject.data]);
+
+  const isPromotingToProduction =
+    !prompt.labels.includes(PRODUCTION_LABEL) &&
+    selectedLabels.includes(PRODUCTION_LABEL);
+
+  const isDemotingFromProduction =
+    prompt.labels.includes(PRODUCTION_LABEL) &&
+    !selectedLabels.includes(PRODUCTION_LABEL);
+
+  const mutatePromptVersionLabels = api.prompts.setLabels.useMutation({
+    onSuccess: () => {
+      void utils.prompts.invalidate();
+    },
+  });
+
+  const handleSubmitLabels = async () => {
+    if (!projectId) {
+      alert("Project ID is missing");
+      return;
+    }
+
+    await mutatePromptVersionLabels.mutateAsync({
+      projectId: projectId as string,
+      promptId: prompt.id,
+      labels: selectedLabels,
+    });
+
+    posthog.capture("prompt:setLabels", { labels: selectedLabels });
+    setIsOpen(false);
+  };
+
+  if (!hasAccess) return null;
+
+  return (
+    <Popover
+      key={prompt.id}
+      open={isOpen}
+      onOpenChange={() => {
+        setIsOpen(!isOpen);
+        setIsAddingLabel(false);
+      }}
+    >
+      <PopoverTrigger asChild>
+        <Button
+          variant="outline"
+          size="icon"
+          aria-label="Set prompt labels"
+          title="Set prompt labels"
+        >
+          <FlagIcon className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Prompt version labels</h2>
+        <h2 className="mb-3 text-xs">
+          Use labels to fetch prompts via SDKs. The <strong>production</strong>{" "}
+          labeled prompt will be served by default.
+        </h2>
+        <Command className="mx-0 my-3 px-0">
+          <CommandList className="max-h-full overflow-hidden">
+            <CommandSeparator />
+            <CommandGroup heading="Promote to production?">
+              <LabelCommandItem
+                {...{
+                  selectedLabels,
+                  setSelectedLabels,
+                  label: PRODUCTION_LABEL,
+                }}
+              />
+            </CommandGroup>
+            <CommandSeparator />
+            <CommandGroup heading="Custom labels">
+              <div
+                className="max-h-[300px] overflow-y-auto overflow-x-hidden"
+                ref={customLabelScrollRef}
+              >
+                {labels
+                  .filter((l) => l !== PRODUCTION_LABEL)
+                  .map((label) => (
+                    <LabelCommandItem
+                      key={label}
+                      {...{ selectedLabels, setSelectedLabels, label }}
+                    />
+                  ))}
+              </div>
+            </CommandGroup>
+          </CommandList>
+          <div className="px-1">
+            {isAddingLabel ? (
+              <AddLabelForm
+                {...{
+                  setLabels,
+                  setSelectedLabels,
+                  onAddLabel: () => {
+                    setTimeout(
+                      () =>
+                        customLabelScrollRef.current?.scrollTo({
+                          top: customLabelScrollRef.current?.scrollHeight,
+                          behavior: "smooth",
+                        }),
+                      0,
+                    );
+                  },
+                }}
+              />
+            ) : (
+              <Button
+                variant="ghost"
+                className="mt-2 w-full justify-start px-2 py-1 text-sm  font-normal"
+                onClick={() => setIsAddingLabel(true)}
+              >
+                <PlusIcon className="mr-2 h-4 w-4" />
+                Add custom label
+              </Button>
+            )}
+          </div>
+        </Command>
+        <Button
+          type="button"
+          variant={
+            isPromotingToProduction || isDemotingFromProduction
+              ? "destructive"
+              : "default"
+          }
+          loading={mutatePromptVersionLabels.isLoading}
+          className="w-full"
+          onClick={handleSubmitLabels}
+        >
+          {isPromotingToProduction
+            ? "Save and promote to production"
+            : isDemotingFromProduction
+              ? "Save and remove from production"
+              : "Save"}
+        </Button>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/features/prompts/components/promote-prompt.tsx

@@ -1,94 +0,0 @@
-import { Button } from "@/src/components/ui/button";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { api } from "@/src/utils/api";
-import { PlayIcon } from "lucide-react";
-import { useState } from "react";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/src/components/ui/popover";
-import { usePostHog } from "posthog-js/react";
-import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
-
-export function PromotePrompt({
-  promptId,
-  promptName,
-  disabled,
-  variant,
-}: {
-  promptId: string;
-  promptName: string;
-  disabled: boolean;
-  variant?: "ghost" | "outline";
-}) {
-  const projectId = useProjectIdFromURL();
-  const utils = api.useUtils();
-  const posthog = usePostHog();
-  const [isOpen, setIsOpen] = useState(false);
-  const hasAccess = useHasAccess({ projectId, scope: "prompts:CUD" });
-
-  const mutPromotePrompt = api.prompts.promote.useMutation({
-    onSuccess: () => {
-      void utils.prompts.invalidate();
-    },
-  });
-
-  if (!hasAccess) {
-    return null;
-  }
-
-  return (
-    <Popover
-      key={promptId}
-      open={isOpen}
-      onOpenChange={() => setIsOpen(!isOpen)}
-    >
-      <PopoverTrigger asChild>
-        <Button
-          variant={variant ?? "ghost"}
-          size={variant ? "icon" : "xs"}
-          disabled={disabled}
-          aria-label="Promote Prompt to production"
-          title="Promote Prompt to production"
-        >
-          <PlayIcon className="h-4 w-4" />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent>
-        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
-        <p className="mb-3 text-sm">
-          This action promotes the prompt to production. SDKs requesting a
-          prompt with name{" "}
-          <code className="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm font-semibold">
-            {promptName}
-          </code>
-          will receive this prompt. Make sure that the variables match.
-        </p>
-        <div className="flex justify-end space-x-4">
-          <Button
-            type="button"
-            variant="destructive"
-            loading={mutPromotePrompt.isLoading}
-            onClick={() => {
-              if (!projectId) {
-                console.error("Project ID is missing");
-
-                return;
-              }
-
-              void mutPromotePrompt.mutateAsync({
-                promptId,
-                projectId,
-              });
-              posthog.capture("prompt:promote");
-              setIsOpen(false);
-            }}
-          >
-            Promote to production
-          </Button>
-        </div>
-      </PopoverContent>
-    </Popover>
-  );
-}

web/src/features/prompts/components/prompt-detail.tsx

@@ -14,8 +14,7 @@ import { Button } from "@/src/components/ui/button";
 import { CodeView, JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
-import { PromotePrompt } from "@/src/features/prompts/components/promote-prompt";
-import { PromptType } from "@/src/features/prompts/server/validation";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
@@ -23,8 +22,8 @@ import { extractVariables } from "@/src/utils/string";
 import { type Prompt } from "@langfuse/shared";
 import { ScrollArea } from "@radix-ui/react-scroll-area";
 import { TagPromptDetailsPopover } from "@/src/features/tag/components/TagPromptDetailsPopover";
-
 import { PromptHistoryNode } from "./prompt-history";
+import { SetPromptVersionLabels } from "@/src/features/prompts/components/SetPromptVersionLabels";
 
 export const PromptDetail = () => {
   const projectId = useProjectIdFromURL();
@@ -101,12 +100,7 @@ export const PromptDetail = () => {
             ]}
             actionButtons={
               <>
-                <PromotePrompt
-                  promptId={prompt.id}
-                  promptName={prompt.name}
-                  disabled={prompt.isActive}
-                  variant="outline"
-                />
+                <SetPromptVersionLabels prompt={prompt} />
 
                 <Link
                   href={`/project/${projectId}/playground?promptId=${encodeURIComponent(prompt.id)}`}
@@ -185,6 +179,18 @@ export const PromptDetail = () => {
           {prompt.config && JSON.stringify(prompt.config) !== "{}" && (
             <JSONView className="mt-5" json={prompt.config} title="Config" />
           )}
+          <p className="mt-6 text-xs text-gray-600">
+            Fetch prompts via Python or JS/TS SDKs. See{" "}
+            <a
+              href="https://langfuse.com/docs/prompts"
+              className="underline"
+              target="_blank"
+              rel="noopener noreferrer"
+            >
+              documentation
+            </a>{" "}
+            for details.
+          </p>
         </div>
         <div className="flex h-screen flex-col">
           <div className="text-m px-3 font-medium">

web/src/features/prompts/components/prompt-history.tsx

@@ -1,4 +1,5 @@
 import { StatusBadge } from "@/src/components/layouts/status-badge";
+import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
 import { type RouterOutputs } from "@/src/utils/api";
 import { type NextRouter, useRouter } from "next/router";
 
@@ -9,36 +10,51 @@ const PromptHistoryTraceNode = (props: {
   setCurrentPromptVersion: (version: number | undefined) => void;
   router: NextRouter;
   projectId: string;
-}) => (
-  <div
-    className={`group mb-2 flex cursor-pointer flex-col gap-1 rounded-sm p-2 hover:bg-gray-50 ${
-      props.currentPromptVersion === props.prompt.version ? "bg-gray-100" : ""
-    }`}
-    onClick={() => {
-      props.index === 0
-        ? props.setCurrentPromptVersion(undefined)
-        : props.setCurrentPromptVersion(props.prompt.version);
-    }}
-  >
-    <div className="flex items-center gap-2">
-      <span className="rounded-sm bg-gray-200 p-1 text-xs">
-        Version {props.prompt.version}
-      </span>
-      {props.prompt.isActive ? <StatusBadge type={"production"} /> : null}
-    </div>
+}) => {
+  const { prompt } = props;
+  let badges: JSX.Element[] = prompt.labels
+    .sort((a, b) =>
+      a === PRODUCTION_LABEL
+        ? -1
+        : b === PRODUCTION_LABEL
+          ? 1
+          : a.localeCompare(b),
+    )
+    .map((label) => {
+      return <StatusBadge type={label} key={label} />;
+    });
 
-    <div className="flex gap-2">
-      <span className="text-xs text-gray-500">
-        {props.prompt.createdAt.toLocaleString()}
-      </span>
+  return (
+    <div
+      className={`group mb-2 flex cursor-pointer flex-col gap-1 rounded-sm p-2 hover:bg-gray-50 ${
+        props.currentPromptVersion === prompt.version ? "bg-gray-100" : ""
+      }`}
+      onClick={() => {
+        props.index === 0
+          ? props.setCurrentPromptVersion(undefined)
+          : props.setCurrentPromptVersion(prompt.version);
+      }}
+    >
+      <div className="flex flex-wrap items-center gap-2">
+        <span className="rounded-sm bg-gray-200 p-1 text-xs">
+          Version {prompt.version}
+        </span>
+        {badges}
+      </div>
+
+      <div className="flex gap-2">
+        <span className="text-xs text-gray-500">
+          {prompt.createdAt.toLocaleString()}
+        </span>
+      </div>
+      <div className="flex gap-2">
+        <span className="text-xs text-gray-500">
+          by {prompt.creator || prompt.createdBy}
+        </span>
+      </div>
     </div>
-    <div className="flex gap-2">
-      <span className="text-xs text-gray-500">
-        by {props.prompt.creator || props.prompt.createdBy}
-      </span>
-    </div>
-  </div>
-);
+  );
+};
 
 export const PromptHistoryNode = (props: {
   prompts: RouterOutputs["prompts"]["allVersions"];

web/src/features/prompts/components/prompts-table.tsx

@@ -25,7 +25,7 @@ type PromptTableRow = {
   version: number;
   id: string;
   createdAt: Date;
-  isActive: boolean;
+  labels: string[];
   type: string;
   numberOfObservations: number;
   tags: string[];
@@ -189,7 +189,7 @@ export function PromptTable() {
       version: item.version,
       createdAt: item.createdAt,
       type: item.type,
-      isActive: item.isActive,
+      labels: item.labels,
       numberOfObservations: Number(item.observationCount),
       tags: item.tags,
     };
@@ -204,6 +204,25 @@ export function PromptTable() {
         )}
         filterState={filterState}
         setFilterState={setFilterState}
+        actionButtons={
+          <Link href={`/project/${projectId}/prompts/new`}>
+            <Button
+              variant="secondary"
+              disabled={!hasCUDAccess}
+              aria-label="Promote Prompt to Production"
+            >
+              {hasCUDAccess ? (
+                <PlusIcon className="-ml-0.5 mr-1.5" aria-hidden="true" />
+              ) : (
+                <LockIcon
+                  className="-ml-0.5 mr-1.5 h-3 w-3"
+                  aria-hidden="true"
+                />
+              )}
+              New prompt
+            </Button>
+          </Link>
+        }
       />
       <DataTable
         columns={promptColumns}
@@ -230,21 +249,6 @@ export function PromptTable() {
           state: paginationState,
         }}
       />
-      <Link href={`/project/${projectId}/prompts/new`}>
-        <Button
-          variant="secondary"
-          className="mt-4"
-          disabled={!hasCUDAccess}
-          aria-label="Promote Prompt to Production"
-        >
-          {hasCUDAccess ? (
-            <PlusIcon className="-ml-0.5 mr-1.5" aria-hidden="true" />
-          ) : (
-            <LockIcon className="-ml-0.5 mr-1.5 h-3 w-3" aria-hidden="true" />
-          )}
-          New prompt
-        </Button>
-      </Link>
     </div>
   );
 }

web/src/features/prompts/constants.ts

@@ -0,0 +1,2 @@
+export const PRODUCTION_LABEL = "production";
+export const LATEST_PROMPT_LABEL = "latest";

web/src/features/prompts/server/actions/createPrompt.ts

@@ -1,24 +1,27 @@
 import {
   type CreatePromptTRPCType,
   PromptType,
-} from "@/src/features/prompts/server/validation";
-import { ValidationError } from "@/src/server/errors/ValidationError";
+} from "@/src/features/prompts/server/utils/validation";
+import { ValidationError } from "@langfuse/shared";
 import { jsonSchema } from "@/src/utils/zod";
 import { type PrismaClient } from "@langfuse/shared/src/db";
+import { LATEST_PROMPT_LABEL } from "@/src/features/prompts/constants";
+
+export type CreatePromptParams = CreatePromptTRPCType & {
+  createdBy: string;
+  prisma: PrismaClient;
+};
 
 export const createPrompt = async ({
   projectId,
   name,
   prompt,
   type = PromptType.Text,
-  isActive = true,
+  labels = [],
   config,
   createdBy,
   prisma,
-}: CreatePromptTRPCType & {
-  createdBy: string;
-  prisma: PrismaClient;
-}) => {
+}: CreatePromptParams) => {
   const latestPrompt = await prisma.prompt.findFirst({
     where: { projectId, name },
     orderBy: [{ version: "desc" }],
@@ -30,8 +33,14 @@ export const createPrompt = async ({
     );
   }
 
-  const latestActivePrompt = await prisma.prompt.findFirst({
-    where: { projectId, name, isActive: true },
+  const finalLabels = [...labels, LATEST_PROMPT_LABEL]; // Newly created prompts are always labeled as 'latest'
+
+  const previousLabeledPrompts = await prisma.prompt.findMany({
+    where: {
+      projectId,
+      name,
+      labels: { hasSome: finalLabels },
+    },
     orderBy: [{ version: "desc" }],
   });
 
@@ -41,7 +50,7 @@ export const createPrompt = async ({
         prompt,
         name,
         createdBy,
-        isActive,
+        labels: [...new Set(finalLabels)], // Ensure labels are unique
         type,
         tags: latestPrompt?.tags,
         version: latestPrompt?.version ? latestPrompt.version + 1 : 1,
@@ -50,18 +59,21 @@ export const createPrompt = async ({
       },
     }),
   ];
-  if (latestActivePrompt && isActive)
-    // If we're creating a new active prompt, we need to deactivate the old one
-    create.push(
-      prisma.prompt.update({
-        where: {
-          id: latestActivePrompt.id,
-        },
-        data: {
-          isActive: false,
-        },
-      }),
-    );
+
+  if (finalLabels.length > 0)
+    // If we're creating a new labeled prompt, we must remove those labels on previous prompts since labels are unique
+    previousLabeledPrompts.forEach((prevPrompt) => {
+      create.push(
+        prisma.prompt.update({
+          where: { id: prevPrompt.id },
+          data: {
+            labels: prevPrompt.labels.filter(
+              (prevLabel) => !finalLabels.includes(prevLabel),
+            ),
+          },
+        }),
+      );
+    });
 
   const [createdPrompt] = await prisma.$transaction(create);
 

web/src/features/prompts/server/actions/getPromptByName.ts

@@ -0,0 +1,106 @@
+import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
+import {
+  LangfuseNotFoundError,
+  ValidationError,
+  type Prompt,
+} from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+
+type GetPromptByNameParams = {
+  promptName: string;
+  projectId: string;
+  version?: number | null;
+  label?: string;
+};
+
+export const getPromptByName = async (
+  params: GetPromptByNameParams,
+): Promise<Prompt | null> => {
+  const { promptName, projectId, version, label } = params;
+
+  if (version && label)
+    throw new ValidationError("Cannot specify both version and label");
+
+  if (version) return getPromptByVersion({ projectId, promptName, version });
+
+  if (label) return getPromptByLabel({ projectId, promptName, label });
+
+  return getProductionPrompt(params);
+};
+
+const getProductionPrompt = async ({
+  promptName,
+  projectId,
+}: {
+  promptName: string;
+  projectId: string;
+}): Promise<Prompt> => {
+  const productionPrompt = await prisma.prompt.findFirst({
+    where: {
+      projectId: projectId,
+      name: promptName,
+      labels: {
+        has: PRODUCTION_LABEL,
+      },
+    },
+  });
+
+  if (!productionPrompt)
+    throw new LangfuseNotFoundError(
+      `No production-labeled prompt found with name '${promptName}' in project ${projectId}`,
+    );
+
+  return productionPrompt;
+};
+
+const getPromptByVersion = async ({
+  promptName,
+  projectId,
+  version,
+}: {
+  promptName: string;
+  projectId: string;
+  version: number;
+}): Promise<Prompt> => {
+  const prompt = await prisma.prompt.findFirst({
+    where: {
+      projectId: projectId,
+      name: promptName,
+      version: version,
+    },
+  });
+
+  if (!prompt)
+    throw new LangfuseNotFoundError(
+      `No prompt found with name '${promptName}' in project ${projectId} with version ${version}`,
+    );
+
+  return prompt;
+};
+
+const getPromptByLabel = async ({
+  promptName,
+  projectId,
+  label,
+}: {
+  promptName: string;
+  projectId: string;
+  label: string;
+}): Promise<Prompt> => {
+  const prompt = await prisma.prompt.findFirst({
+    where: {
+      projectId: projectId,
+      name: promptName,
+      labels: {
+        has: label,
+      },
+    },
+  });
+
+  if (!prompt)
+    throw new LangfuseNotFoundError(
+      `No prompt found with name '${promptName}' in project ${projectId} with label ${label}`,
+    );
+
+  return prompt;
+};

web/src/features/prompts/server/actions/getPromptsMeta.ts

@@ -0,0 +1,112 @@
+import { type GetPromptsMetaType } from "@/src/features/prompts/server/utils/validation";
+import { promptsTableCols } from "@/src/server/api/definitions/promptsTable";
+import {
+  tableColumnsToSqlFilterAndPrefix,
+  type FilterState,
+} from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+
+export type GetPromptsMetaParams = GetPromptsMetaType & { projectId: string };
+
+export const getPromptsMeta = async (
+  params: GetPromptsMetaParams,
+): Promise<PromptsMetaResponse> => {
+  const { projectId, page, limit } = params;
+
+  const promptsMeta = (await prisma.$queryRaw`
+    SELECT
+        p.name AS name,
+        p.tags AS tags,
+        array_agg(DISTINCT p.version) AS versions,
+        COALESCE(array_agg(DISTINCT label) FILTER (WHERE label IS NOT NULL), '{}'::text[]) AS labels --- COALESCE is necessary to return an empty array if there are no labels and remove NULLs
+    FROM
+        prompts p
+    LEFT JOIN LATERAL unnest(p.labels) AS label ON true
+    WHERE 
+        p."project_id" = ${projectId} 
+        ${getPromptsFilterCondition(params)}
+    GROUP BY
+        p.name, p.tags --- tags are the same for all versions of a prompt
+    ORDER BY
+        p.name --- necessary for consistent pagination
+    LIMIT
+        ${limit}
+    OFFSET
+        ${limit * (page - 1)}
+  `) as PromptsMeta[];
+
+  const [{ count: totalItemsCount }] = (await prisma.$queryRaw`
+    SELECT COUNT(DISTINCT p.name) AS count
+    FROM prompts p
+    WHERE "project_id" = ${projectId} 
+    ${getPromptsFilterCondition(params)}
+  `) as { count: BigInt }[];
+
+  const totalItems = Number(totalItemsCount);
+  const totalPages = Math.ceil(totalItems / limit);
+
+  return {
+    data: promptsMeta,
+    pagination: { page, limit, totalPages, totalItems },
+  };
+};
+
+type PromptsMeta = {
+  name: string;
+  versions: number[];
+  labels: string[];
+  tags: string[];
+};
+
+export type PromptsMetaResponse = {
+  data: PromptsMeta[];
+  pagination: {
+    page: number;
+    limit: number;
+    totalPages: number;
+    totalItems: number;
+  };
+};
+
+const getPromptsFilterCondition = (params: GetPromptsMetaType) => {
+  const { name, version, label, tag } = params;
+  const filters: FilterState = [];
+
+  if (name) {
+    filters.push({
+      column: "name",
+      type: "string",
+      operator: "=",
+      value: name,
+    });
+  }
+
+  if (version) {
+    filters.push({
+      column: "version",
+      type: "number",
+      operator: "=",
+      value: version,
+    });
+  }
+
+  if (label) {
+    filters.push({
+      column: "labels",
+      type: "arrayOptions",
+      operator: "any of",
+      value: [label],
+    });
+  }
+
+  if (tag) {
+    filters.push({
+      column: "tags",
+      type: "arrayOptions",
+      operator: "any of",
+      value: [tag],
+    });
+  }
+
+  return tableColumnsToSqlFilterAndPrefix(filters, promptsTableCols, "prompts");
+};

web/src/features/prompts/server/handlers/promptNameHandler.ts

@@ -0,0 +1,26 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+
+import { getPromptByName } from "@/src/features/prompts/server/actions/getPromptByName";
+import { GetPromptByNameSchema } from "@/src/features/prompts/server/utils/validation";
+import { withMiddlewares } from "@/src/server/utils/withMiddlewares";
+
+import { authorizePromptRequestOrThrow } from "../utils/authorizePromptRequest";
+
+const getPromptNameHandler = async (
+  req: NextApiRequest,
+  res: NextApiResponse,
+) => {
+  const authCheck = await authorizePromptRequestOrThrow(req);
+  const { promptName, version, label } = GetPromptByNameSchema.parse(req.query);
+
+  const prompt = await getPromptByName({
+    promptName: promptName,
+    projectId: authCheck.scope.projectId,
+    version,
+    label,
+  });
+
+  return res.status(200).json(prompt);
+};
+
+export const promptNameHandler = withMiddlewares({ GET: getPromptNameHandler });

web/src/features/prompts/server/handlers/promptsHandler.ts

@@ -0,0 +1,47 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+
+import { createPrompt } from "@/src/features/prompts/server/actions/createPrompt";
+import { getPromptsMeta } from "@/src/features/prompts/server/actions/getPromptsMeta";
+import {
+  CreatePromptSchema,
+  GetPromptsMetaSchema,
+} from "@/src/features/prompts/server/utils/validation";
+import { withMiddlewares } from "@/src/server/utils/withMiddlewares";
+import { prisma } from "@langfuse/shared/src/db";
+
+import { authorizePromptRequestOrThrow } from "../utils/authorizePromptRequest";
+
+const getPromptsHandler = async (req: NextApiRequest, res: NextApiResponse) => {
+  const authCheck = await authorizePromptRequestOrThrow(req);
+
+  const input = GetPromptsMetaSchema.parse(req.query);
+  const promptsMetadata = await getPromptsMeta({
+    ...input,
+    projectId: authCheck.scope.projectId,
+  });
+
+  return res.status(200).json(promptsMetadata);
+};
+
+const postPromptsHandler = async (
+  req: NextApiRequest,
+  res: NextApiResponse,
+) => {
+  const authCheck = await authorizePromptRequestOrThrow(req);
+
+  const input = CreatePromptSchema.parse(req.body);
+  const createdPrompt = await createPrompt({
+    ...input,
+    config: input.config ?? {},
+    projectId: authCheck.scope.projectId,
+    createdBy: "API",
+    prisma: prisma,
+  });
+
+  return res.status(201).json(createdPrompt);
+};
+
+export const promptsHandler = withMiddlewares({
+  GET: getPromptsHandler,
+  POST: postPromptsHandler,
+});

web/src/features/prompts/server/routers/promptRouter.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 
 import { auditLog } from "@/src/features/audit-logs/auditLog";
-import { CreatePromptTRPCSchema } from "@/src/features/prompts/server/validation";
+import { CreatePromptTRPCSchema } from "@/src/features/prompts/server/utils/validation";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import {
   createTRPCRouter,
@@ -10,7 +10,7 @@ import {
 import { DB } from "@/src/server/db";
 import { type Prompt, Prisma } from "@langfuse/shared/src/db";
 
-import { createPrompt } from "./createPrompt";
+import { createPrompt } from "../actions/createPrompt";
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
 import { promptsTableCols } from "@/src/server/api/definitions/promptsTable";
 import { paginationZod } from "@/src/utils/zod";
@@ -59,7 +59,7 @@ export const promptRouter = createTRPCRouter({
           p.type,
           p.updated_at as "updatedAt",
           p.created_at as "createdAt",
-          p.is_active as "isActive",
+          p.labels,
           p.tags`,
           input.projectId,
           filterCondition,
@@ -315,8 +315,14 @@ export const promptRouter = createTRPCRouter({
         throw e;
       }
     }),
-  promote: protectedProjectProcedure
-    .input(z.object({ promptId: z.string(), projectId: z.string() }))
+  setLabels: protectedProjectProcedure
+    .input(
+      z.object({
+        promptId: z.string(),
+        projectId: z.string(),
+        labels: z.array(z.string()),
+      }),
+    )
     .mutation(async ({ input, ctx }) => {
       try {
         throwIfNoAccess({
@@ -325,31 +331,35 @@ export const promptRouter = createTRPCRouter({
           scope: "prompts:CUD",
         });
 
-        const toBePromotedPrompt = await ctx.prisma.prompt.findUniqueOrThrow({
+        const toBeLabeledPrompt = await ctx.prisma.prompt.findUniqueOrThrow({
           where: {
             id: input.promptId,
+            projectId: input.projectId,
           },
         });
 
+        const newLabels = [...new Set(input.labels)];
+
         await auditLog(
           {
             session: ctx.session,
             resourceType: "prompt",
-            resourceId: toBePromotedPrompt.id,
-            action: "promote",
+            resourceId: toBeLabeledPrompt.id,
+            action: "setLabel",
             after: {
-              ...toBePromotedPrompt,
-              isActive: true,
+              ...toBeLabeledPrompt,
+              labels: newLabels,
             },
           },
           ctx.prisma,
         );
 
-        const latestActivePrompt = await ctx.prisma.prompt.findFirst({
+        const previousLabeledPrompts = await ctx.prisma.prompt.findMany({
           where: {
             projectId: input.projectId,
-            name: toBePromotedPrompt.name,
-            isActive: true,
+            name: toBeLabeledPrompt.name,
+            labels: { hasSome: newLabels },
+            id: { not: input.promptId },
           },
           orderBy: [{ version: "desc" }],
         });
@@ -357,30 +367,53 @@ export const promptRouter = createTRPCRouter({
         const toBeExecuted = [
           ctx.prisma.prompt.update({
             where: {
-              id: toBePromotedPrompt.id,
+              id: toBeLabeledPrompt.id,
+              projectId: input.projectId,
             },
             data: {
-              isActive: true,
+              labels: newLabels,
             },
           }),
         ];
-        if (latestActivePrompt)
+
+        // Remove label from previous labeled prompts
+        previousLabeledPrompts.forEach((prevPrompt) => {
           toBeExecuted.push(
             ctx.prisma.prompt.update({
               where: {
-                id: latestActivePrompt.id,
+                id: prevPrompt.id,
+                projectId: input.projectId,
               },
               data: {
-                isActive: false,
+                labels: prevPrompt.labels.filter((l) => !newLabels.includes(l)),
               },
             }),
           );
+        });
         await ctx.prisma.$transaction(toBeExecuted);
       } catch (e) {
         console.log(e);
         throw e;
       }
     }),
+  allLabels: protectedProjectProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "prompts:read",
+      });
+
+      const labels = await ctx.prisma.$queryRaw<{ label: string }[]>`
+        SELECT DISTINCT UNNEST(labels) AS label
+        FROM prompts
+        WHERE project_id = ${input.projectId}      
+        AND labels IS NOT NULL;
+      `;
+
+      return labels.map((l) => l.label);
+    }),
   updateTags: protectedProjectProcedure
     .input(
       z.object({

web/src/features/prompts/server/utils/authorizePromptRequest.ts

@@ -0,0 +1,15 @@
+import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
+import { type NextApiRequest } from "next";
+import { UnauthorizedError, ForbiddenError } from "@langfuse/shared";
+
+export async function authorizePromptRequestOrThrow(req: NextApiRequest) {
+  const authCheck = await verifyAuthHeaderAndReturnScope(
+    req.headers.authorization,
+  );
+  if (!authCheck.validKey) throw new UnauthorizedError(authCheck.error);
+  if (authCheck.scope.accessLevel !== "all")
+    throw new ForbiddenError(
+      `Access denied - need to use basic auth with secret key to ${req.method} prompts`,
+    );
+  return authCheck;
+}

web/src/features/prompts/server/utils/validation.ts

@@ -12,9 +12,18 @@ export enum PromptType {
   Text = "text",
 }
 
+export const PromptLabelSchema = z
+  .string()
+  .min(1)
+  .max(20)
+  .regex(
+    /^[a-z0-9_\-.]+$/,
+    "Label must be lowercase alphanumeric with optional underscores, hyphens, or periods",
+  );
+
 export const CreateTextPromptSchema = z.object({
   name: z.string(),
-  isActive: z.boolean(),
+  labels: z.array(PromptLabelSchema).default([]),
   type: z.literal(PromptType.Text).optional(),
   prompt: z.string(),
   config: jsonSchema.nullable().default({}),
@@ -22,7 +31,7 @@ export const CreateTextPromptSchema = z.object({
 
 export const CreateChatPromptSchema = z.object({
   name: z.string(),
-  isActive: z.boolean(),
+  labels: z.array(PromptLabelSchema).default([]),
   type: z.literal(PromptType.Chat),
   prompt: z.array(ChatMessageSchema),
   config: jsonSchema.nullable().default({}),
@@ -49,11 +58,28 @@ export const CreatePromptTRPCSchema = z.union([
 
 export type CreatePromptTRPCType = z.infer<typeof CreatePromptTRPCSchema>;
 
+export const GetPromptsMetaSchema = z.object({
+  name: z.string().optional(),
+  version: z.coerce.number().int().nullish(),
+  label: z.string().optional(),
+  tag: z.string().optional(),
+  page: z.coerce.number().int().min(1).default(1),
+  limit: z.coerce.number().int().min(1).max(100).default(10),
+});
+
+export type GetPromptsMetaType = z.infer<typeof GetPromptsMetaSchema>;
+
 export const GetPromptSchema = z.object({
   name: z.string().transform((v) => decodeURIComponent(v)),
   version: z.coerce.number().int().nullish(),
 });
 
+export const GetPromptByNameSchema = z.object({
+  promptName: z.string(),
+  version: z.coerce.number().int().nullish(),
+  label: z.string().optional(),
+});
+
 export const TextPromptSchema = z.object({
   id: z.string(),
   createdAt: z.date(),
@@ -62,7 +88,7 @@ export const TextPromptSchema = z.object({
   createdBy: z.string(),
   version: z.number(),
   name: z.string(),
-  isActive: z.boolean(),
+  labels: z.array(PromptLabelSchema),
   tags: z.array(z.string()),
   type: z.literal(PromptType.Text),
   prompt: z.string(),
@@ -83,7 +109,7 @@ export const ChatPromptSchema = z.object({
   version: z.number(),
   name: z.string(),
   tags: z.array(z.string()),
-  isActive: z.boolean(),
+  labels: z.array(PromptLabelSchema),
   type: z.literal(PromptType.Chat),
   prompt: z.array(ChatMessageSchema),
   config: jsonSchema,
@@ -96,3 +122,14 @@ export type ChatPromptType =
 
 export const PromptSchema = z.union([TextPromptSchema, ChatPromptSchema]);
 export type ValidatedPrompt = z.infer<typeof PromptSchema>;
+
+// Backward compat for V1 prompts endpoint
+export const LegacyCreatePromptSchema = z.union([
+  CreateTextPromptSchema.extend({ isActive: z.boolean() }),
+  CreateChatPromptSchema.extend({ isActive: z.boolean() }),
+]);
+export const LegacyPromptSchema = z.union([
+  TextPromptSchema.extend({ isActive: z.boolean() }),
+  ChatPromptSchema.extend({ isActive: z.boolean() }),
+]);
+export type LegacyValidatedPrompt = z.infer<typeof LegacyPromptSchema>;

web/src/features/public-api/components/LLMApiKeyList.tsx

@@ -258,7 +258,7 @@ export function CreateLlmApiKeyComponent(props: {
                       <Input placeholder="sk-proj-...Uwj9" {...field} />
                     </FormControl>
                     <FormDescription>
-                      Your API keys are stored ancrypted on our servers.
+                      Your API keys are stored encrypted on our servers.
                     </FormDescription>
                     <FormMessage />
                   </FormItem>

web/src/features/rbac/constants/roleAccessRights.ts

@@ -34,6 +34,8 @@ const scopes = [
   "job:read",
   "job:CUD",
 
+  "jobExecution:read",
+
   "llmApiKeys:read",
   "llmApiKeys:create",
   "llmApiKeys:delete",
@@ -67,6 +69,7 @@ export const roleAccessRights: Record<MembershipRole, Scope[]> = {
     "evalTemplate:read",
     "job:CUD",
     "job:read",
+    "jobExecution:read",
     "llmApiKeys:read",
     "llmApiKeys:create",
     "llmApiKeys:delete",
@@ -93,12 +96,14 @@ export const roleAccessRights: Record<MembershipRole, Scope[]> = {
     "evalTemplate:read",
     "job:CUD",
     "job:read",
+    "jobExecution:read",
     "llmApiKeys:read",
     "llmApiKeys:create",
     "llmApiKeys:delete",
   ],
   MEMBER: [
     "members:read",
+    "apiKeys:read",
     "objects:publish",
     "objects:bookmark",
     "objects:tag",
@@ -110,6 +115,8 @@ export const roleAccessRights: Record<MembershipRole, Scope[]> = {
     "evalTemplate:read",
     "job:read",
     "job:CUD",
+    "jobExecution:read",
+    "llmApiKeys:read",
   ],
   VIEWER: ["prompts:read"],
 };

web/src/pages/api/public/ingestion.ts

@@ -34,7 +34,7 @@ import {
   BaseError,
   ForbiddenError,
   UnauthorizedError,
-} from "@/src/server/errors";
+} from "@langfuse/shared";
 
 export const config = {
   api: {
@@ -126,7 +126,7 @@ export default async function handler(
       res,
     );
   } catch (error: unknown) {
-    console.error(error);
+    console.error("error handling ingestion event", error);
 
     if (error instanceof BaseError) {
       return res.status(error.httpCode).json({
@@ -141,6 +141,7 @@ export default async function handler(
       });
     }
     if (error instanceof z.ZodError) {
+      console.log(`Zod exception`, error.errors);
       return res.status(400).json({
         message: "Invalid request data",
         error: error.errors,

web/src/pages/api/public/prompts.ts

@@ -1,4 +1,4 @@
-import { createPrompt } from "@/src/features/prompts/server/createPrompt";
+import { createPrompt } from "@/src/features/prompts/server/actions/createPrompt";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
 import { prisma } from "@langfuse/shared/src/db";
@@ -6,16 +6,17 @@ import { isPrismaException } from "@/src/utils/exceptions";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { z } from "zod";
 import {
-  CreatePromptSchema,
+  LegacyCreatePromptSchema,
   GetPromptSchema,
-} from "@/src/features/prompts/server/validation";
+} from "@/src/features/prompts/server/utils/validation";
 import {
   UnauthorizedError,
-  NotFoundError,
+  LangfuseNotFoundError,
   BaseError,
   MethodNotAllowedError,
   ForbiddenError,
-} from "@/src/server/errors";
+} from "@langfuse/shared";
+import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
 
 export default async function handler(
   req: NextApiRequest,
@@ -42,27 +43,40 @@ export default async function handler(
           projectId: authCheck.scope.projectId,
           name: searchParams.name,
           version: searchParams.version ?? undefined, // if no version is given, we take the latest active prompt
-          isActive: !searchParams.version ? true : undefined, // if no prompt is active, there will be no prompt available
+          labels: !searchParams.version
+            ? {
+                has: PRODUCTION_LABEL,
+              }
+            : undefined, // if no prompt is active, there will be no prompt available
         },
       });
 
-      if (!prompt) throw new NotFoundError("Prompt not found");
+      if (!prompt) throw new LangfuseNotFoundError("Prompt not found");
 
-      return res.status(200).json(prompt);
+      return res.status(200).json({
+        ...prompt,
+        isActive: prompt.labels.includes(PRODUCTION_LABEL),
+      });
     }
 
     // Handle POST requests
     if (req.method === "POST") {
-      const input = CreatePromptSchema.parse(req.body);
+      const input = LegacyCreatePromptSchema.parse(req.body);
       const prompt = await createPrompt({
         ...input,
+        labels: input.isActive
+          ? [...new Set([...input.labels, PRODUCTION_LABEL])] // Ensure labels are unique
+          : input.labels, // If production label is already present, this will still promote the prompt
         config: input.config ?? {}, // Config can be null in which case zod default value is not used
         projectId: authCheck.scope.projectId,
         createdBy: "API",
         prisma: prisma,
       });
 
-      return res.status(201).json(prompt);
+      return res.status(201).json({
+        ...prompt,
+        isActive: prompt.labels.includes(PRODUCTION_LABEL),
+      });
     }
 
     throw new MethodNotAllowedError();

web/src/pages/api/public/v2/prompts/[promptName].ts

@@ -0,0 +1 @@
+export { promptNameHandler as default } from "@/src/features/prompts/server/handlers/promptNameHandler";

web/src/pages/api/public/v2/prompts/index.ts

@@ -0,0 +1 @@
+export { promptsHandler as default } from "@/src/features/prompts/server/handlers/promptsHandler";

web/src/pages/project/[projectId]/evals/configs/[configId].tsx

@@ -1,3 +1,3 @@
-import { EvalConfigDetail } from "@/src/features/evals/components/eval-config-detail";
+import { EvalConfigDetail } from "@/src/ee/features/evals/components/eval-config-detail";
 
 export default EvalConfigDetail;

web/src/pages/project/[projectId]/evals/configs/index.tsx

@@ -1,47 +1 @@
-// NOTE: We may transition this feature from our MIT licensed repository to the
-// a commercial License (ee folder) once we release a first stable version.
-// Please consider this when planning long-term use and integration of this functionality into your projects.
-// For more information see https://langfuse.com/docs/open-source
-
-import Header from "@/src/components/layouts/header";
-import { useRouter } from "next/router";
-import { Button } from "@/src/components/ui/button";
-import Link from "next/link";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { Lock } from "lucide-react";
-import EvalConfigTable from "@/src/features/evals/components/eval-config-table";
-
-export default function ConfigsPage() {
-  const router = useRouter();
-  const projectId = router.query.projectId as string;
-
-  const hasWriteAccess = useHasAccess({
-    projectId,
-    scope: "job:CUD",
-  });
-
-  return (
-    <div>
-      <Header
-        title="Eval configs"
-        help={{
-          description: "XXX",
-          href: "https://langfuse.com/docs/evals",
-        }}
-        actionButtons={
-          <Button disabled={!hasWriteAccess} asChild>
-            <Link
-              href={
-                hasWriteAccess ? `/project/${projectId}/evals/configs/new` : "#"
-              }
-            >
-              {!hasWriteAccess && <Lock size={16} className="mr-2" />}
-              Add eval config
-            </Link>
-          </Button>
-        }
-      />
-      <EvalConfigTable projectId={projectId} />
-    </div>
-  );
-}
+export { default as default } from "@/src/ee/features/evals/pages/configs";

web/src/pages/project/[projectId]/evals/configs/new.tsx

@@ -1,26 +1 @@
-import Header from "@/src/components/layouts/header";
-import { EvalConfigForm } from "@/src/features/evals/components/eval-config-form";
-import { api } from "@/src/utils/api";
-
-import { useRouter } from "next/router";
-
-export default function NewConfigsPage() {
-  const router = useRouter();
-  const projectId = router.query.projectId as string;
-
-  const evalTemplates = api.evals.allTemplates.useQuery({
-    projectId,
-    limit: 500,
-    page: 0,
-  });
-
-  return (
-    <div>
-      <Header title="Create eval config" />
-      <EvalConfigForm
-        projectId={projectId}
-        evalTemplates={evalTemplates.data?.templates ?? []}
-      />
-    </div>
-  );
-}
+export { default as default } from "@/src/ee/features/evals/pages/new-configs";

web/src/pages/project/[projectId]/evals/log.tsx

@@ -0,0 +1,26 @@
+// NOTE: We may transition this feature from our MIT licensed repository to the
+// a commercial License (ee folder) once we release a first stable version.
+// Please consider this when planning long-term use and integration of this functionality into your projects.
+// For more information see https://langfuse.com/docs/open-source
+
+import Header from "@/src/components/layouts/header";
+import { useRouter } from "next/router";
+import EvalLogTable from "@/src/ee/features/evals/components/eval-log";
+
+export default function TemplatesPage() {
+  const router = useRouter();
+  const projectId = router.query.projectId as string;
+
+  return (
+    <div>
+      <Header
+        title="Eval Log"
+        help={{
+          description: "View of all running evals.",
+          href: "https://langfuse.com/docs/evals",
+        }}
+      />
+      <EvalLogTable projectId={projectId} />
+    </div>
+  );
+}

web/src/pages/project/[projectId]/evals/templates/[id].tsx

@@ -1,3 +1,3 @@
-import { EvalTemplateDetail } from "@/src/features/evals/components/eval-template-detail";
+import { EvalTemplateDetail } from "@/src/ee/features/evals/components/eval-template-detail";
 
 export default EvalTemplateDetail;

web/src/pages/project/[projectId]/evals/templates/index.tsx

@@ -1,49 +1 @@
-// NOTE: We may transition this feature from our MIT licensed repository to the
-// a commercial License (ee folder) once we release a first stable version.
-// Please consider this when planning long-term use and integration of this functionality into your projects.
-// For more information see https://langfuse.com/docs/open-source
-
-import Header from "@/src/components/layouts/header";
-import { useRouter } from "next/router";
-import { Button } from "@/src/components/ui/button";
-import Link from "next/link";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { Lock } from "lucide-react";
-import EvalsTemplateTable from "@/src/features/evals/components/eval-templates-table";
-
-export default function TemplatesPage() {
-  const router = useRouter();
-  const projectId = router.query.projectId as string;
-
-  const hasWriteAccess = useHasAccess({
-    projectId,
-    scope: "evalTemplate:create",
-  });
-
-  return (
-    <div>
-      <Header
-        title="Eval Templates"
-        help={{
-          description: "XXX",
-          href: "https://langfuse.com/docs/evals",
-        }}
-        actionButtons={
-          <Button disabled={!hasWriteAccess} asChild>
-            <Link
-              href={
-                hasWriteAccess
-                  ? `/project/${projectId}/evals/templates/new`
-                  : "#"
-              }
-            >
-              {!hasWriteAccess && <Lock size={16} className="mr-2" />}
-              Add eval template
-            </Link>
-          </Button>
-        }
-      />
-      <EvalsTemplateTable projectId={projectId} />
-    </div>
-  );
-}
+export { default as default } from "@/src/ee/features/evals/pages/templates";

web/src/pages/project/[projectId]/evals/templates/new.tsx

@@ -1,34 +1 @@
-import Header from "@/src/components/layouts/header";
-import { EvalTemplateForm } from "@/src/features/evals/components/template-form";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { api } from "@/src/utils/api";
-
-import { useRouter } from "next/router";
-
-export default function NewTemplatesPage() {
-  const router = useRouter();
-  const projectId = router.query.projectId as string;
-
-  const hasAccess = useHasAccess({ projectId, scope: "llmApiKeys:read" });
-
-  if (!hasAccess) {
-    return null;
-  }
-
-  const llmApiKeys = api.llmApiKey.all.useQuery({
-    projectId: projectId,
-  });
-
-  return llmApiKeys.isLoading || !llmApiKeys.data ? (
-    <div>Loading...</div>
-  ) : (
-    <div className="md:container">
-      <Header title="Create eval template" />
-      <EvalTemplateForm
-        projectId={projectId}
-        isEditing={true}
-        existingLlmApiKeys={llmApiKeys.data?.data ?? []}
-      />
-    </div>
-  );
-}
+export { default as default } from "@/src/ee/features/evals/pages/new-template";

web/src/server/api/definitions/promptsTable.ts

@@ -1,4 +1,4 @@
-import { PromptType } from "@/src/features/prompts/server/validation";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import {
   type ColumnDefinition,
   type OptionsDefinition,
@@ -30,6 +30,13 @@ export const promptsTableCols: ColumnDefinition[] = [
     internal: 'p."type"',
     options: Object.values(PromptType).map((value) => ({ value })),
   },
+  {
+    name: "Labels",
+    id: "labels",
+    type: "arrayOptions",
+    internal: 'p."labels"',
+    options: [], // to be added at runtime
+  },
   {
     name: "Tags",
     id: "tags",

web/src/server/api/definitions/scoresTable.ts

@@ -37,6 +37,12 @@ export const scoresTableCols: ColumnDefinition[] = [
   },
   { name: "Value", id: "value", type: "number", internal: 's."value"' },
   { name: "User ID", id: "userId", type: "string", internal: 't."user_id"' },
+  {
+    name: "Eval Configuration ID",
+    id: "jobConfigurationId",
+    type: "string",
+    internal: 'je."job_configuration_id"',
+  },
 ];
 
 export type ScoreOptions = {

web/src/server/api/root.ts

@@ -12,9 +12,9 @@ import { environmentRouter } from "@/src/server/api/routers/environment";
 import { usageMeteringRouter } from "@/src/features/usage-metering/server/usageMeteringRouter";
 import { observationsRouter } from "@/src/server/api/routers/observations";
 import { sessionRouter } from "@/src/server/api/routers/sessions";
-import { promptRouter } from "@/src/features/prompts/server/prompt-router";
+import { promptRouter } from "@/src/features/prompts/server/routers/promptRouter";
 import { modelRouter } from "@/src/server/api/routers/models";
-import { evalRouter } from "@/src/features/evals/server/router";
+import { evalRouter } from "@/src/ee/features/evals/server/router";
 import { posthogIntegrationRouter } from "@/src/features/posthog-integration/posthog-integration-router";
 import { llmApiKeyRouter } from "@/src/features/llm-api-key/server/router";
 

web/src/server/api/routers/scores.ts

@@ -48,7 +48,13 @@ export const scoresRouter = createTRPCRouter({
       );
 
       const scores = await ctx.prisma.$queryRaw<
-        Array<Score & { traceName: string | null; userId: string | null }>
+        Array<
+          Score & {
+            traceName: string | null;
+            userId: string | null;
+            jobConfigurationId: string | null;
+          }
+        >
       >(
         generateScoresQuery(
           Prisma.sql` 
@@ -60,7 +66,9 @@ export const scoresRouter = createTRPCRouter({
           s.trace_id as "traceId",
           s.observation_id as "observationId",
           t.user_id as "userId",
-          t.name as "traceName"`,
+          t.name as "traceName",
+          je.job_configuration_id as "jobConfigurationId"
+          `,
           input.projectId,
           filterCondition,
           orderByCondition,
@@ -322,7 +330,7 @@ const generateScoresQuery = (
   SELECT
    ${select}
   FROM scores s
-  JOIN traces t ON t.id = s.trace_id
+  JOIN traces t ON t.id = s.trace_id LEFT JOIN job_executions je ON je.job_output_score_id = s.id AND je.project_id = ${projectId}
   WHERE t.project_id = ${projectId}
   ${filterCondition}
   ${orderCondition}

web/src/server/api/services/EventProcessor.ts

@@ -28,7 +28,7 @@ import { v4 } from "uuid";
 import { type z } from "zod";
 import { jsonSchema } from "@/src/utils/zod";
 import { sendToBetterstack } from "@/src/features/betterstack/server/betterstack-webhook";
-import { ForbiddenError } from "@/src/server/errors";
+import { ForbiddenError } from "@langfuse/shared";
 
 export interface EventProcessor {
   process(

web/src/server/api/services/event-service.ts

@@ -2,6 +2,8 @@ import { type Prisma, type PrismaClient } from "@langfuse/shared/src/db";
 import { type NextApiRequest } from "next";
 import { type jsonSchema } from "@/src/utils/zod";
 import lodash from "lodash";
+import { env } from "@/src/env.mjs";
+import { randomUUID } from "crypto";
 
 // This function persists raw events to the database which came via API
 // It relates each event to a project
@@ -16,6 +18,9 @@ export const persistEventMiddleware = async (
   data: Prisma.JsonObject,
   metadata?: Zod.infer<typeof jsonSchema> | null,
 ) => {
+  // If event logging is disabled, do nothing
+  if (env.ENABLE_EVENT_LOG === "false") return;
+
   const langfuseHeadersObject = Object.fromEntries(
     Object.entries(req.headers).filter(([key]) => key.startsWith("x-langfuse")),
   );
@@ -23,13 +28,15 @@ export const persistEventMiddleware = async (
   // combine metadata from the request and langfuseHeadersObject
   const combinedMetadata = lodash.merge(metadata, langfuseHeadersObject);
 
-  await prisma.events.create({
-    data: {
-      project: { connect: { id: projectId } },
-      url: req.url,
-      method: req.method,
-      data: data,
-      headers: combinedMetadata,
-    },
-  });
+  await prisma.$queryRaw`
+    INSERT INTO events (id, project_id, url, method, data, headers)
+    VALUES (
+      ${randomUUID()},
+      ${projectId},
+      ${req.url},
+      ${req.method},
+      ${data},
+      ${combinedMetadata}
+    );
+  `;
 };