chore: release v2.22.0

* feat: enable up/down nav for traces/datasetItems coming from list of runitems

* fix

.env.dev.example

@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""
 
 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret

.env.local.example

@@ -16,4 +16,11 @@ SALT="salt"
 
 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword

.env.prod.example

@@ -37,6 +37,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
+# AUTH_DISABLE_SIGNUP=true
 
 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -76,11 +77,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # DB_EXPORT_PAGE_SIZE=1000
 
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
@@ -105,6 +101,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
+# Prompt playground
+# OPENAI_API_KEY=""
+# ANTHROPIC_API_KEY=""
+
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 

.github/CODEOWNERS

@@ -1 +1,2 @@
-* @langfuse/founders
+# Currently inactive
+# * @langfuse/maintainers

.github/dependabot.yml

@@ -7,6 +7,7 @@ version: 2
 updates:
   - package-ecosystem: npm
     directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
     schedule:
       interval: "daily"
     versioning-strategy: "increase"

.github/workflows/pipeline.yml

@@ -54,7 +54,7 @@ jobs:
           file: ./worker/Dockerfile
           push: false
 
-  tests:
+  tests-web:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -102,7 +102,56 @@ jobs:
         run: (pnpm run start&)
 
       - name: run tests
-        run: pnpm run test
+        run: pnpm --filter=web run test
+
+  tests-worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 8
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+
+      - name: Build
+        run: pnpm --filter=worker... run build
+
+      - name: run tests
+        run: pnpm --filter=worker run test
 
   e2e-tests:
     runs-on: ubuntu-latest
@@ -140,9 +189,7 @@ jobs:
         run: pnpm run build
 
       - name: Install playwright
-        run: |
-          npm add --global playwright
-          pnpm exec playwright install
+        run: pnpm --filter=web exec playwright install --with-deps
 
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
@@ -150,7 +197,7 @@ jobs:
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests, test-docker-build]
+    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
     if: always()
     steps:
       - name: Successful deploy
@@ -195,7 +242,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
         uses: docker/metadata-action@v4
         with:
           images: |
@@ -208,11 +255,34 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
           context: .
           file: ./web/Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ github.repository }}-worker
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}

CONTRIBUTING.md

@@ -192,7 +192,7 @@ Requirements
   To run migrations, you can execute the following command.
 
   ```bash
-  pnpm --filter=shared run db:migrate
+  pnpm run db:migrate -- --name <name of the migration>
   ```
 
 > [!NOTE]

docker-compose.build.yml

@@ -16,11 +16,9 @@ services:
       - SALT=mysalt
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   worker:
@@ -30,7 +28,6 @@ services:
     depends_on:
       - db
       - redis
-      - server
     ports:
       - "3030:3030"
     environment:
@@ -39,9 +36,10 @@ services:
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
       - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
       - REDIS_PORT=${REDIS_PORT:-6379}
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   redis:

docker-compose.yml

@@ -13,7 +13,6 @@ services:
       - SALT=mysalt
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
 
   db:

fern/apis/server/definition/commons.yml

@@ -122,29 +122,46 @@ types:
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
-  Dataset:
+  DatasetCore:
     properties:
       id: string
       name: string
+      description: optional<string>
       projectId: string
       createdAt: datetime
       updatedAt: datetime
+  DatasetWithReferences: # GET datasets
+    extends: DatasetCore
+    properties:
+      items:
+        type: list<string>
+        docs: list of dataset item ids
+      runs:
+        type: list<string>
+        docs: list of dataset run names
+  Dataset: # GET datasets/[name]
+    docs: Dataset including all items
+    extends: DatasetCore
+    properties:
       items: list<DatasetItem>
       runs: list<string>
   DatasetItem:
     properties:
       id: string
       status: DatasetStatus
-      input: unknown
+      input: optional<unknown>
       expectedOutput: optional<unknown>
+      sourceTraceId: optional<string>
       sourceObservationId: optional<string>
       datasetId: string
+      datasetName: string
       createdAt: datetime
       updatedAt: datetime
   DatasetRunItem:
     properties:
       id: string
       datasetRunId: string
+      datasetRunName: string
       datasetItemId: string
       traceId: string
       observationId: optional<string>
@@ -154,8 +171,10 @@ types:
     properties:
       id: string
       name: string
+      description: optional<string>
       metadata: optional<unknown>
       datasetId: string
+      datasetName: string
       createdAt: datetime
       updatedAt: datetime
       datasetRunItems: list<DatasetRunItem>

fern/apis/server/definition/dataset-items.yml

@@ -24,7 +24,7 @@ types:
   CreateDatasetItemRequest:
     properties:
       datasetName: string
-      input: unknown
+      input: optional<unknown>
       expectedOutput: optional<unknown>
       id:
         type: optional<string>

fern/apis/server/definition/dataset-run-items.yml

@@ -15,6 +15,9 @@ types:
   CreateDatasetRunItemRequest:
     properties:
       runName: string
+      runDescription:
+        type: optional<string>
+        docs: Description of the run. If run exists, description will be updated.
       metadata:
         type: optional<unknown>
         docs: Metadata of the dataset run, updates run if run already exists

fern/apis/server/definition/datasets.yml

@@ -1,10 +1,21 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
   endpoints:
+    list:
+      method: GET
+      docs: Get all datasets
+      path: /datasets
+      request:
+        name: GetDatasetsRequest
+        query-parameters:
+          page: optional<integer>
+          limit: optional<integer>
+      response: PaginatedDatasets
     get:
       method: GET
       docs: Get a dataset and its items
@@ -28,6 +39,11 @@ service:
       response: commons.DatasetRun
 
 types:
+  PaginatedDatasets:
+    properties:
+      data: list<commons.DatasetWithReferences>
+      meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
+      description: optional<string>

fern/apis/server/definition/metrics.yml

@@ -13,8 +13,12 @@ service:
       request:
         name: GetDailyMetricsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
           traceName:
             type: optional<string>
             docs: Optional filter by the name of the trace

fern/apis/server/definition/observations.yml

@@ -22,8 +22,12 @@ service:
       request:
         name: GetObservationsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
           name: optional<string>
           userId: optional<string>
           type: optional<string>

fern/apis/server/definition/prompts.yml

@@ -25,6 +25,20 @@ service:
 
 types:
   CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
     properties:
       name: string
       isActive:
@@ -34,8 +48,29 @@ types:
       config: optional<unknown>
 
   Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
     properties:
       name: string
       version: integer
-      prompt: string
       config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>

fern/apis/server/definition/score.yml

@@ -19,8 +19,12 @@ service:
       request:
         name: GetScoresRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
           userId: optional<string>
           name: optional<string>
           fromTimestamp:

fern/apis/server/definition/trace.yml

@@ -22,8 +22,12 @@ service:
       request:
         name: GetTracesRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
           userId: optional<string>
           name: optional<string>
           fromTimestamp:

fern/apis/server/generators.yml

@@ -11,14 +11,16 @@ groups:
           namespaceExport: Langfuse
           allowCustomFetcher: true
       - name: fernapi/fern-python-sdk
-        version: 0.7.3
+        version: 1.1.0-rc2
         output:
           location: local-file-system
           path: ../../../generated/python
         config:
           client_class_name: FernLangfuse
+          improved_imports: false
           pydantic_config:
             require_optional_fields: false
+            use_str_enums: false
       - name: fernapi/fern-typescript-node-sdk
         version: 0.7.1
         output:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.16.36"
+  "version": "0.19.30"
 }

generated/openapi-client/openapi.yml

@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer

generated/postman/collection.json

@@ -137,7 +137,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"runName\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\",\n    \"traceId\": \"example\"\n}",
+              "raw": "{\n    \"runName\": \"example\",\n    \"runDescription\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\",\n    \"traceId\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -154,6 +154,42 @@
       "description": null,
       "name": "Datasets",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all datasets",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/datasets?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "datasets"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": null
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -209,7 +245,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -357,12 +393,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 },
                 {
                   "key": "traceName",
@@ -447,12 +483,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 },
                 {
                   "key": "name",
@@ -592,7 +628,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -661,12 +697,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 },
                 {
                   "key": "userId",
@@ -854,12 +890,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 },
                 {
                   "key": "userId",

package.json

@@ -1,8 +1,9 @@
 {
   "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.22.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
+  "private": true,
   "engines": {
     "node": ">=20"
   },
@@ -15,17 +16,57 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=worker run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate"
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
   },
   "devDependencies": {
+    "@release-it/bumper": "^6.0.1",
     "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
+    "release-it": "^17.1.1",
     "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
   }
 }

packages/config-eslint/package.json

@@ -8,12 +8,12 @@
     "next.js"
   ],
   "devDependencies": {
-    "@vercel/style-guide": "^5.2.0",
+    "@vercel/style-guide": "^6.0.0",
     "eslint-config-turbo": "^1.12.4",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-only-warn": "^1.1.0",
     "@typescript-eslint/parser": "^7.1.0",
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "typescript": "^5.4.3"
+    "typescript": "^5.4.4"
   }
 }

packages/shared/package.json

@@ -5,12 +5,23 @@
   "private": true,
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
   "engines": {
     "node": ">=20"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    }
+  },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
@@ -28,24 +39,24 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.9",
-    "@langchain/core": "^0.1.53",
-    "@langchain/openai": "^0.0.25",
-    "@prisma/client": "^5.12.0",
+    "@langchain/anthropic": "^0.1.12",
+    "@langchain/core": "^0.1.55",
+    "@langchain/openai": "^0.0.26",
+    "@prisma/client": "^5.12.1",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.30",
+    "langchain": "^0.1.32",
     "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/lodash": "^4.14.202",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.4",
+    "@types/pg": "^8.11.5",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.2.0",
     "eslint": "^8.57.0",
@@ -55,12 +66,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.0",
+    "prisma": "^5.12.1",
     "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
-    "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
-    "typescript": "^5.4.3",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.4",
     "vitest": "^1.3.1"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
     API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,17 @@ export const DatasetStatus = {
     ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
     id: string;
     user_id: string;
@@ -95,6 +107,7 @@ export type CronJobs = {
 export type Dataset = {
     id: string;
     name: string;
+    description: string | null;
     project_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
@@ -102,8 +115,9 @@ export type Dataset = {
 export type DatasetItem = {
     id: string;
     status: Generated<DatasetStatus>;
-    input: unknown;
+    input: unknown | null;
     expected_output: unknown | null;
+    source_trace_id: string | null;
     source_observation_id: string | null;
     dataset_id: string;
     created_at: Generated<Timestamp>;
@@ -121,11 +135,25 @@ export type DatasetRunItems = {
 export type DatasetRuns = {
     id: string;
     name: string;
+    description: string | null;
     metadata: unknown | null;
     dataset_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -136,6 +164,33 @@ export type Events = {
     url: string | null;
     method: string | null;
 };
+export type JobConfiguration = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
+};
 export type Membership = {
     project_id: string;
     user_id: string;
@@ -250,11 +305,13 @@ export type Prompt = {
     updated_at: Generated<Timestamp>;
     project_id: string;
     created_by: string;
-    prompt: string;
+    prompt: unknown;
     name: string;
     version: number;
+    type: Generated<string>;
     is_active: boolean;
     config: Generated<unknown>;
+    tags: Generated<string[]>;
 };
 export type Score = {
     id: string;
@@ -341,7 +398,10 @@ export type DB = {
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
     datasets: Dataset;
+    eval_templates: EvalTemplate;
     events: Events;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
     membership_invitations: MembershipInvitation;
     memberships: Membership;
     models: Model;

packages/shared/prisma/migrations/20240328065738_dataset_item_input_nullable/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ALTER COLUMN "input" DROP NOT NULL;

packages/shared/prisma/migrations/20240404203640_dataset_item_source_trace_id/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "source_trace_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "dataset_items" ADD CONSTRAINT "dataset_items_source_trace_id_fkey" FOREIGN KEY ("source_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240404210315_dataset_add_descriptions/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_runs" ADD COLUMN     "description" TEXT;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "description" TEXT;

packages/shared/prisma/migrations/20240404232317_dataset_items_backfill_source_trace_id/migration.sql

@@ -0,0 +1,7 @@
+-- Backfill source_trace_id for existing dataset_items based on the linked source_observation_id
+UPDATE dataset_items
+SET source_trace_id = observations.trace_id
+FROM observations
+WHERE dataset_items.source_observation_id = observations.id
+  AND dataset_items.source_observation_id IS NOT NULL
+  AND dataset_items.source_trace_id IS NULL

packages/shared/prisma/migrations/20240405124810_prompt_to_json/migration.sql

@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;

packages/shared/prisma/migrations/20240408133037_add_objects_for_evals/migration.sql

@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240408134328_prompt_table_add_tags/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];

packages/shared/prisma/migrations/20240408134330_prompt_table_add_index_to_tags/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_tags_idx" ON "prompts" USING GIN ("tags" array_ops);

packages/shared/prisma/migrations/20240411134330_model_updates/migration.sql

@@ -0,0 +1,30 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://ai.google.dev/models/gemini#model-variations
+  -- https://cloud.google.com/vertex-ai/generative-ai/pricing
+  ('cluv2sjeo000008ih0fv23hi0', NULL, 'gemini-1.0-pro-latest', '(?i)^(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  -- stable versions priced differently
+  -- https://developers.googleblog.com/2024/02/gemini-15-available-for-private-preview-in-google-ai-studio.html
+  ('cluv2subq000108ih2mlrga6a', NULL, 'gemini-1.0-pro', '(?i)^(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2sx04000208ihbek75lsz', NULL, 'gemini-1.0-pro-001', '(?i)^(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2szw0000308ihch3n79x7', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+
+  ('cluv2t2x0000408ihfytl45l1', NULL, 'gemini-1.5-pro-latest', '(?i)^(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
+
+
+  

packages/shared/prisma/schema.prisma

@@ -92,22 +92,25 @@ model VerificationToken {
 }
 
 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id               String                 @id @default(cuid())
+    createdAt        DateTime               @default(now()) @map("created_at")
+    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
+    name             String
+    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members          Membership[]
+    traces           Trace[]
+    observations     Observation[]
+    apiKeys          ApiKey[]
+    dataset          Dataset[]
+    RawEvents        Events[]
+    invitations      MembershipInvitation[]
+    sessions         TraceSession[]
+    Prompt           Prompt[]
+    Model            Model[]
+    AuditLog         AuditLog[]
+    EvalTemplate     EvalTemplate[]
+    JobConfiguration JobConfiguration[]
+    JobExecution     JobExecution[]
 
     @@map("projects")
 }
@@ -208,6 +211,8 @@ model Trace {
 
     scores          Score[]
     DatasetRunItems DatasetRunItems[]
+    DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -361,16 +366,17 @@ enum ObservationLevel {
 }
 
 model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
     name          String
     value         Float
     source        ScoreSource
     comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]
 
     @@unique([id, traceId]) // used for upsert
     @@index(timestamp)
@@ -384,6 +390,7 @@ model Score {
 enum ScoreSource {
     API
     REVIEW
+    EVAL
 }
 
 enum PricingUnit {
@@ -421,6 +428,7 @@ model CronJobs {
 model Dataset {
     id           String        @id @default(cuid())
     name         String
+    description  String?
     projectId    String        @map("project_id")
     project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     createdAt    DateTime      @default(now()) @map("created_at")
@@ -436,8 +444,10 @@ model Dataset {
 model DatasetItem {
     id                  String            @id @default(cuid())
     status              DatasetStatus     @default(ACTIVE)
-    input               Json
+    input               Json?
     expectedOutput      Json?             @map("expected_output")
+    sourceTraceId       String?           @map("source_trace_id")
+    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
     sourceObservationId String?           @map("source_observation_id")
     sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
     datasetId           String            @map("dataset_id")
@@ -459,6 +469,7 @@ enum DatasetStatus {
 model DatasetRuns {
     id              String            @id @default(cuid())
     name            String
+    description     String?
     metadata        Json?
     datasetId       String            @map("dataset_id")
     dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
@@ -516,17 +527,20 @@ model Prompt {
 
     createdBy String @map("created_by")
 
-    prompt      String
+    prompt      Json
     name        String
     version     Int
+    type        String        @default("text")
     isActive    Boolean       @map("is_active")
     config      Json          @default("{}")
+    tags        String[]      @default([])
     Observation Observation[]
 
     @@unique([projectId, name, version])
     @@index([projectId, name, version])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("prompts")
 }
 
@@ -574,3 +588,89 @@ model AuditLog {
     @@index([createdAt])
     @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType       @map("job_type")
+    evalTemplateId String?       @map("eval_template_id")
+    evalTemplate   EvalTemplate? @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}

packages/shared/prisma/seed.ts

@@ -4,13 +4,14 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
 
 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -168,28 +169,37 @@ async function main() {
       const dataset = await prisma.dataset.create({
         data: {
           name: `demo-dataset-${datasetNumber}`,
+          description:
+            datasetNumber === 0 ? "Dataset test description" : undefined,
           projectId: project2.id,
         },
       });
 
       const datasetItemIds = [];
       for (let i = 0; i < 18; i++) {
-        const sourceObservationId =
-          Math.random() > 0.5
-            ? observations[Math.floor(Math.random() * observations.length)].id
+        const sourceObservation =
+          Math.random() > 0.3
+            ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
         const datasetItem = await prisma.datasetItem.create({
           data: {
             datasetId: dataset.id,
-            sourceObservationId: sourceObservationId,
-            input: [
-              {
-                role: "user",
-                content: "How can i create a React component?",
-              },
-            ],
+            sourceTraceId: sourceObservation?.traceId,
+            sourceObservationId:
+              Math.random() > 0.5 ? sourceObservation?.id : undefined,
+            input:
+              Math.random() > 0.3
+                ? [
+                    {
+                      role: "user",
+                      content: "How can i create a React component?",
+                    },
+                  ]
+                : undefined,
             expectedOutput:
-              "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.",
+              Math.random() > 0.3
+                ? "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both."
+                : undefined,
           },
         });
         datasetItemIds.push(datasetItem.id);
@@ -199,6 +209,7 @@ async function main() {
         const datasetRun = await prisma.datasetRuns.create({
           data: {
             name: `demo-dataset-run-${datasetRunNumber}`,
+            description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
             metadata: [
               undefined,
@@ -703,6 +714,16 @@ async function generatePrompts(project: Project) {
       version: 1,
       isActive: true,
     },
+    {
+      id: `prompt-${v4()}`,
+      projectId: project.id,
+      createdBy: "user-1",
+      prompt: "Prompt 4 content",
+      name: "Prompt 4",
+      version: 1,
+      isActive: true,
+      tags: ["tag1", "tag2"],
+    },
   ];
 
   for (const prompt of prompts) {
@@ -722,6 +743,7 @@ async function generatePrompts(project: Project) {
         name: prompt.name,
         version: prompt.version,
         isActive: prompt.isActive,
+        tags: prompt.tags,
       },
       update: {
         id: prompt.id,

packages/shared/src/db.ts

@@ -3,6 +3,14 @@
 
 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";
 
 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
@@ -22,6 +30,23 @@ declare global {
 }
 
 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+
+export const kyselyPrisma = prisma.$extends(
+  kyselyExtension({
+    kysely: (driver) =>
+      new Kysely<DB>({
+        dialect: {
+          // This is where the magic happens!
+          createDriver: () => driver,
+          // Don't forget to customize these to match your database!
+          createAdapter: () => new PostgresAdapter(),
+          createIntrospector: (db) => new PostgresIntrospector(db),
+          createQueryCompiler: () => new PostgresQueryCompiler(),
+        },
+      }),
+  })
+);
+
 export * from "@prisma/client";
 
 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;

packages/shared/src/features/evals/types.ts

@@ -0,0 +1,31 @@
+import z from "zod";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(),
+    objectName: z.string().nullish(), // can be null as this is only required for langfuseObjects other than trace
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});

packages/shared/src/filterToPrisma.ts

@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
   type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";
 
 const operatorReplacements = {
   "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
     const col = tableColumns.find(
       (c) =>
         // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
     );
     if (!col) {
       console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
         ? Prisma.raw(
             arrayOperatorReplacements[
               filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
           )
         : filter.operator in operatorReplacements
           ? Prisma.raw(
               operatorReplacements[
                 filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
             )
           : Prisma.raw(filter.operator); //checked by zod
 
@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
         break;
       case "stringOptions":
         valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
         )})`;
         break;
       case "arrayOptions":
         valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
           filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
         )}] `;
         break;
 
@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
       filter.type === "string" || filter.type === "stringObject"
         ? [
             ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw("'%' || ")
               : Prisma.empty,
             ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw(" || '%'")
               : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
   return column.name === "type" &&
     (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
   safeColumn: string,
   operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
   if (!dateOperators.includes(operator)) {
     throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
   }
 
   return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
   )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };

packages/shared/src/index.ts

@@ -1,9 +1,19 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
-
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./features/evals/types";
+export * from "./observationsTable";
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
+export * from "./server/evals/types";
 
-export { ModelUsageUnit };
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/tableDefinition.ts

@@ -41,6 +41,7 @@ export const tableNames = [
   "traces_metrics",
   "traces_parent_observation_scores",
   "sessions",
+  "prompts",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/observationsTable.ts

@@ -1,8 +1,8 @@
+import { ObservationLevel } from "@prisma/client";
 import {
   type OptionsDefinition,
   type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -164,7 +164,7 @@ export type ObservationOptions = {
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {

packages/shared/src/queues.ts

@@ -0,0 +1,41 @@
+import { z } from "zod";
+
+export const QueueEnvelope = z.object({
+  timestamp: z.string().datetime({ offset: true }),
+  id: z.string(),
+});
+
+export const TraceUpsertEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    traceId: z.string(),
+  }),
+});
+
+export const EvalExecutionEvent = QueueEnvelope.extend({
+  data: z.object({
+    projectId: z.string(),
+    jobExecutionId: z.string(),
+  }),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};

packages/shared/src/server/evals/types.ts

@@ -0,0 +1,77 @@
+import z from "zod";
+import { ModelProvider, observationsTableCols, tracesTableCols } from "../..";
+
+export const evalObjects = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      ...tracesTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      ...observationsTableCols.map((c) => ({
+        name: c.name,
+        id: c.id,
+        internal: c.internal,
+      })),
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];
+
+export const evalModels = [
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-3.5-turbo",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-4-turbo-preview",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+] as const;
+
+export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -14,13 +14,17 @@ import { ChatOpenAI } from "@langchain/openai";
 import {
   ChatMessage,
   ChatMessageRole,
+  LLMFunctionCall,
   ModelParams,
   ModelProvider,
 } from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";
 
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -30,18 +34,27 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
+    functionCall: undefined;
   }
 ): Promise<IterableReadableStream<Uint8Array>>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
+    functionCall: undefined;
   }
 ): Promise<string>;
 
+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array>> {
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   const { messages, modelParams, streaming } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
@@ -71,6 +84,22 @@ export async function fetchLLMCompletion(
     });
   }
 
+  console.log("Making LLM call with params: ", modelParams);
+
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
   if (streaming) {
     return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
   }

packages/shared/src/server/llm/types.ts

@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
   role: ChatMessageRole;
   content: string;
@@ -23,11 +26,13 @@ export type UIModelParams = Required<
 };
 
 // Generic config
-export type ModelConfig = {
-  max_tokens?: number;
-  temperature?: number;
-  top_p?: number;
-};
+export type ModelConfig = z.infer<typeof ZodModelConfig>;
+
+export const ZodModelConfig = z.object({
+  max_tokens: z.number().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+});
 
 // OpenAI
 export type OpenAIModelParams = {
@@ -72,3 +77,9 @@ export const supportedModels = {
   [ModelProvider.Anthropic]: anthropicModels,
   [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};

packages/shared/src/tracesTable.ts

@@ -1,8 +1,5 @@
-import {
-  type OptionsDefinition,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
 import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";
 
 export const tracesTableCols: ColumnDefinition[] = [
   { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
@@ -122,7 +119,7 @@ export type TraceOptions = {
 };
 
 export function tracesTableColsWithOptions(
-  options?: TraceOptions,
+  options?: TraceOptions
 ): ColumnDefinition[] {
   return tracesTableCols.map((col) => {
     if (col.id === "scores_avg") {

packages/shared/src/types.ts

@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";
 
 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;
@@ -29,4 +29,5 @@ export type TableName =
   | "generations"
   | "sessions"
   | "scores"
+  | "prompts"
   | "dashboard";

packages/shared/tsconfig.json

@@ -6,7 +6,8 @@
     "lib": ["ES2020"],
     "outDir": "./dist",
     "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

web/generated/openapi-server/openapi.yml

@@ -152,25 +152,32 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRunItemRequest'
-  /api/public/datasets/{datasetName}:
+  /api/public/datasets:
     get:
-      description: Get a dataset and its items
-      operationId: datasets_get
+      description: Get all datasets
+      operationId: datasets_list
       tags:
         - Datasets
       parameters:
-        - name: datasetName
-          in: path
-          required: true
+        - name: page
+          in: query
+          required: false
           schema:
-            type: string
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
       responses:
         '200':
           description: ''
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Dataset'
+                $ref: '#/components/schemas/PaginatedDatasets'
         '400':
           description: ''
           content:
@@ -197,7 +204,6 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
-  /api/public/datasets:
     post:
       description: Create a dataset
       operationId: datasets_create
@@ -243,6 +249,51 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRequest'
+  /api/public/datasets/{datasetName}:
+    get:
+      description: Get a dataset and its items
+      operationId: datasets_get
+      tags:
+        - Datasets
+      parameters:
+        - name: datasetName
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Dataset'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/datasets/{datasetName}/runs/{runName}:
     get:
       description: Get a dataset run and its items
@@ -396,12 +447,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -519,12 +572,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -792,12 +847,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -1043,12 +1100,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -1420,14 +1479,17 @@ components:
         - value
         - source
         - timestamp
-    Dataset:
-      title: Dataset
+    DatasetCore:
+      title: DatasetCore
       type: object
       properties:
         id:
           type: string
         name:
           type: string
+        description:
+          type: string
+          nullable: true
         projectId:
           type: string
         createdAt:
@@ -1436,6 +1498,36 @@ components:
         updatedAt:
           type: string
           format: date-time
+      required:
+        - id
+        - name
+        - projectId
+        - createdAt
+        - updatedAt
+    DatasetWithReferences:
+      title: DatasetWithReferences
+      type: object
+      properties:
+        items:
+          type: array
+          items:
+            type: string
+          description: list of dataset item ids
+        runs:
+          type: array
+          items:
+            type: string
+          description: list of dataset run names
+      required:
+        - items
+        - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
+    Dataset:
+      title: Dataset
+      type: object
+      description: Dataset including all items
+      properties:
         items:
           type: array
           items:
@@ -1445,13 +1537,10 @@ components:
           items:
             type: string
       required:
-        - id
-        - name
-        - projectId
-        - createdAt
-        - updatedAt
         - items
         - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
     DatasetItem:
       title: DatasetItem
       type: object
@@ -1460,14 +1549,20 @@ components:
           type: string
         status:
           $ref: '#/components/schemas/DatasetStatus'
-        input: {}
+        input:
+          nullable: true
         expectedOutput:
           nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
         sourceObservationId:
           type: string
           nullable: true
         datasetId:
           type: string
+        datasetName:
+          type: string
         createdAt:
           type: string
           format: date-time
@@ -1477,8 +1572,8 @@ components:
       required:
         - id
         - status
-        - input
         - datasetId
+        - datasetName
         - createdAt
         - updatedAt
     DatasetRunItem:
@@ -1489,6 +1584,8 @@ components:
           type: string
         datasetRunId:
           type: string
+        datasetRunName:
+          type: string
         datasetItemId:
           type: string
         traceId:
@@ -1505,6 +1602,7 @@ components:
       required:
         - id
         - datasetRunId
+        - datasetRunName
         - datasetItemId
         - traceId
         - createdAt
@@ -1517,10 +1615,15 @@ components:
           type: string
         name:
           type: string
+        description:
+          type: string
+          nullable: true
         metadata:
           nullable: true
         datasetId:
           type: string
+        datasetName:
+          type: string
         createdAt:
           type: string
           format: date-time
@@ -1535,6 +1638,7 @@ components:
         - id
         - name
         - datasetId
+        - datasetName
         - createdAt
         - updatedAt
         - datasetRunItems
@@ -1587,7 +1691,8 @@ components:
       properties:
         datasetName:
           type: string
-        input: {}
+        input:
+          nullable: true
         expectedOutput:
           nullable: true
         id:
@@ -1596,13 +1701,16 @@ components:
           description: Dataset items are upserted on their id
       required:
         - datasetName
-        - input
     CreateDatasetRunItemRequest:
       title: CreateDatasetRunItemRequest
       type: object
       properties:
         runName:
           type: string
+        runDescription:
+          type: string
+          nullable: true
+          description: Description of the run. If run exists, description will be updated.
         metadata:
           nullable: true
           description: Metadata of the dataset run, updates run if run already exists
@@ -1620,12 +1728,28 @@ components:
       required:
         - runName
         - datasetItemId
+    PaginatedDatasets:
+      title: PaginatedDatasets
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetWithReferences'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     CreateDatasetRequest:
       title: CreateDatasetRequest
       type: object
       properties:
         name:
           type: string
+        description:
+          type: string
+          nullable: true
       required:
         - name
     HealthResponse:
@@ -2294,6 +2418,50 @@ components:
         - name
     CreatePromptRequest:
       title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
       type: object
       properties:
         name:
@@ -2311,20 +2479,75 @@ components:
         - prompt
     Prompt:
       title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
       type: object
       properties:
         name:
           type: string
         version:
           type: integer
-        prompt:
-          type: string
         config: {}
       required:
         - name
         - version
-        - prompt
         - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object

web/next.config.mjs

@@ -12,7 +12,7 @@ import { env } from "./src/env.mjs";
  */
 const cspHeader = `
   default-src 'self' https://ph.langfuse.com https://*.posthog.com wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' ${process.env.NODE_ENV === "development" ? "'unsafe-eval'" : ""} https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
   style-src 'self' 'unsafe-inline' https://*.crisp.chat;
   img-src 'self' https: blob: data:;
   font-src 'self' https://*.crisp.chat;
@@ -77,22 +77,22 @@ const nextConfig = {
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
     ];
   },

web/package.json

@@ -1,14 +1,14 @@
 {
   "name": "web",
-  "version": "2.15.4",
+  "version": "2.22.0",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
-    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
+    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -18,22 +18,22 @@
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
     "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.515.0",
-    "@aws-sdk/s3-request-presigner": "^3.507.0",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/s3-request-presigner": "^3.540.0",
+    "@codemirror/lang-json": "^6.0.1",
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.9",
-    "@langchain/core": "^0.1.53",
-    "@langchain/openai": "^0.0.25",
+    "@langchain/anthropic": "^0.1.12",
+    "@langchain/core": "^0.1.55",
+    "@langchain/openai": "^0.0.26",
     "@langfuse/shared": "workspace:*",
-    "@marsidev/react-turnstile": "^0.5.3",
+    "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -72,11 +72,13 @@
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
-    "ai": "^3.0.17",
+    "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/react-codemirror": "^4.21.25",
+    "ai": "^3.0.20",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
-    "cmdk": "^0.2.1",
+    "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
@@ -84,16 +86,16 @@
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.10",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.30",
+    "langchain": "^0.1.32",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.330.0",
+    "lucide-react": "^0.364.0",
     "next": "^14.1.4",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.105.9",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.0",
+    "prisma": "^5.12.1",
     "react": "18.2.0",
     "react-day-picker": "^8.10.0",
     "react-dom": "18.2.0",
@@ -112,8 +114,7 @@
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.0",
     "@testing-library/jest-dom": "^6.4.2",
     "@testing-library/react": "^14.2.2",
     "@types/bcryptjs": "^2.4.6",
@@ -123,8 +124,8 @@
     "@types/lodash": "^4.14.202",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.74",
-    "@types/react-dom": "^18.2.23",
+    "@types/react": "^18.2.75",
+    "@types/react-dom": "^18.2.24",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
@@ -138,42 +139,16 @@
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.13",
-    "release-it": "^17.1.1",
     "tailwindcss": "^3.4.3",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.7.1",
-    "typescript": "^5.4.3"
+    "tsx": "^4.7.2",
+    "typescript": "^5.4.4"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"
   },
   "optionalDependencies": {
     "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
   }
 }

web/public/.gitignore

@@ -0,0 +1 @@
+openapi*.yml

web/public/openapi-client.yml

@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer

web/src/__tests__/api-auth.servertest.ts

@@ -1,4 +1,7 @@
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
+import {
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server/auth";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { prisma } from "@langfuse/shared/src/db";
 

web/src/__tests__/datasets.servertest.ts

@@ -11,6 +11,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
   it("should create and get a dataset", async () => {
     await makeAPICall("POST", "/api/public/datasets", {
       name: "dataset-name",
+      description: "dataset-description",
     });
 
     const dbDataset = await prisma.dataset.findMany({
@@ -29,6 +30,101 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(getDataset.status).toBe(200);
     expect(getDataset.body).toMatchObject({
       name: "dataset-name",
+      description: "dataset-description",
+    });
+  });
+
+  it("GET datasets", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-1",
+    });
+
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-2",
+    });
+
+    const datasetItemId = v4();
+
+    const createItemRes = await makeAPICall<{
+      datasetName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name-2",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+      id: datasetItemId,
+    });
+
+    expect(createItemRes.status).toBe(200);
+    expect(createItemRes.body).toMatchObject({
+      datasetName: "dataset-name-2", // not included in db table
+    });
+
+    const traceId = v4();
+    const observationId = v4();
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+        {
+          id: v4(),
+          type: "observation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: observationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            startTime: "2021-01-01T00:00:00.000Z",
+            endTime: "2021-01-01T00:00:00.000Z",
+            modelParameters: { key: "value" },
+            input: { key: "value" },
+            metadata: { key: "value" },
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(response.status).toBe(207);
+
+    await makeAPICall("POST", "/api/public/dataset-run-items", {
+      datasetItemId: datasetItemId,
+      observationId: observationId,
+      runName: "test-run",
+      metadata: { key: "value" },
+    });
+
+    const getDatasets = await makeAPICall("GET", `/api/public/datasets`);
+
+    expect(getDatasets.status).toBe(200);
+    expect(getDatasets.body).toMatchObject({
+      data: expect.arrayContaining([
+        expect.objectContaining({
+          name: "dataset-name-1",
+          items: [],
+          runs: [],
+        }),
+        expect.objectContaining({
+          name: "dataset-name-2",
+          items: [datasetItemId],
+          runs: ["test-run"],
+        }),
+      ]),
+      meta: expect.objectContaining({
+        totalItems: 2,
+        page: 1,
+      }),
     });
   });
 
@@ -63,6 +159,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
           id: dbDatasetItem!.id,
           input: { key: "value" },
           expectedOutput: { key: "value" },
+          datasetName: "dataset-name", // not included in db table
         },
       ],
     });
@@ -76,6 +173,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       id: dbDatasetItem!.id,
       input: { key: "value" },
       expectedOutput: { key: "value" },
+      datasetName: "dataset-name", // not included in db table
     });
   });
 
@@ -175,6 +273,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
         datasetItemId: "dataset-item-id",
         observationId: observationId,
         runName: "run-only-observation",
+        runDescription: "run-description",
         metadata: { key: "value" },
       },
     );
@@ -189,6 +288,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dbRunObservation).not.toBeNull();
     expect(dbRunObservation?.datasetId).toBe(dataset.body.id);
     expect(dbRunObservation?.metadata).toMatchObject({ key: "value" });
+    expect(dbRunObservation?.description).toBe("run-description");
     expect(runItemObservation.status).toBe(200);
     expect(dbRunObservation?.datasetRunItems[0]).toMatchObject({
       datasetItemId: "dataset-item-id",
@@ -196,16 +296,41 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       traceId: traceId,
     });
 
-    const runItemTrace = await makeAPICall(
-      "POST",
-      "/api/public/dataset-run-items",
-      {
-        datasetItemId: "dataset-item-id",
-        traceId: traceId,
-        runName: "run-only-trace",
-        metadata: { key: "value" },
-      },
+    const getRunAPI = await makeAPICall(
+      "GET",
+      `/api/public/datasets/dataset-name/runs/run-only-observation`,
     );
+    expect(getRunAPI.status).toBe(200);
+    expect(getRunAPI.body).toMatchObject({
+      name: "run-only-observation",
+      description: "run-description",
+      metadata: { key: "value" },
+      datasetId: dataset.body.id,
+      datasetName: "dataset-name",
+      datasetRunItems: expect.arrayContaining([
+        expect.objectContaining({
+          datasetItemId: "dataset-item-id",
+          observationId: observationId,
+          traceId: traceId,
+          datasetRunName: "run-only-observation",
+        }),
+      ]),
+    });
+
+    const runItemTrace = await makeAPICall<{
+      datasetRunName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-run-items", {
+      datasetItemId: "dataset-item-id",
+      traceId: traceId,
+      runName: "run-only-trace",
+      metadata: { key: "value" },
+    });
+
+    expect(runItemTrace.status).toBe(200);
+    expect(runItemTrace.body).toMatchObject({
+      datasetRunName: "run-only-trace", // not included in db table
+    });
+
     const dbRunTrace = await prisma.datasetRuns.findFirst({
       where: {
         name: "run-only-trace",

web/src/__tests__/ingestion.servertest.ts

@@ -520,8 +520,6 @@ describe("/api/public/ingestion API Endpoint", () => {
 
       expect(response.status).toBe(207);
 
-      console.log("response body", response.body);
-
       const dbGeneration = await prisma.observation.findUnique({
         where: {
           id: generationId,
@@ -851,7 +849,6 @@ describe("/api/public/ingestion API Endpoint", () => {
       ],
     });
 
-    console.log(responseOne.body);
     expect(responseOne.status).toBe(207);
 
     expect("errors" in responseOne.body).toBe(true);
@@ -901,7 +898,6 @@ describe("/api/public/ingestion API Endpoint", () => {
       ],
     });
 
-    console.log(responseOne.body);
     expect(responseOne.status).toBe(207);
 
     expect("errors" in responseOne.body).toBe(true);
@@ -1167,8 +1163,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     });
     expect(responseOne.status).toBe(207);
 
-    console.log(responseOne.body);
-
     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
     const errors = responseOne.body.errors;
 
@@ -1419,7 +1413,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
       input,
     )} ${JSON.stringify(expected)}`, () => {
       const cleanedEvent = cleanEvent(input);
-      console.log(cleanedEvent);
       expect(cleanedEvent).toStrictEqual(expected);
     });
   });

web/src/__tests__/observations.servertest.ts

@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { type ObservationView } from "@langfuse/shared/src/db";
+import { type ObservationView } from "@langfuse/shared";
 
 describe("/api/public/observations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -41,6 +41,19 @@ describe("/api/public/observations API Endpoint", () => {
       },
     });
 
+    const prompt = await prisma.prompt.create({
+      data: {
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
     await prisma.observation.create({
       data: {
         id: uuidv4(),
@@ -62,6 +75,9 @@ describe("/api/public/observations API Endpoint", () => {
         },
         internalModel: "gpt-3.5-turbo",
         unit: ModelUsageUnit.Tokens,
+        prompt: {
+          connect: { id: prompt.id },
+        },
       },
     });
 
@@ -95,6 +111,7 @@ describe("/api/public/observations API Endpoint", () => {
     expect(
       fetchedObservations.body.data[0]?.calculatedTotalCost,
     ).toBeGreaterThan(0);
+    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
   });
   it("should fetch all observations, filtered by generations", async () => {
     await pruneDatabase();

web/src/__tests__/orderByToPrisma.servertest.ts

@@ -1,5 +1,5 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
-import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+import { tracesTableCols } from "@langfuse/shared";
 
 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {

web/src/__tests__/prompts.servertest.ts

@@ -3,7 +3,12 @@
 import { prisma } from "@langfuse/shared/src/db";
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
-import { type Prompt } from "@langfuse/shared/src/db";
+import { type Prompt } from "@langfuse/shared";
+import {
+  PromptSchema,
+  PromptType,
+  type ValidatedPrompt,
+} from "@/src/features/prompts/server/validation";
 
 describe("/api/public/prompts API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -43,10 +48,12 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.id).toBe(promptId);
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
+    expect(fetchedObservations.body.tags).toEqual([]);
   });
 
   it("should fetch active prompt only if no prompt version is given", async () => {
@@ -190,6 +197,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.id).toBe(promptIdTwo);
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(2);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
@@ -221,6 +229,7 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("API");
@@ -342,8 +351,6 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(response.status).toBe(207);
 
-    console.log("response body", response.body);
-
     const dbGeneration = await prisma.observation.findUnique({
       where: {
         id: generationId,
@@ -375,11 +382,185 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(fetchedObservations.body.name).toBe("prompt-name");
     expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({});
   });
+
+  it("should create and fetch a chat prompt", async () => {
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(201);
+
+    const { body: fetchedPrompt } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+
+    const validatedPrompt = validatePrompt(fetchedPrompt);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+  });
+
+  it("should fail if chat prompt has string prompt", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if chat prompt has incorrect messages format", async () => {
+    const promptName = "prompt-name";
+    const incorrectChatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", message: "What's up?" },
+    ];
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: incorrectChatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+  it("should fail if text prompt has message format", async () => {
+    const promptName = "prompt-name";
+    const response = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: [{ role: "system", content: "You are a bot" }],
+      type: "text",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(response.status).toBe(400);
+
+    const { body, status } = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(status).toBe(404);
+    expect(body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
+
+  it("should fail if previous versions have different prompt type", async () => {
+    // Create a chat prompt
+    const promptName = "prompt-name";
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const postResponse1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: chatMessages,
+      type: "chat",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse1.status).toBe(201);
+
+    // Try creating a text prompt with the same name
+    const postResponse2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: promptName,
+      prompt: "prompt",
+      type: "text",
+      isActive: true,
+      version: 2,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    expect(postResponse2.status).toBe(400);
+    expect(postResponse2.body).toEqual({
+      error: "ValidationError",
+      message:
+        "Previous versions have different prompt type. Create a new prompt with a different name.",
+    });
+
+    // Check if the prompt is still the chat prompt
+    const getResponse1 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}`,
+      undefined,
+    );
+    expect(getResponse1.status).toBe(200);
+
+    const validatedPrompt = validatePrompt(getResponse1.body);
+
+    expect(validatedPrompt.name).toBe("prompt-name");
+    expect(validatedPrompt.prompt).toEqual(chatMessages);
+    expect(validatedPrompt.type).toBe("chat");
+    expect(validatedPrompt.version).toBe(1);
+    expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.createdBy).toBe("API");
+    expect(validatedPrompt.config).toEqual({});
+
+    // Check that the text prompt has not been created
+    const getResponse2 = await makeAPICall(
+      "GET",
+      `/api/public/prompts?name=${promptName}&version=2`,
+      undefined,
+    );
+    expect(getResponse2.status).toBe(404);
+    expect(getResponse2.body).toEqual({
+      error: "NotFoundError",
+      message: "Prompt not found",
+    });
+  });
 });
 
 const isPrompt = (x: unknown): x is Prompt => {
@@ -393,6 +574,18 @@ const isPrompt = (x: unknown): x is Prompt => {
     typeof prompt.isActive === "boolean" &&
     typeof prompt.projectId === "string" &&
     typeof prompt.createdBy === "string" &&
-    typeof prompt.config === "object"
+    typeof prompt.config === "object" &&
+    Object.values(PromptType).includes(prompt.type as PromptType)
   );
 };
+
+const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+  Object.keys(obj).forEach((key) => {
+    obj[key] =
+      key === "createdAt" || key === "updatedAt"
+        ? new Date(obj[key] as string)
+        : obj[key];
+  });
+
+  return PromptSchema.parse(obj);
+};

web/src/components/grouped-score-badge.tsx

@@ -40,7 +40,7 @@ export const GroupedScoreBadges = ({
               <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
                 <MessageCircle size={12} />
               </HoverCardTrigger>
-              <HoverCardContent>
+              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
                 <p>{s.comment}</p>
               </HoverCardContent>
             </HoverCard>

web/src/components/json-editor.tsx

@@ -0,0 +1,33 @@
+import CodeMirror, { EditorView } from "@uiw/react-codemirror";
+import { githubLight } from "@uiw/codemirror-theme-github";
+import { json } from "@codemirror/lang-json";
+
+// todo: add json linting
+
+export function JsonEditor({
+  defaultValue,
+  onChange,
+  editable = true,
+  lineWrapping = true,
+}: {
+  defaultValue: string;
+  onChange?: (value: string) => void;
+  editable?: boolean;
+  lineWrapping?: boolean;
+}) {
+  return (
+    <CodeMirror
+      value={defaultValue}
+      theme={githubLight}
+      basicSetup={{
+        foldGutter: true,
+      }}
+      lang={"json"}
+      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      defaultValue={defaultValue}
+      onChange={onChange}
+      className="overflow-hidden rounded-md border"
+      editable={editable}
+    />
+  );
+}

web/src/components/layouts/layout.tsx

@@ -309,8 +309,8 @@ export default function Layout(props: PropsWithChildren) {
               </ul>
             </nav>
 
-            <Menu as="div" className="relative left-1">
-              <Menu.Button className="flex w-full items-center gap-x-4 p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+            <Menu as="div" className="relative">
+              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
                 <span className="sr-only">Open user menu</span>
                 <Avatar className="h-8 w-8">
                   <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -342,7 +342,10 @@ export default function Layout(props: PropsWithChildren) {
                 leaveFrom="transform opacity-100 scale-100"
                 leaveTo="transform opacity-0 scale-95"
               >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                    {session.data?.user?.email}
+                  </span>
                   {userNavigation.map((item) => (
                     <Menu.Item key={item.name}>
                       {({ active }) => (
@@ -403,7 +406,10 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="transform opacity-100 scale-100"
               leaveTo="transform opacity-0 scale-95"
             >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 w-32 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  {session.data?.user?.email}
+                </span>
                 {userNavigation.map((item) => (
                   <Menu.Item key={item.name}>
                     {({ active }) => (
@@ -496,6 +502,7 @@ const MainNavigation: React.FC<{
                   "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
                 )}
                 onClick={onNavitemClick}
+                target={item.newTab ? "_blank" : undefined}
               >
                 {item.icon && (
                   <item.icon
@@ -574,6 +581,7 @@ const MainNavigation: React.FC<{
                                 : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
                               "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
                             )}
+                            target={subItem.newTab ? "_blank" : undefined}
                           >
                             {subItem.name}
                             {subItem.label && (

web/src/components/layouts/routes.ts

@@ -9,6 +9,7 @@ import {
   Settings,
   UsersIcon,
   PenSquareIcon,
+  LibraryBig,
   TerminalIcon,
 } from "lucide-react";
 
@@ -21,6 +22,7 @@ export type Route = {
   pathname?: string; // link, ignored if children
   children?: Array<Route>; // folder
   bottom?: boolean; // bottom of the sidebar, only for first level routes
+  newTab?: boolean; // open in new tab
 };
 
 export const ROUTES: Route[] = [
@@ -53,6 +55,16 @@ export const ROUTES: Route[] = [
         name: "Models",
         pathname: `/project/[projectId]/models`,
       },
+      {
+        name: "Templates",
+        pathname: `/project/[projectId]/evals/templates`,
+        featureFlag: "evals",
+      },
+      {
+        name: "Configs",
+        pathname: `/project/[projectId]/evals/configs`,
+        featureFlag: "evals",
+      },
     ],
   },
   {
@@ -83,6 +95,13 @@ export const ROUTES: Route[] = [
     icon: Settings,
     bottom: true,
   },
+  {
+    name: "Docs",
+    pathname: "https://langfuse.com/docs",
+    icon: LibraryBig,
+    bottom: true,
+    newTab: true,
+  },
   {
     name: "Support",
     pathname: "/project/[projectId]/support",

web/src/components/projectNavigation.tsx

@@ -7,7 +7,6 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { cn } from "@/src/utils/tailwind";
-import { Code } from "lucide-react";
 import { useRouter } from "next/router";
 
 interface Project {

web/src/components/table/data-table-toolbar.tsx

@@ -2,9 +2,9 @@ import { Button } from "@/src/components/ui/button";
 import React, { type Dispatch, type SetStateAction, useState } from "react";
 import { Input } from "@/src/components/ui/input";
 import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-table-column-visibility-filter";
-import { type FilterState } from "@/src/features/filters/types";
-import { FilterBuilder } from "@/src/features/filters/components/filter-builder";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import { type FilterState } from "@langfuse/shared";
+import { PopoverFilterBuilder } from "@/src/features/filters/components/filter-builder";
+import { type ColumnDefinition } from "@langfuse/shared";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 
@@ -16,11 +16,11 @@ interface SearchConfig {
 
 interface DataTableToolbarProps<TData, TValue> {
   columns: LangfuseColumnDef<TData, TValue>[];
-  filterColumnDefinition: ColumnDefinition[];
+  filterColumnDefinition?: ColumnDefinition[];
   searchConfig?: SearchConfig;
   actionButtons?: React.ReactNode;
-  filterState: FilterState;
-  setFilterState: Dispatch<SetStateAction<FilterState>>;
+  filterState?: FilterState;
+  setFilterState?: Dispatch<SetStateAction<FilterState>>;
   columnVisibility?: VisibilityState;
   setColumnVisibility?: Dispatch<SetStateAction<VisibilityState>>;
 }
@@ -64,11 +64,13 @@ export function DataTableToolbar<TData, TValue>({
             </Button>
           </div>
         )}
-        <FilterBuilder
-          columns={filterColumnDefinition}
-          filterState={filterState}
-          onChange={setFilterState}
-        />
+        {!!filterColumnDefinition && !!filterState && !!setFilterState && (
+          <PopoverFilterBuilder
+            columns={filterColumnDefinition}
+            filterState={filterState}
+            onChange={setFilterState}
+          />
+        )}
         <div className="flex-1" />
         {!!columnVisibility && !!setColumnVisibility && (
           <DataTableColumnVisibilityFilter

web/src/components/table/table-link.tsx

@@ -4,22 +4,26 @@ export type TableLinkProps = {
   path: string;
   value: string;
   truncateAt?: number;
+  icon?: React.ReactNode;
 };
 
 export default function TableLink({
   path,
   value,
   truncateAt = 7,
+  icon,
 }: TableLinkProps) {
+  const truncatedValue =
+    value.length - truncateAt > 3
+      ? `...${value.substring(value.length - truncateAt)}`
+      : value;
   return (
     <Link
       className="inline-block rounded bg-indigo-50 px-2 py-1 text-xs font-semibold text-blue-600 shadow-sm hover:bg-indigo-100"
       href={path}
       title={value}
     >
-      {value.length - truncateAt > 3
-        ? `...${value.substring(value.length - truncateAt)}`
-        : value}
+      {icon ? icon : truncatedValue}
     </Link>
   );
 }

web/src/components/table/use-cases/IOCell.tsx

@@ -13,7 +13,7 @@ export const IOCell = ({
 }) => {
   return (
     <>
-      {isLoading  ? (
+      {isLoading ? (
         <JsonSkeleton className="h-[250px] w-[500px] px-3 py-1" />
       ) : (
         <JSONView json={data} className="h-[250px] w-[500px] overflow-y-auto" />
@@ -42,10 +42,8 @@ export const JsonSkeleton = ({
   return (
     <div className={cn("w-[500px] rounded-md border", className)}>
       <div className="flex flex-col gap-1">
-        {[...Array<number>(numRows)].map((_) => (
-          <>
-            <Skeleton className={generateRandomSize()} />
-          </>
+        {[...Array<number>(numRows)].map((_, i) => (
+          <Skeleton className={generateRandomSize()} key={i} />
         ))}
         <br />
       </div>

web/src/components/table/use-cases/generations.tsx

@@ -22,7 +22,6 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
 import {
   formatIntervalSeconds,
   intervalInSeconds,
@@ -30,14 +29,15 @@ import {
 } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type ObservationLevel } from "@langfuse/shared/src/db";
+import { type ObservationLevel } from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
 import { usdFormatter } from "@/src/utils/numbers";
 import {
   exportOptions,
   type ExportFileFormats,
-} from "@/src/server/api/interfaces/exportTypes";
+  observationsTableColsWithOptions,
+} from "@langfuse/shared";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
@@ -695,6 +695,7 @@ const GenerationsIOCell = ({
           skipBatch: true,
         },
       },
+      refetchOnMount: false, // prevents refetching loops
     },
   );
   return (

web/src/components/table/use-cases/scores.tsx

@@ -6,11 +6,11 @@ import useColumnVisibility from "@/src/features/column-visibility/hooks/useColum
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import {
-  ScoreOptions,
+  type ScoreOptions,
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
-import { RouterOutput, type RouterInput } from "@/src/utils/types";
+import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
 export type ScoresTableRow = {

web/src/components/table/use-cases/sessions.tsx

@@ -6,7 +6,7 @@ import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsTableColsWithOptions } from "@/src/server/api/definitions/sessionsView";

web/src/components/table/use-cases/traces.tsx

@@ -10,17 +10,9 @@ import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import { Checkbox } from "@/src/components/ui/checkbox";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import { type FilterState } from "@/src/features/filters/types";
-import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
-import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
-import {
-  TraceOptions,
-  tracesTableColsWithOptions,
-} from "@/src/server/api/definitions/tracesTable";
 import { api } from "@/src/utils/api";
 import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import { type RouterInput, type RouterOutput } from "@/src/utils/types";
-import { type ObservationLevel, type Score } from "@langfuse/shared/src/db";
 import { type RowSelectionState } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
 import {
@@ -37,6 +29,15 @@ import { LevelColors } from "@/src/components/level-colors";
 import { cn } from "@/src/utils/tailwind";
 import { IOCell } from "./IOCell";
 import { setSmallPaginationIfColumnsVisible } from "@/src/features/column-visibility/hooks/setSmallPaginationIfColumnsVisible";
+import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
+import {
+  type FilterState,
+  type TraceOptions,
+  tracesTableColsWithOptions,
+  type ObservationLevel,
+  type Score,
+} from "@langfuse/shared";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -124,6 +125,7 @@ export default function TracesTable({
     filter: filterState,
     searchQuery,
     orderBy: orderByState,
+    returnIO: false,
   };
   const traces = api.traces.all.useQuery(tracesAllQueryFilter);
 
@@ -465,9 +467,7 @@ export default function TracesTable({
       id: "input",
       cell: ({ row }) => {
         const traceId: string = row.getValue("id");
-        return (
-          <TracesIOCell traceId={traceId} projectId={projectId} io="input" />
-        );
+        return <TracesIOCell traceId={traceId} io="input" />;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -478,9 +478,7 @@ export default function TracesTable({
       id: "output",
       cell: ({ row }) => {
         const traceId: string = row.getValue("id");
-        return (
-          <TracesIOCell traceId={traceId} projectId={projectId} io="output" />
-        );
+        return <TracesIOCell traceId={traceId} io="output" />;
       },
       enableHiding: true,
       defaultHidden: true,
@@ -642,30 +640,14 @@ export default function TracesTable({
 }
 
 const TracesIOCell = ({
-  projectId,
   traceId,
   io,
 }: {
-  projectId: string;
   traceId: string;
   io: "input" | "output";
 }) => {
-  const trace = api.traces.all.useQuery(
-    {
-      projectId: projectId,
-      filter: [
-        {
-          column: "id",
-          type: "string",
-          operator: "=",
-          value: traceId,
-        },
-      ],
-      searchQuery: null,
-      orderBy: null,
-      page: 0,
-      limit: 1,
-    },
+  const trace = api.traces.byId.useQuery(
+    { traceId: traceId },
     {
       enabled: typeof traceId === "string",
       trpc: {
@@ -673,16 +655,13 @@ const TracesIOCell = ({
           skipBatch: true,
         },
       },
+      refetchOnMount: false, // prevents refetching loops
     },
   );
   return (
     <IOCell
       isLoading={trace.isLoading}
-      data={
-        io === "output"
-          ? trace.data?.traces[0]?.output
-          : trace.data?.traces[0]?.input
-      }
+      data={io === "output" ? trace.data?.output : trace.data?.input}
     />
   );
 };

web/src/components/token-usage-badge.tsx

@@ -1,6 +1,6 @@
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
-import { type Observation } from "@langfuse/shared/src/db";
+import { type Observation } from "@langfuse/shared";
 
 export const TraceAggUsageBadge = (props: {
   observations: ObservationReturnType[];

web/src/components/trace/IOPreview.tsx

@@ -136,18 +136,18 @@ const ChatMlMessageSchema = z
     content,
     json: Object.keys(other).length === 0 ? undefined : other,
   }));
-const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
+export const ChatMlArraySchema = z.array(ChatMlMessageSchema).min(1);
 
-const OpenAiMessageView: React.FC<{
+export const OpenAiMessageView: React.FC<{
+  title?: string;
   messages: z.infer<typeof ChatMlArraySchema>;
-}> = ({ messages }) => {
+}> = ({ title, messages }) => {
   const COLLAPSE_THRESHOLD = 3;
   const [isCollapsed, setCollapsed] = useState(
     messages.length > COLLAPSE_THRESHOLD ? true : null,
   );
 
   const transformedMessages = messages;
-  console.log(transformedMessages);
   // const transformedMessages = messages.map(
   //   ({ role, name, content, ...rest }) => ({
   //     role,
@@ -158,58 +158,65 @@ const OpenAiMessageView: React.FC<{
   // );
 
   return (
-    <div className="flex flex-col gap-2 rounded-md border p-3">
-      {transformedMessages
-        .filter(
-          (_, i) =>
-            // show all if not collapsed or null; show first and last n if collapsed
-            !isCollapsed || i == 0 || i > messages.length - COLLAPSE_THRESHOLD,
-        )
-        .map((message, index) => (
-          <Fragment key={index}>
-            <div>
-              {!!message.content && (
-                <JSONView
-                  title={message.name ?? message.role}
-                  json={message.content}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.json && "rounded-b-none",
-                  )}
-                />
-              )}
-              {!!message.json && (
-                <JSONView
-                  title={
-                    message.content ? undefined : message.name ?? message.role
-                  }
-                  json={message.json}
-                  className={cn(
-                    "bg-gray-100",
-                    message.role === "system" && "bg-gray-100",
-                    message.role === "assistant" && "bg-green-50",
-                    message.role === "user" && "bg-white",
-                    !!message.content && "rounded-t-none border-t-0",
-                  )}
-                />
-              )}
-            </div>
-            {isCollapsed !== null && index === 0 ? (
-              <Button
-                variant="ghost"
-                size="xs"
-                onClick={() => setCollapsed((v) => !v)}
-              >
-                {isCollapsed
-                  ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
-                  : "Hide history"}
-              </Button>
-            ) : null}
-          </Fragment>
-        ))}
+    <div className="rounded-md border">
+      {title && (
+        <div className="border-b px-3 py-1 text-xs font-medium">{title}</div>
+      )}
+      <div className="flex flex-col gap-2 p-3">
+        {transformedMessages
+          .filter(
+            (_, i) =>
+              // show all if not collapsed or null; show first and last n if collapsed
+              !isCollapsed ||
+              i == 0 ||
+              i > messages.length - COLLAPSE_THRESHOLD,
+          )
+          .map((message, index) => (
+            <Fragment key={index}>
+              <div>
+                {!!message.content && (
+                  <JSONView
+                    title={message.name ?? message.role}
+                    json={message.content}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.json && "rounded-b-none",
+                    )}
+                  />
+                )}
+                {!!message.json && (
+                  <JSONView
+                    title={
+                      message.content ? undefined : message.name ?? message.role
+                    }
+                    json={message.json}
+                    className={cn(
+                      "bg-gray-100",
+                      message.role === "system" && "bg-gray-100",
+                      message.role === "assistant" && "bg-green-50",
+                      message.role === "user" && "bg-white",
+                      !!message.content && "rounded-t-none border-t-0",
+                    )}
+                  />
+                )}
+              </div>
+              {isCollapsed !== null && index === 0 ? (
+                <Button
+                  variant="ghost"
+                  size="xs"
+                  onClick={() => setCollapsed((v) => !v)}
+                >
+                  {isCollapsed
+                    ? `Show ${messages.length - COLLAPSE_THRESHOLD} more ...`
+                    : "Hide history"}
+                </Button>
+              ) : null}
+            </Fragment>
+          ))}
+      </div>
     </div>
   );
 };

web/src/components/trace/ObservationPreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/code";
-import { type Score } from "@langfuse/shared/src/db";
+import { type Score } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -17,7 +17,7 @@ import {
   TableRow,
 } from "@/src/components/ui/table";
 import { ManualScoreButton } from "@/src/features/manual-scoring/components/ManualScoreButton";
-import { NewDatasetItemFromObservationButton } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
+import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { api } from "@/src/utils/api";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -128,11 +128,12 @@ export const ObservationPreview = (props: {
             scores={props.scores}
           />
           {observationWithInputAndOutput.data ? (
-            <NewDatasetItemFromObservationButton
+            <NewDatasetItemFromTrace
+              traceId={preloadedObservation.traceId}
               observationId={preloadedObservation.id}
               projectId={props.projectId}
-              observationInput={observationWithInputAndOutput.data.input}
-              observationOutput={observationWithInputAndOutput.data.output}
+              input={observationWithInputAndOutput.data.input}
+              output={observationWithInputAndOutput.data.output}
               key={preloadedObservation.id}
             />
           ) : null}

web/src/components/trace/ObservationTree.tsx

@@ -1,6 +1,6 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score, $Enums } from "@langfuse/shared/src/db";
+import { type Trace, type Score, $Enums } from "@langfuse/shared";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";

web/src/components/trace/TracePreview.tsx

@@ -1,5 +1,5 @@
 import { JSONView } from "@/src/components/ui/code";
-import { type Trace, type Score } from "@langfuse/shared/src/db";
+import { type Trace, type Score } from "@langfuse/shared";
 import {
   Card,
   CardContent,
@@ -22,6 +22,7 @@ import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
 import { formatIntervalSeconds } from "@/src/utils/dates";
+import { NewDatasetItemFromTrace } from "@/src/features/datasets/components/NewDatasetItemFromObservationButton";
 
 export const TracePreview = ({
   trace,
@@ -58,11 +59,20 @@ export const TracePreview = ({
             )}
           </div>
         </div>
-        <ManualScoreButton
-          projectId={trace.projectId}
-          traceId={trace.id}
-          scores={scores}
-        />
+        <div className="flex gap-2">
+          <ManualScoreButton
+            projectId={trace.projectId}
+            traceId={trace.id}
+            scores={scores}
+          />
+          <NewDatasetItemFromTrace
+            traceId={trace.id}
+            projectId={trace.projectId}
+            input={trace.input}
+            output={trace.output}
+            key={trace.id}
+          />
+        </div>
       </CardHeader>
       <CardContent className="flex flex-col gap-4">
         <IOPreview

web/src/components/trace/index.tsx

@@ -1,4 +1,4 @@
-import { type Trace, type Score } from "@langfuse/shared/src/db";
+import { type Trace, type Score } from "@langfuse/shared";
 import { ObservationTree } from "./ObservationTree";
 import { ObservationPreview } from "./ObservationPreview";
 import { TracePreview } from "./TracePreview";

web/src/components/ui/command.tsx

@@ -120,7 +120,7 @@ const CommandItem = React.forwardRef<
   <CommandPrimitive.Item
     ref={ref}
     className={cn(
-      "relative flex cursor-pointer select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none aria-selected:bg-slate-100 aria-selected:text-slate-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:aria-selected:bg-slate-800 dark:aria-selected:text-slate-50",
+      "relative flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none aria-selected:bg-slate-100 aria-selected:text-slate-900 dark:aria-selected:bg-slate-800 dark:aria-selected:text-slate-50",
       className,
     )}
     {...props}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.15.4";
+export const VERSION = "v2.22.0";

web/src/env.mjs

@@ -61,6 +61,7 @@ export const env = createEnv({
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
+    AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
     // EMAIL
     EMAIL_FROM_ADDRESS: z.string().optional(),
     SMTP_CONNECTION_URL: z.string().optional(),
@@ -72,6 +73,9 @@ export const env = createEnv({
     S3_REGION: z.string().optional(),
     // Database exports
     DB_EXPORT_PAGE_SIZE: z.number().optional(),
+    // Worker
+    LANGFUSE_WORKER_HOST: z.string().optional(),
+    LANGFUSE_WORKER_PASSWORD: z.string().optional(),
     // Prompt playground
     OPENAI_API_KEY: z.string().optional(),
     ANTHROPIC_API_KEY: z.string().optional(),
@@ -82,6 +86,8 @@ export const env = createEnv({
    * Specify your client-side environment variables schema here. This way you can ensure the app
    * isn't built with invalid env vars. To expose them to the client, prefix them with
    * `NEXT_PUBLIC_`.
+   * 
+   * WARNING: They do not work when used in Docker builds as NEXT_PUBLIC variables are not runtime but compile-time.
    */
   client: {
     // NEXT_PUBLIC_CLIENTVAR: z.string().min(1),
@@ -137,14 +143,17 @@ export const env = createEnv({
     AUTH_OKTA_CLIENT_ID: process.env.AUTH_OKTA_CLIENT_ID,
     AUTH_OKTA_CLIENT_SECRET: process.env.AUTH_OKTA_CLIENT_SECRET,
     AUTH_OKTA_ISSUER: process.env.AUTH_OKTA_ISSUER,
-    AUTH_OKTA_ALLOW_ACCOUNT_LINKING: process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
+    AUTH_OKTA_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
     AUTH_AUTH0_CLIENT_ID: process.env.AUTH_AUTH0_CLIENT_ID,
     AUTH_AUTH0_CLIENT_SECRET: process.env.AUTH_AUTH0_CLIENT_SECRET,
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
-    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
+    AUTH_DISABLE_SIGNUP: process.env.AUTH_DISABLE_SIGNUP,
     // Email
     EMAIL_FROM_ADDRESS: process.env.EMAIL_FROM_ADDRESS,
     SMTP_CONNECTION_URL: process.env.SMTP_CONNECTION_URL,
@@ -156,6 +165,9 @@ export const env = createEnv({
     S3_REGION: process.env.S3_REGION,
     // Database exports
     DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
+    // Worker
+    LANGFUSE_WORKER_HOST: process.env.LANGFUSE_WORKER_HOST,
+    LANGFUSE_WORKER_PASSWORD: process.env.LANGFUSE_WORKER_PASSWORD,
     // Prompt playground
     OPENAI_API_KEY: process.env.OPENAI_API_KEY,
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,

web/src/features/audit-logs/auditLog.ts

@@ -1,5 +1,5 @@
 import { prisma as _prisma } from "@langfuse/shared/src/db";
-import { type MembershipRole } from "@langfuse/shared/src/db";
+import { type MembershipRole } from "@langfuse/shared";
 
 export type AuditableResource =
   | "membership"
@@ -13,7 +13,9 @@ export type AuditableResource =
   | "model"
   | "prompt"
   | "session"
-  | "apiKey";
+  | "apiKey"
+  | "evalTemplate"
+  | "job";
 
 type AuditLog = {
   resourceType: AuditableResource;

web/src/features/column-visibility/hooks/setSmallPaginationIfColumnsVisible.tsx

@@ -1,7 +1,6 @@
-import { type LangfuseColumnDef } from "@/src/components/table/types";
-import React from "react";
-import { TracesTableRow } from "@/src/components/table/use-cases/traces";
-import { GenerationsTableRow } from "../../../components/table/use-cases/generations";
+import type React from "react";
+import type { TracesTableRow } from "@/src/components/table/use-cases/traces";
+import type { GenerationsTableRow } from "../../../components/table/use-cases/generations";
 
 export type TableColumn =
   | (keyof GenerationsTableRow)[]

web/src/features/dashboard/components/ChartScores.tsx

@@ -5,7 +5,7 @@ import {
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { BaseTimeSeriesChart } from "@/src/features/dashboard/components/BaseTimeSeriesChart";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
   extractTimeSeriesData,
   fillMissingValuesAndTransform,

web/src/features/dashboard/components/LatencyChart.tsx

@@ -3,7 +3,7 @@ import {
   dateTimeAggregationSettings,
   type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import {
   getAllModels,
   extractTimeSeriesData,

web/src/features/dashboard/components/LatencyTables.tsx

@@ -1,7 +1,7 @@
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 
 import { type DatabaseRow } from "@/src/server/api/services/query-builder";

web/src/features/dashboard/components/MetricTable.tsx

@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { RightAlignedCell } from "@/src/features/dashboard/components/RightAlignedCell";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { TotalMetric } from "./TotalMetric";

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -4,7 +4,7 @@ import {
   dateTimeAggregationSettings,
   type DateTimeAggregationOption,
 } from "@/src/features/dashboard/lib/timeseries-aggregation";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 
 import {
   getAllModels,

web/src/features/dashboard/components/ScoresTable.tsx

@@ -2,7 +2,7 @@ import DocPopup from "@/src/components/layouts/doc-popup";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
 import { DashboardTable } from "@/src/features/dashboard/components/cards/DashboardTable";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { RightAlignedCell } from "./RightAlignedCell";

web/src/features/dashboard/components/TracesBarListChart.tsx

@@ -1,5 +1,5 @@
 import { api } from "@/src/utils/api";
-import { type FilterState } from "@/src/features/filters/types";
+import { type FilterState } from "@langfuse/shared";
 import { ExpandListButton } from "@/src/features/dashboard/components/cards/ChevronButton";
 import { useState } from "react";
 import DocPopup from "@/src/components/layouts/doc-popup";