chore: release v2.43.2

---------

Co-authored-by: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com>

.env.dev.example

@@ -14,6 +14,9 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 NEXTAUTH_URL="http://localhost:3000"
 NEXTAUTH_SECRET="secret"
 
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
 # Langfuse experimental features
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
 
@@ -43,4 +46,6 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
+# openssl rand -base64 32 used only here
+ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.prod.example

@@ -76,6 +76,23 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
+# Disable logging to `events` table in db
+# ENABLE_EVENT_LOG=false
+
+
+
+### START Envs to be deprecated in Langfuse v3.0
+
+# Disable the expensive analytics queries and related features
+# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
+
+# Customize the default datetime offset filter used in for the UI tables
+# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
+
+### END Envs to be deprecated in Langfuse v3.0
+
+
+
 
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -12,10 +12,24 @@ body:
   - type: textarea
     attributes:
       label: To reproduce
-      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+      description: Describe how to reproduce your bug. Please provide detailed steps, code snippets, reproduction repos etc.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: SDK and container versions
+      description: If you're experiencing an issue with an integration or SDK, please ensure you're using the latest version. If you're self-hosting Langfuse, check that you're running the most recent version. If updating isn't an option, please provide the specific versions you're currently using.
   - type: textarea
     attributes:
       label: Additional information
       description: Add any other information related to the bug here, screenshots if applicable.
+  - type: dropdown
+    id: contribute
+    attributes:
+      label: Are you interested to contribute a fix for this bug?
+      description: If this is a confirmed bug, the maintainers are happy to support with guidance and review.
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true

.github/workflows/pipeline.yml

@@ -109,8 +109,6 @@ jobs:
     strategy:
       matrix:
         node-version: [20]
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master

CONTRIBUTING.md

@@ -53,53 +53,63 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+
 ```mermaid
 flowchart TB
-   subgraph s4["Clients"]
-      subgraph s2["langfuse/langfuse-python"]
-         Python["Python low-level SDK"]
-         Decorator["observe() decorator"] -->|extends| Python
-         OAI["OpenAI drop-in replacement"] -->|extends| Python
-         Llamaindex["LlamaIndex Integration"] -->|extends| Python
-         LCPYTHON["Langchain Python Integration"] -->|extends| Python
-         Langflow -->|uses| LCPYTHON
-         LiteLLM -->|uses| Python
-      end
-      subgraph s3["langfuse/langfuse-js"]
-         JS["JS SDK"]
-         LCJS["Langchain JS Integration"]  -->|extends| JS
-         Flowise -->|uses| LCJS
-      end
-   end
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
+    end
 
-   DB[Postgres Database]
-	subgraph s1["Application (langfuse/langfuse)"]
-      API[Public HTTP API]
-      G[TRPC API]
-      I[NextAuth]
-      H[React Frontend]
-      Prisma[Prisma ORM]
-      H --> G
-      H --> I
-      G --> I
-      G --- Prisma
-      API --- Prisma
-      I --- Prisma
-	end
-   Prisma --- DB
-   JS --- API
-   Python --- API
+    DB[Postgres Database]
+    Redis[Redis]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        Prisma[Prisma ORM]
+        H --> G
+        H --> I
+        G --> I
+        G --- Prisma
+        API --- Prisma
+        I --- Prisma
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker_API[Public HTTP API]
+    end
+
+    API --> Worker_API
+    Worker_API --- DB
+    Worker_API --- Redis
+
+    Prisma --- DB
+    JS --- API
+    Python --- API
 ```
 
-### Database Overview
+### Network Overview
 
-The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
-
-Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
-
-<img src="./packages/shared/prisma/database.svg">
-
-### Infrastructure & Network Overview
+> [!NOTE]
+> This will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
 
 ```mermaid
 flowchart LR
@@ -114,6 +124,14 @@ flowchart LR
    App --- DB
 ```
 
+### Database Overview
+
+The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
+
+Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
+
+<img src="./packages/shared/prisma/database.svg">
+
 ## Repository Structure
 
 We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https://turbo.build/repo/docs) to manage the dependencies and build process. The monorepo contains the following packages:
@@ -135,7 +153,7 @@ Requirements
 
 **Steps**
 
-1. Fork the the repository and clone it locally
+1. Fork the repository and clone it locally
 2. Run the development database
 
    ```bash
@@ -272,6 +290,16 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Using secrets stored in 1Password
+
+When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
+
+Example:
+
+```bash
+op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
+```
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2023 Finto Technologies GmbH
+Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 

README.md

@@ -26,7 +26,7 @@
    <br/>
    <br/>
    <div>
-      <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
+      <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
       <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse" alt="langfuse npm package"></a>
@@ -37,17 +37,22 @@
 
 ## Overview
 
+_Unmute video for voice-over_
+
+https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+
 ### Develop
 
 - **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
-- **Prompts:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts))
+- **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
+- **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
 - **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
 - **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals))
+  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
   - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 

ee/LICENSE

@@ -0,0 +1,37 @@
+Langfuse Enterprise License (the “Enterprise License”)
+Copyright (c) 2024 Langfuse GmbH ('Langfuse')
+
+With regard to the Langfuse Software:
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Langfuse Terms of Service, available
+at https://langfuse.com/terms (the “Enterprise Terms”), or other
+agreement governing the use of the Software, as agreed by you and Langfuse,
+and otherwise have a valid Langfuse Enterprise license.
+
+Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that Langfuse
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Langfuse Enterprise license.
+
+Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription. You agree that Langfuse and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications. You are not
+granted any other rights beyond what is expressly stated herein. Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the Langfuse Software, those
+components are licensed under the original license provided by the owner of the 
+applicable component.

ee/package.json

@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "zod": "^3.22.4"
   },

fern/apis/client/generators.yml

@@ -15,7 +15,7 @@ groups:
         version: 0.0.26
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-client
+          path: ../../../web/public/generated/api-client
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true

fern/apis/server/definition/api.yml

@@ -1,11 +1,17 @@
 name: langfuse
 docs: |
   ## Authentication
-  Authenticate with the API using Basic Auth, get API keys in the project settings:
+
+  Authenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:
 
   - username: Langfuse Public Key
   - password: Langfuse Secret Key
 
+  ## Exports
+
+  - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/postman/collection.json
+
 error-discrimination:
   strategy: status-code
 auth: basic

fern/apis/server/definition/dataset-items.yml

@@ -32,3 +32,6 @@ types:
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id
+      status:
+        type: optional<commons.DatasetStatus>
+        docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/metrics.yml

@@ -31,10 +31,10 @@ service:
             docs: Optional filter for metrics where traces include all of these tags
           fromTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces on or after a certain timestamp
+            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
           toTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces before a certain timestamp
+            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:

fern/apis/server/definition/observations.yml

@@ -35,7 +35,7 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this timestamp.
+            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -4,18 +4,42 @@ imports:
   pagination: ./utils/pagination.yml
 service:
   auth: true
-  base-path: /api/public
+  base-path: /api/public/v2
   endpoints:
     get:
       docs: Get a prompt
       method: GET
+      path: /prompts/{promptName}
+      path-parameters:
+        promptName:
+          type: string
+          docs: The name of the prompt
+      request:
+        name: GetPromptRequest
+        query-parameters:
+          version:
+            type: optional<integer>
+            docs: Version of the prompt to be retrieved.
+          label:
+            type: optional<string>
+            docs: Label of the prompt to be retrieved. Defaults to "production" if no label or version is set.
+      response: Prompt
+
+    list:
+      docs: Get a list of prompt names with versions and labels
+      method: GET
       path: /prompts
       request:
-        name: GetParameterRequest
+        name: ListPromptsMetaRequest
         query-parameters:
-          name: string
-          version: optional<integer>
-      response: Prompt
+          name: optional<string>
+          label: optional<string>
+          tag: optional<string>
+          page: optional<integer>
+          limit: optional<integer>
+
+      response: PromptMetaListResponse
+
     create:
       docs: Create a prompt
       method: POST
@@ -24,6 +48,18 @@ service:
       response: Prompt
 
 types:
+  PromptMetaListResponse:
+    properties:
+      data: list<PromptMeta>
+      meta: pagination.MetaResponse
+
+  PromptMeta:
+    properties:
+      name: string
+      versions: list<integer>
+      labels: list<string>
+      tags: list<string>
+
   CreatePromptRequest:
     union:
       chat: CreateChatPromptRequest
@@ -32,20 +68,16 @@ types:
   CreateChatPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: list<ChatMessage>
       config: optional<unknown>
+      labels: optional<list<string>>
 
   CreateTextPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: string
       config: optional<unknown>
+      labels: optional<list<string>>
 
   Prompt:
     union:
@@ -57,6 +89,7 @@ types:
       name: string
       version: integer
       config: unknown
+      labels: list<string>
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -29,7 +29,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this timestamp.
+            docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
             type: commons.ScoreSource
             docs: Retrieve only scores from a specific source.

fern/apis/server/definition/trace.yml

@@ -32,7 +32,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this timestamp.
+            docs: Retrieve only traces newer than this datetime (ISO 8601).
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.0.28
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-server
+          path: ../../../web/public/generated/api
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true
@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../generated/postman
+          path: ../../../web/public/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.30.2",
+  "version": "2.43.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -30,8 +30,8 @@
     "@release-it/bumper": "^6.0.1",
     "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.0",
-    "turbo": "^1.13.2"
+    "release-it": "^17.2.1",
+    "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -11,9 +11,9 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.7.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.2",
+    "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -44,14 +44,14 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.28",
-    "@prisma/client": "^5.12.1",
+    "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -71,12 +71,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.0"
+    "vitest": "^1.5.3"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,13 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const MembershipRole = {
+export const ProjectRole = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
     VIEWER: "VIEWER"
 } as const;
-export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -96,7 +96,7 @@ export type AuditLog = {
     updated_at: Generated<Timestamp>;
     user_id: string;
     project_id: string;
-    user_project_role: MembershipRole;
+    user_project_role: ProjectRole;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -199,17 +199,19 @@ export type JobExecution = {
     job_input_trace_id: string | null;
     job_output_score_id: string | null;
 };
-export type Membership = {
-    project_id: string;
-    user_id: string;
-    role: MembershipRole;
+export type LlmApiKeys = {
+    id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    provider: string;
+    display_secret_key: string;
+    secret_key: string;
+    project_id: string;
 };
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: MembershipRole;
+    role: ProjectRole;
     project_id: string;
     sender_id: string | null;
     created_at: Generated<Timestamp>;
@@ -298,6 +300,7 @@ export type PosthogIntegration = {
     posthog_host_name: string;
     last_sync_at: Timestamp | null;
     enabled: boolean;
+    created_at: Generated<Timestamp>;
 };
 export type Pricing = {
     id: string;
@@ -314,6 +317,13 @@ export type Project = {
     name: string;
     cloud_config: unknown | null;
 };
+export type ProjectMembership = {
+    project_id: string;
+    user_id: string;
+    role: ProjectRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Prompt = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -324,13 +334,15 @@ export type Prompt = {
     name: string;
     version: number;
     type: Generated<string>;
-    is_active: boolean;
+    is_active: boolean | null;
     config: Generated<unknown>;
     tags: Generated<string[]>;
+    labels: Generated<string[]>;
 };
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
+    project_id: string;
     name: string;
     value: number;
     source: ScoreSource;
@@ -367,6 +379,8 @@ export type Trace = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type TraceSession = {
     id: string;
@@ -424,13 +438,14 @@ export type DB = {
     events: Events;
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
+    llm_api_keys: LlmApiKeys;
     membership_invitations: MembershipInvitation;
-    memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
     pricings: Pricing;
+    project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
     scores: Score;

packages/shared/prisma/migrations/20240420134232_posthog_integration_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "posthog_integrations" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240423174013_update_models/migration.sql

@@ -0,0 +1,26 @@
+DELETE FROM models WHERE id = 'cluv2t5k3000508ih5kve9zag';
+DELETE FROM models WHERE id = 'clrkvq6iq000008ju6c16gynt';
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- updating tokenizer model to gpt-4-turbo-2024-04-09
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-turbo-2024-04-09" }'),
+  
+  -- update gpt-4-1106-preview naming replacing gpt-4-turbo
+  ('clrkvq6iq000008ju6c16gynt', NULL, 'gpt-4-1106-preview', '(?i)^(gpt-4-1106-preview)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }'),
+  
+  -- apparently azure supports gpt-4-preview
+  ('clv2o2x0p000008jsf9afceau', NULL, ' gpt-4-preview', '(?i)^(gpt-4-preview)$',  NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-turbo-preview" }')
+  

packages/shared/prisma/migrations/20240423192655_add_llm_api_keys/migration.sql

@@ -0,0 +1,24 @@
+-- CreateTable
+CREATE TABLE "llm_api_keys" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "provider" TEXT NOT NULL,
+    "display_secret_key" TEXT NOT NULL,
+    "secret_key" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+
+    CONSTRAINT "llm_api_keys_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_api_keys_id_key" ON "llm_api_keys"("id");
+
+-- CreateIndex
+CREATE INDEX "llm_api_keys_project_id_provider_idx" ON "llm_api_keys"("project_id", "provider");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_api_keys_project_id_provider_key" ON "llm_api_keys"("project_id", "provider");
+
+-- AddForeignKey
+ALTER TABLE "llm_api_keys" ADD CONSTRAINT "llm_api_keys_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240424150909_add_job_execution_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_status_idx" ON "job_executions"("project_id", "status");

packages/shared/prisma/migrations/20240429124411_add_prompt_version_labels/migration.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+-- Step 1: Alter the 'prompts' table to add the 'labels' column
+ALTER TABLE prompts
+ADD COLUMN labels TEXT[] DEFAULT ARRAY[]::TEXT[];
+
+-- Step 2: Update the 'labels' column to include 'production' for active prompts
+UPDATE prompts
+SET labels = array_append(labels, 'production')
+WHERE is_active = TRUE;
+
+-- Step 3: Drop the required constraint on 'is_active' column.
+ALTER TABLE prompts
+ALTER COLUMN is_active DROP NOT NULL;
+
+COMMIT;

packages/shared/prisma/migrations/20240429194411_add_latest_prompt_tag/migration.sql

@@ -0,0 +1,15 @@
+UPDATE
+	prompts
+SET
+	labels = array_append(labels, 'latest')
+WHERE
+	id = (
+		SELECT
+			id
+		FROM
+			prompts AS p2
+		WHERE
+			p2.name = prompts.name
+		ORDER BY
+			created_at DESC
+		LIMIT 1);

packages/shared/prisma/migrations/20240503125742_traces_add_createdat_updatedat/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240503130335_traces_index_created_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_created_at_idx" ON "traces"("created_at");

packages/shared/prisma/migrations/20240503130520_traces_index_updated_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_updated_at_idx" ON "traces"("updated_at");

packages/shared/prisma/migrations/20240508132621_scores_add_project_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "project_id" TEXT;

packages/shared/prisma/migrations/20240508132735_scores_add_projectid_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_project_id_idx" ON "scores"("project_id");

packages/shared/prisma/migrations/20240508132736_scores_backfill_project_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill project_id on existing scores based on linked trace_id
+UPDATE scores
+SET project_id = traces.project_id
+FROM traces 
+WHERE scores.trace_id = traces.id AND scores.project_id IS NULL;

packages/shared/prisma/migrations/20240512151529_rename_memberships_to_project_memberships/migration.sql

@@ -0,0 +1,15 @@
+-- Rename table, will cause a downtime as it needs to be locked and synced with TRPC API
+ALTER TABLE "memberships"
+RENAME TO "project_memberships";
+
+-- AlterTable
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_pkey" TO "project_memberships_pkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_project_id_fkey" TO "project_memberships_project_id_fkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_user_id_fkey" TO "project_memberships_user_id_fkey";
+
+-- RenameIndex
+ALTER INDEX "memberships_user_id_idx" RENAME TO "project_memberships_user_id_idx";

packages/shared/prisma/migrations/20240512155020_rename_enum_membership_role_to_project_role/migration.sql

@@ -0,0 +1,37 @@
+-- Goal: Rename the enum `MembershipRole` to `ProjectRole` and update all tables that use it
+
+-- Create a new Enum
+CREATE TYPE "ProjectRole" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER');
+
+-- Add a temporary column to tables (for each table)
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role_temp" text;
+ALTER TABLE "membership_invitations" ADD COLUMN "role_temp" text;
+ALTER TABLE "project_memberships" ADD COLUMN "role_temp" text;
+
+-- Copy data over to temporary columns (for each table)
+UPDATE "audit_logs" SET "user_project_role_temp" = "user_project_role"::text;
+UPDATE "membership_invitations" SET "role_temp" = "role"::text;
+UPDATE "project_memberships" SET "role_temp" = "role"::text;
+
+-- Drop old columns
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "membership_invitations" DROP COLUMN "role";
+ALTER TABLE "project_memberships" DROP COLUMN "role";
+
+-- Rename temporary columns to old column names
+ALTER TABLE "audit_logs" RENAME COLUMN "user_project_role_temp" TO "user_project_role";
+ALTER TABLE "membership_invitations" RENAME COLUMN "role_temp" TO "role";
+ALTER TABLE "project_memberships" RENAME COLUMN "role_temp" TO "role";
+
+-- Convert the text columns to enum columns (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" TYPE "ProjectRole" USING "user_project_role"::"ProjectRole";
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+ALTER TABLE "project_memberships" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+
+-- Make the columns NOT NULL (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" SET NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" SET NOT NULL;
+ALTER TABLE "project_memberships" ALTER COLUMN "role" SET NOT NULL;
+
+-- Now finally, you can drop your old enum
+DROP TYPE "MembershipRole";

packages/shared/prisma/migrations/20240512155021_add_pricing_gpt4o/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o
+  ('b9854a5c92dc496b997d99d20', NULL, 'gpt-4o', '(?i)^(gpt-4o)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-2024-05-13
+  ('b9854a5c92dc496b997d99d21', NULL, 'gpt-4o-2024-05-13', '(?i)^(gpt-4o-2024-05-13)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o-2024-05-13" }')
+  

packages/shared/prisma/migrations/20240512155021_scores_drop_fk_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_trace_id_fkey";

packages/shared/prisma/migrations/20240512155022_scores_non_null_and_add_fk_project_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Made the column `project_id` on table `scores` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "project_id" SET NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240513082203_scores_unique_id_and_projectid_instead_of_id_and_traceid/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "scores_id_trace_id_key";

packages/shared/prisma/migrations/20240513082204_scores_unique_id_and_projectid_instead_of_id_and_traceid_index/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "scores_id_project_id_key" ON "scores"("id", "project_id");

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views"]
+    previewFeatures = ["tracing", "views", "relationJoins"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id            String                 @id @default(cuid())
-    name          String?
-    email         String?                @unique
-    emailVerified DateTime?              @map("email_verified")
-    password      String?
-    image         String?
-    admin         Boolean                @default(false)
-    accounts      Account[]
-    sessions      Session[]
-    memberships   Membership[]
-    invitations   MembershipInvitation[]
-    createdAt     DateTime               @default(now()) @map("created_at")
-    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags  String[]               @default([]) @map("feature_flags")
-    AuditLog      AuditLog[]
+    id                 String                 @id @default(cuid())
+    name               String?
+    email              String?                @unique
+    emailVerified      DateTime?              @map("email_verified")
+    password           String?
+    image              String?
+    admin              Boolean                @default(false)
+    accounts           Account[]
+    sessions           Session[]
+    projectMemberships ProjectMembership[]
+    invitations        MembershipInvitation[]
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags       String[]               @default([]) @map("feature_flags")
+    AuditLog           AuditLog[]
 
     @@map("users")
 }
@@ -97,7 +97,7 @@ model Project {
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
     cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members            Membership[]
+    projectMembers     ProjectMembership[]
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -111,7 +111,9 @@ model Project {
     EvalTemplate       EvalTemplate[]
     JobConfiguration   JobConfiguration[]
     JobExecution       JobExecution[]
+    LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
+    Score              Score[]
 
     @@map("projects")
 }
@@ -136,37 +138,54 @@ model ApiKey {
     @@map("api_keys")
 }
 
-model Membership {
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String         @map("user_id")
-    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      MembershipRole
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+model LlmApiKeys {
+    id        String   @id @unique @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    provider         String
+    displaySecretKey String @map("display_secret_key")
+    secretKey        String @map("secret_key")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    @@unique([projectId, provider])
+    @@index([projectId, provider])
+    @@map("llm_api_keys")
+}
+
+model ProjectMembership {
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String      @map("user_id")
+    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      ProjectRole
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
-    @@map("memberships")
+    @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String         @id @unique @default(cuid())
+    id        String      @id @unique @default(cuid())
     email     String
-    role      MembershipRole
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?        @map("sender_id")
-    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    role      ProjectRole
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?     @map("sender_id")
+    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum MembershipRole {
+enum ProjectRole {
     OWNER
     ADMIN
     MEMBER
@@ -209,8 +228,9 @@ model Trace {
     output     Json?
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
+    createdAt  DateTime      @default(now()) @map("created_at")
+    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    scores          Score[]
     DatasetRunItems DatasetRunItems[]
     DatasetItem     DatasetItem[]
     JobExecution    JobExecution[]
@@ -223,6 +243,8 @@ model Trace {
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
+    @@index(updatedAt)
+    @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
 }
@@ -254,6 +276,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -283,7 +307,6 @@ model Observation {
     outputCost          Decimal?          @map("output_cost")
     totalCost           Decimal?          @map("total_cost")
     completionStartTime DateTime?         @map("completion_start_time")
-    scores              Score[]
     project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
     datasetRunItems     DatasetRunItems[]
@@ -370,19 +393,20 @@ enum ObservationLevel {
 model Score {
     id            String         @id @default(cuid())
     timestamp     DateTime       @default(now())
+    projectId     String         @map("project_id")
+    project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
     value         Float
     source        ScoreSource
     comment       String?
     traceId       String         @map("trace_id")
-    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
     observationId String?        @map("observation_id")
-    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
     JobExecution  JobExecution[]
 
-    @@unique([id, traceId]) // used for upsert
+    @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
+    @@index([projectId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
@@ -535,9 +559,10 @@ model Prompt {
     name        String
     version     Int
     type        String        @default("text")
-    isActive    Boolean       @map("is_active")
+    isActive    Boolean?      @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
     config      Json          @default("{}")
     tags        String[]      @default([])
+    labels      String[]      @default([])
     Observation Observation[]
 
     @@unique([projectId, name, version])
@@ -575,16 +600,16 @@ model Model {
 }
 
 model AuditLog {
-    id              String         @id @default(cuid())
-    createdAt       DateTime       @default(now()) @map("created_at")
-    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
-    userId          String         @map("user_id")
-    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String         @map("project_id")
-    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole MembershipRole @map("user_project_role")
-    resourceType    String         @map("resource_type")
-    resourceId      String         @map("resource_id")
+    id              String      @id @default(cuid())
+    createdAt       DateTime    @default(now()) @map("created_at")
+    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
+    userId          String      @map("user_id")
+    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String      @map("project_id")
+    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole ProjectRole @map("user_project_role")
+    resourceType    String      @map("resource_type")
+    resourceId      String      @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
@@ -681,6 +706,7 @@ model JobExecution {
     jobOutputScoreId String? @map("job_output_score_id")
     score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
+    @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
     @@map("job_executions")
@@ -708,6 +734,7 @@ model PosthogIntegration {
     posthogHostName        String    @map("posthog_host_name")
     lastSyncAt             DateTime? @map("last_sync_at")
     enabled                Boolean
+    createdAt              DateTime  @default(now()) @map("created_at")
 
     @@index([projectId])
     @@map("posthog_integrations")

packages/shared/prisma/seed.ts

@@ -51,7 +51,7 @@ async function main() {
     create: {
       id: seedProjectId,
       name: "llm-app",
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,
@@ -72,7 +72,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      isActive: true,
+      labels: ["production", "latest"],
       version: 1,
       createdBy: "user-1",
     },
@@ -110,7 +110,7 @@ async function main() {
       create: {
         id: "239ad00f-562f-411d-af14-831c75ddd875",
         name: "demo-app",
-        members: {
+        projectMembers: {
           create: {
             role: "OWNER",
             userId: user.id,
@@ -459,7 +459,7 @@ function createObjects(
     const session =
       Math.random() > 0.3
         ? {
-            id: `session-${i % 10}`,
+            id: `session-${i % 3}`,
             projectId: projectId,
           }
         : undefined;
@@ -501,6 +501,7 @@ function createObjects(
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.REVIEW,
+              projectId,
             },
           ]
         : []),
@@ -512,6 +513,7 @@ function createObjects(
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
               source: ScoreSource.API,
+              projectId,
             },
           ]
         : []),
@@ -590,7 +592,9 @@ function createObjects(
         const model = models[Math.floor(Math.random() * models.length)];
         const promptId =
           promptIds.get(projectId)![
-            Math.floor(Math.random() * promptIds.get(projectId)!.length)
+            Math.floor(
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+            )
           ];
 
         const generation = {
@@ -693,6 +697,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -701,6 +706,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -764,7 +770,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -773,7 +779,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -782,7 +788,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -791,7 +797,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -812,7 +818,7 @@ async function generatePrompts(project: Project) {
         prompt: prompt.prompt,
         name: prompt.name,
         version: prompt.version,
-        isActive: prompt.isActive,
+        labels: prompt.labels,
         tags: prompt.tags,
       },
       update: {
@@ -833,7 +839,6 @@ async function generatePrompts(project: Project) {
         temperature: 0.7,
       },
       version: 1,
-      isActive: false,
     },
     {
       id: `prompt-${v4()}`,
@@ -846,7 +851,7 @@ async function generatePrompts(project: Project) {
         topP: 0.9,
       },
       version: 2,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -860,7 +865,7 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
-      isActive: false,
+      labels: ["production", "latest"],
     },
   ];
 
@@ -881,7 +886,7 @@ async function generatePrompts(project: Project) {
         name: version.name,
         config: version.config,
         version: version.version,
-        isActive: version.isActive,
+        labels: version.labels,
       },
       update: {
         id: version.id,
@@ -910,7 +915,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        isActive: i === 20,
+        labels: i === 20 ? ["production", "latest"] : [],
       },
       update: {
         id: promptId,

packages/shared/src/errors/ApiError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class ApiError extends BaseError {
+  constructor(description = "Api call failed") {
+    super("ApiError", 500, description, true);
+  }
+}

packages/shared/src/errors/BaseError.ts

@@ -7,7 +7,7 @@ export class BaseError extends Error {
     name: string,
     httpCode: number,
     description: string,
-    isOperational: boolean,
+    isOperational: boolean
   ) {
     super(description);
     Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain

packages/shared/src/errors/NotFoundError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class LangfuseNotFoundError extends BaseError {
+  constructor(description = "Not Found") {
+    super("LangfuseNotFoundError", 404, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -1,6 +1,7 @@
 export { BaseError } from "./BaseError";
-export { NotFoundError } from "./NotFoundError";
+export { LangfuseNotFoundError } from "./NotFoundError";
 export { ValidationError } from "./ValidationError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
+export { ApiError } from "./ApiError";

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, observationsTableCols } from "../..";
+import { ModelProvider, UIModelParams } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -34,6 +34,17 @@ export const wipVariableMapping = z.object({
   selectedColumnId: z.string().nullish(),
 });
 
+const observationCols = [
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 'o."metadata"',
+  },
+  { name: "Input", id: "input", internal: 'o."input"' },
+  { name: "Output", id: "output", internal: 'o."output"' },
+];
+
 export const availableEvalVariables = [
   {
     id: "trace",
@@ -52,54 +63,52 @@ export const availableEvalVariables = [
   {
     id: "span",
     display: "Span",
-    availableColumns: [
-      {
-        name: "Metadata",
-        id: "metadata",
-        type: "stringObject",
-        internal: 'o."metadata"',
-      },
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
+    availableColumns: observationCols,
   },
   {
     id: "generation",
     display: "Generation",
-    availableColumns: [
-      {
-        name: "Metadata",
-        id: "metadata",
-        type: "stringObject",
-        internal: 'o."metadata"',
-      },
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
+    availableColumns: observationCols,
+  },
+  {
+    id: "event",
+    display: "Event",
+    availableColumns: observationCols,
   },
-  { id: "event", display: "Event", availableColumns: observationsTableCols },
 ];
 
-export const evalLLMModels = [
+export const evalLLMModels: UIModelParams[] = [
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-3.5-turbo",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-3.5-turbo", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-4-turbo-preview",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4-turbo-preview", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4o", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
 ] as const;
 
-export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+export const EvalModelNames = z.enum([
+  "gpt-3.5-turbo",
+  "gpt-4-turbo-preview",
+  "gpt-4o",
+]);
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/index.ts

@@ -21,3 +21,6 @@ export * from "./features/evals/types";
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
+
+// errors
+export * from "./errors/index";

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -27,6 +27,7 @@ type LLMCompletionParams = {
   modelParams: ModelParams;
   functionCall?: LLMFunctionCall;
   callbacks?: BaseCallbackHandler[];
+  apiKey?: string;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -55,7 +56,8 @@ export async function fetchLLMCompletion(
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
-  const { messages, modelParams, streaming, callbacks } = params;
+  // the apiKey must never be printed to the console
+  const { messages, modelParams, streaming, callbacks, apiKey } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -68,7 +70,16 @@ export async function fetchLLMCompletion(
   let chatModel: ChatOpenAI | ChatAnthropic;
   if (modelParams.provider === ModelProvider.Anthropic) {
     chatModel = new ChatAnthropic({
-      anthropicApiKey: process.env.ANTHROPIC_API_KEY,
+      anthropicApiKey: apiKey,
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+    });
+  } else if (modelParams.provider === ModelProvider.OpenAI) {
+    chatModel = new ChatOpenAI({
+      openAIApiKey: apiKey,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
@@ -76,14 +87,9 @@ export async function fetchLLMCompletion(
       callbacks,
     });
   } else {
-    chatModel = new ChatOpenAI({
-      openAIApiKey: process.env.OPENAI_API_KEY,
-      modelName: modelParams.model,
-      temperature: modelParams.temperature,
-      maxTokens: modelParams.max_tokens,
-      topP: modelParams.top_p,
-      callbacks,
-    });
+    // eslint-disable-next-line no-unused-vars
+    const _exhaustiveCheck: never = modelParams;
+    throw new Error("This model provider is not supported.");
   }
 
   console.log("Making LLM call with params: ", modelParams);

packages/shared/src/server/llm/types.ts

@@ -13,17 +13,22 @@ export enum ModelProvider {
   Anthropic = "anthropic",
   OpenAI = "openai",
 }
+
 export enum ChatMessageRole {
   System = "system",
   User = "user",
   Assistant = "assistant",
 }
 export type ModelParams = AnthropicModelParams | OpenAIModelParams;
-export type UIModelParams = Required<
-  AnthropicModelParams | OpenAIModelParams
-> & {
-  maxTemperature: number;
+
+type RecordWithEnabledFlag<T> = {
+  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
+export type UIModelParams = RecordWithEnabledFlag<
+  Required<AnthropicModelParams | OpenAIModelParams> & {
+    maxTemperature: number;
+  }
+>;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
@@ -41,6 +46,8 @@ export type OpenAIModelParams = {
 } & ModelConfig;
 
 export const openAIModels = [
+  "gpt-4o",
+  "gpt-4o-2024-05-13",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

packages/shared/src/tracesTable.ts

@@ -1,8 +1,13 @@
 import { ObservationLevel } from "@prisma/client";
 import { ColumnDefinition, OptionsDefinition } from ".";
 
-export const tracesTableCols: ColumnDefinition[] = [
-  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
+const tracesOnlyCols: ColumnDefinition[] = [
+  {
+    name: "⭐️",
+    id: "bookmarked",
+    type: "boolean",
+    internal: "t.bookmarked",
+  },
   { name: "ID", id: "id", type: "string", internal: "t.id" },
   {
     name: "Name",
@@ -24,6 +29,41 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "string",
     internal: 't."session_id"',
   },
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 't."metadata"',
+  },
+  {
+    name: "Version",
+    id: "version",
+    type: "string",
+    internal: 't."version"',
+  },
+  {
+    name: "Release",
+    id: "release",
+    type: "string",
+    internal: 't."release"',
+  },
+  {
+    name: "Level",
+    id: "level",
+    type: "stringOptions",
+    internal: '"level"',
+    options: Object.values(ObservationLevel).map((value) => ({ value })),
+  },
+  {
+    name: "Tags",
+    id: "tags",
+    type: "arrayOptions",
+    internal: 't."tags"',
+    options: [], // to be filled in at runtime
+  },
+];
+export const tracesTableCols: ColumnDefinition[] = [
+  ...tracesOnlyCols,
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -48,12 +88,7 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: 'tm."totalTokens"',
   },
-  {
-    name: "Metadata",
-    id: "metadata",
-    type: "stringObject",
-    internal: 't."metadata"',
-  },
+
   {
     name: "Scores",
     id: "scores_avg",
@@ -84,34 +119,10 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: '"calculatedTotalCost"',
   },
-  {
-    name: "Version",
-    id: "version",
-    type: "string",
-    internal: 't."version"',
-  },
-  {
-    name: "Release",
-    id: "release",
-    type: "string",
-    internal: 't."release"',
-  },
-  {
-    name: "Level",
-    id: "level",
-    type: "stringOptions",
-    internal: '"level"',
-    options: Object.values(ObservationLevel).map((value) => ({ value })),
-  },
-  {
-    name: "Tags",
-    id: "tags",
-    type: "arrayOptions",
-    internal: 't."tags"',
-    options: [], // to be filled in at runtime
-  },
 ];
 
+export const evalTableCols: ColumnDefinition[] = tracesOnlyCols;
+
 export type TraceOptions = {
   scores_avg: Array<string>;
   name: Array<OptionsDefinition>;
@@ -119,9 +130,10 @@ export type TraceOptions = {
 };
 
 export function tracesTableColsWithOptions(
-  options?: TraceOptions
+  options?: TraceOptions,
+  cols: ColumnDefinition[] = tracesTableCols
 ): ColumnDefinition[] {
-  return tracesTableCols.map((col) => {
+  return cols.map((col) => {
     if (col.id === "scores_avg") {
       return { ...col, keyOptions: options?.scores_avg ?? [] };
     }

packages/shared/tsconfig.json

@@ -7,7 +7,7 @@
     "outDir": "./dist",
     "types": ["node"],
     "target": "ES2020",
-    "rootDir": ".",
+    "rootDir": "."
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

web/package.json

@@ -1,14 +1,13 @@
 {
   "name": "web",
-  "version": "2.30.2",
+  "version": "2.43.2",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
-    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -23,17 +22,17 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
     "@headlessui/react": "^1.7.19",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.28",
-    "@langfuse/shared": "workspace:*",
     "@langfuse/ee": "workspace:*",
+    "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -56,18 +55,20 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.16",
-    "@react-email/render": "^0.0.12",
+    "@react-email/components": "^0.0.17",
+    "@react-email/render": "^0.0.13",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.110.1",
-    "@sentry/node": "^7.110.1",
-    "@sentry/profiling-node": "^7.110.1",
-    "@sentry/types": "^7.110.1",
+    "@sentry/nextjs": "^7.113.0",
+    "@sentry/node": "^7.113.0",
+    "@sentry/profiling-node": "^7.113.0",
+    "@sentry/types": "^7.113.0",
     "@t3-oss/env-nextjs": "^0.8.0",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.11.1",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
@@ -78,25 +79,26 @@
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
+    "dompurify": "^3.1.2",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.11",
+    "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
@@ -117,9 +119,10 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
     "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^15.0.2",
+    "@testing-library/react": "^15.0.6",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
@@ -133,7 +136,7 @@
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",

web/public/.gitignore

@@ -1 +0,0 @@
-openapi*.yml

web/public/generated/api/openapi.yml

@@ -6,13 +6,23 @@ info:
     ## Authentication
 
 
-    Authenticate with the API using Basic Auth, get API keys in the project
-    settings:
+    Authenticate with the API using [Basic
+    Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API
+    keys in the project settings:
 
 
     - username: Langfuse Public Key
 
     - password: Langfuse Secret Key
+
+
+    ## Exports
+
+
+    - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+
+    - Postman collection:
+    https://cloud.langfuse.com/generated/postman/collection.json
 paths:
   /api/public/dataset-items:
     post:
@@ -486,7 +496,7 @@ paths:
           in: query
           description: >-
             Optional filter to only include traces on or after a certain
-            timestamp
+            datetime (ISO 8601)
           required: false
           schema:
             type: string
@@ -494,7 +504,9 @@ paths:
             nullable: true
         - name: toTimestamp
           in: query
-          description: Optional filter to only include traces before a certain timestamp
+          description: >-
+            Optional filter to only include traces before a certain datetime
+            (ISO 8601)
           required: false
           schema:
             type: string
@@ -638,7 +650,7 @@ paths:
           in: query
           description: >-
             Retrieve only observations with a start_time greater than this
-            timestamp.
+            datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -717,19 +729,100 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
-  /api/public/prompts:
+  /api/public/v2/prompts/{promptName}:
     get:
       description: Get a prompt
       operationId: prompts_get
       tags:
         - Prompts
       parameters:
-        - name: name
-          in: query
+        - name: promptName
+          in: path
+          description: The name of the prompt
           required: true
           schema:
             type: string
         - name: version
+          in: query
+          description: Version of the prompt to be retrieved.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: label
+          in: query
+          description: >-
+            Label of the prompt to be retrieved. Defaults to "production" if no
+            label or version is set.
+          required: false
+          schema:
+            type: string
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Prompt'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/v2/prompts:
+    get:
+      description: Get a list of prompt names with versions and labels
+      operationId: prompts_list
+      tags:
+        - Prompts
+      parameters:
+        - name: name
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: label
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: tag
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
           in: query
           required: false
           schema:
@@ -741,7 +834,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Prompt'
+                $ref: '#/components/schemas/PromptMetaListResponse'
         '400':
           description: ''
           content:
@@ -895,7 +988,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only scores newer than this timestamp.
+          description: Retrieve only scores newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1171,7 +1264,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only traces newer than this timestamp.
+          description: Retrieve only traces newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1757,6 +1850,10 @@ components:
           type: string
           nullable: true
           description: Dataset items are upserted on their id
+        status:
+          $ref: '#/components/schemas/DatasetStatus'
+          nullable: true
+          description: Defaults to ACTIVE for newly created items
       required:
         - datasetName
     CreateDatasetRunItemRequest:
@@ -2479,6 +2576,42 @@ components:
       required:
         - id
         - name
+    PromptMetaListResponse:
+      title: PromptMetaListResponse
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/PromptMeta'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    PromptMeta:
+      title: PromptMeta
+      type: object
+      properties:
+        name:
+          type: string
+        versions:
+          type: array
+          items:
+            type: integer
+        labels:
+          type: array
+          items:
+            type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - name
+        - versions
+        - labels
+        - tags
     CreatePromptRequest:
       title: CreatePromptRequest
       oneOf:
@@ -2510,18 +2643,19 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: array
           items:
             $ref: '#/components/schemas/ChatMessage'
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     CreateTextPromptRequest:
       title: CreateTextPromptRequest
@@ -2529,16 +2663,17 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: string
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     Prompt:
       title: Prompt
@@ -2574,10 +2709,15 @@ components:
         version:
           type: integer
         config: {}
+        labels:
+          type: array
+          items:
+            type: string
       required:
         - name
         - version
         - config
+        - labels
     ChatMessage:
       title: ChatMessage
       type: object

web/public/generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": "## Authentication\n\nAuthenticate with the API using Basic Auth, get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key"
+    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml\n- Postman collection: https://cloud.langfuse.com/generated/postman/collection.json"
   },
   "variable": [
     {
@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\",\n    \"status\": \"ACTIVE\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -418,12 +418,12 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces on or after a certain timestamp"
+                  "description": "Optional filter to only include traces on or after a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "toTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces before a certain timestamp"
+                  "description": "Optional filter to only include traces before a certain datetime (ISO 8601)"
                 }
               ],
               "variable": []
@@ -528,7 +528,7 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this timestamp."
+                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -585,13 +585,58 @@
           "request": {
             "description": "Get a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts?name=&version=",
+              "raw": "{{baseUrl}}/api/public/v2/prompts/:promptName?version=&label=",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
+                "prompts",
+                ":promptName"
+              ],
+              "query": [
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": "Version of the prompt to be retrieved."
+                },
+                {
+                  "key": "label",
+                  "value": "",
+                  "description": "Label of the prompt to be retrieved. Defaults to \"production\" if no label or version is set."
+                }
+              ],
+              "variable": [
+                {
+                  "key": "promptName",
+                  "value": "",
+                  "description": "The name of the prompt"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get a list of prompt names with versions and labels",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/v2/prompts?name=&label=&tag=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "v2",
                 "prompts"
               ],
               "query": [
@@ -601,7 +646,22 @@
                   "description": null
                 },
                 {
-                  "key": "version",
+                  "key": "label",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "tag",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
                   "value": "",
                   "description": null
                 }
@@ -621,13 +681,14 @@
           "request": {
             "description": "Create a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts",
+              "raw": "{{baseUrl}}/api/public/v2/prompts",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "prompts"
               ],
               "query": [],
@@ -638,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -727,7 +788,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this timestamp."
+                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "source",
@@ -935,7 +996,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this timestamp."
+                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "orderBy",

web/scripts/model-match.ts

@@ -62,6 +62,7 @@ export async function modelMatch() {
       where: {
         internalModel: null,
         type: "GENERATION",
+        model: { not: null },
       },
       take: BATCH_SIZE,
       skip: index * BATCH_SIZE,
@@ -235,14 +236,23 @@ export async function modelMatch() {
     }
   }
 
-  await prisma.observation.updateMany({
-    where: {
-      internalModel: "LANGFUSETMPNOMODEL",
-    },
-    data: {
-      internalModel: null,
-    },
-  });
+  let updatedCount;
+  do {
+    console.log(`Updating LANGFUSETMPNOMODEL ${updatedCount}`);
+    const result = await prisma.$queryRaw<[{ id: string }]>`
+    WITH to_update AS (
+      SELECT id 
+      FROM observations 
+      WHERE internal_model = 'LANGFUSETMPNOMODEL'
+      AND "type" = 'GENERATION'
+      LIMIT 50000
+    )
+    UPDATE observations
+    set internal_model = NULL
+    WHERE id IN (SELECT id FROM to_update)
+    RETURNING id;`;
+    updatedCount = result.length;
+  } while (updatedCount > 0);
 
   const end = Date.now();
 

web/sentry.server.config.ts

@@ -15,13 +15,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     tracesSampler: (samplingContext: SamplingContext) => {
       if (
         samplingContext.request &&
-        samplingContext.request.method === "POST" &&
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.9;
+        return 0.3;
       }
-      return 0.15;
+      return 0.1;
     },
 
     profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
@@ -29,6 +28,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),
       new Sentry.Integrations.Prisma({ client: prisma }),
+      Sentry.metrics.metricsAggregatorIntegration(),
     ],
 
     // filter out passwords from the signup request body

web/src/__e2e__/auth.spec.ts

@@ -70,3 +70,38 @@ test("Signup input validation", async ({ page }) => {
 // random email address to be used in tests
 const randomEmailAddress = () =>
   Math.random().toString(36).substring(2, 11) + "@example.com";
+
+test("Unauthenticated user should be redirected to target URL after login", async ({
+  page,
+}) => {
+  await page.goto("/auth/sign-in");
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  // wait 2 seconds
+  await page.waitForTimeout(2000);
+
+  // project id and prompt from seed.ts
+  const promptUrl = "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
+
+  await page.getByRole("button", { name: /Demo User/ }).click();
+
+  await page.getByRole("menuitem", { name: "Sign Out" }).click();
+
+  await expect(page).toHaveURL("/auth/sign-in");
+
+  await page.goto(promptUrl);
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(/targetPath/);
+
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(promptUrl);
+});

web/src/__tests__/datasets.servertest.ts

@@ -210,11 +210,14 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       datasetName: "dataset-name",
       input: { key: "value2" },
       metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
     expect(item2.status).toBe(200);
     expect(item2.body).toMatchObject({
       id: "dataset-item-id",
       input: { key: "value2" },
+      metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
 
     const dbDatasetItem = await prisma.datasetItem.findFirst({
@@ -223,6 +226,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dbDatasetItem).not.toBeNull();
     expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
     expect(dbDatasetItem?.metadata).toMatchObject(["hello-world"]);
+    expect(dbDatasetItem?.status).toBe("ARCHIVED");
   });
 
   it("should create and get a dataset run, include special characters", async () => {

web/src/__tests__/dom.clienttest.ts

@@ -0,0 +1,19 @@
+import * as DOMPurify from "dompurify";
+
+// Create test cases
+describe("should sanitize dom", () => {
+  it("should sanitize urls", () => {
+    const dirty =
+      "http://www.some.site/page.html?default=<script>alert(document.cookie)</script>";
+    const cleam = DOMPurify.sanitize(dirty);
+
+    expect(cleam).toBe("http://www.some.site/page.html?default=");
+  });
+  it("should should not sanitize valid langfuse urls", () => {
+    const validLangfuseUrl =
+      "https://cloud.langfuse.com/project/abjdnksadn/traces?filter=timestamp%3Bdatetime%3B%3B%3E%3B2024-04-19T00%253A00%253A00.000Z%2Cid%3Bstring%3B%3Bcontains%3Ba%2Ctimestamp%3Bdatetime%3B%3B%3C%3B2024-04-30T22%253A00%253A00.000Z%2CinputTokens%3Bnumber%3B%3B%3E%3B1";
+    const cleam = DOMPurify.sanitize(validLangfuseUrl);
+
+    expect(cleam).toBe(validLangfuseUrl);
+  });
+});

web/src/__tests__/ingestion.servertest.ts

@@ -281,6 +281,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expect(dbScore?.name).toBe("score-name");
       expect(dbScore?.value).toBe(100.5);
       expect(dbScore?.observationId).toBeNull();
+      expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
     });
   });
 
@@ -816,57 +817,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace.length).toBe(1);
   });
 
-  it("should fail for auth errors", async () => {
-    const traceId = v4();
-    const scoreId = v4();
-
-    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
-      batch: [
-        {
-          id: v4(),
-          type: "trace-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: traceId,
-            name: "trace-name",
-            userId: "user-1",
-            metadata: { key: "value" },
-            release: "1.0.0",
-            version: "2.0.0",
-          },
-        },
-        {
-          id: v4(),
-          type: "score-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: scoreId,
-            name: "score-name",
-            value: 100.5,
-            traceId: "some-random-id",
-          },
-        },
-      ],
-    });
-
-    expect(responseOne.status).toBe(207);
-
-    expect("errors" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.errors.length).toBe(1);
-    expect("successes" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.successes.length).toBe(1);
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        name: "trace-name",
-      },
-    });
-
-    expect(dbTrace.length).toBe(1);
-  });
-
   it("should fail for resource not found", async () => {
     const traceId = v4();
 
@@ -1653,121 +1603,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
     expect(dbScore?.value).toBe(scoreValue);
   });
 
-  it("should error on Bearer Auth for a trace from different project", async () => {
-    const otherProjectId = "other_project_id";
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    // Seed db with a trace to be scored
-    try {
-      await prisma.project.create({
-        data: {
-          id: otherProjectId,
-          name: "another-project",
-        },
-      });
-
-      await prisma.trace.create({
-        data: {
-          id: traceId,
-          name: "trace-name",
-          project: { connect: { id: otherProjectId } },
-        },
-      });
-
-      const response = await makeAPICall(
-        "POST",
-        "/api/public/ingestion",
-        {
-          batch: [
-            {
-              id: scoreEventId,
-              type: "score-create",
-              timestamp: new Date().toISOString(),
-              body: {
-                id: scoreId,
-                name: scoreName,
-                value: scoreValue,
-                traceId: traceId,
-              },
-            },
-          ],
-        },
-        bearerAuth,
-      );
-
-      expect(response.status).toBe(207);
-      expect(response.body.successes.length).toBe(0);
-      expect(response.body).toHaveProperty("errors");
-      expect(response.body.errors.length).toEqual(1);
-      expect(response.body.errors[0]?.id).toEqual(scoreEventId);
-
-      expect(await prisma.trace.count()).toBe(1);
-      expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
-
-      const dbScore = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-        },
-      });
-
-      expect(dbScore).toBeNull();
-    } finally {
-      await prisma.project.delete({ where: { id: otherProjectId } });
-    }
-  });
-
-  it("should error on Bearer Auth with a trace that does not exist", async () => {
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    const response = await makeAPICall(
-      "POST",
-      "/api/public/ingestion",
-      {
-        batch: [
-          {
-            id: scoreEventId,
-            type: "score-create",
-            timestamp: new Date().toISOString(),
-            body: {
-              id: scoreId,
-              name: scoreName,
-              value: scoreValue,
-              traceId: traceId,
-            },
-          },
-        ],
-      },
-      bearerAuth,
-    );
-
-    expect(response.status).toBe(207);
-    expect(response.body.successes.length).toBe(0);
-    expect(response.body.errors.length).toBe(1);
-    expect(response.body.errors[0]?.id).toBe(scoreEventId);
-
-    expect(await prisma.trace.count()).toBe(0);
-
-    const dbScore = await prisma.score.findUnique({
-      where: {
-        id: scoreId,
-      },
-    });
-
-    expect(dbScore).toBeNull();
-  });
-
   it("should error on wrong input", async () => {
     const traceId = "trace_id";
     const bearerAuth = "Bearer pk-lf-1234567890";

web/src/__tests__/llm-api-key.servertest.ts

@@ -0,0 +1,92 @@
+/** @jest-environment node */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+
+import type { Session } from "next-auth";
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelProvider } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+
+describe("llmApiKey.all RPC", () => {
+  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+  beforeEach(async () => await pruneDatabase());
+
+  const session: Session = {
+    expires: "1",
+    user: {
+      id: "user-1",
+      name: "Demo User",
+      projects: [
+        {
+          id: projectId,
+          role: "ADMIN",
+          name: "test",
+        },
+      ],
+      featureFlags: {
+        templateFlag: true,
+        evals: true,
+      },
+      admin: true,
+    },
+  };
+
+  const ctx = createInnerTRPCContext({ session });
+  const caller = appRouter.createCaller({ ...ctx, prisma });
+
+  it("should create an llm api key", async () => {
+    const secret = "test-secret";
+
+    await caller.llmApiKey.create({
+      projectId,
+      secretKey: secret,
+      provider: ModelProvider.OpenAI,
+    });
+
+    const llmApiKeys = await prisma.llmApiKeys.findMany({
+      where: {
+        projectId,
+      },
+    });
+
+    expect(llmApiKeys.length).toBe(1);
+    expect(llmApiKeys[0].projectId).toBe(projectId);
+    expect(llmApiKeys[0].secretKey).not.toBeNull();
+    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
+    expect(llmApiKeys[0].provider).toBe(ModelProvider.OpenAI);
+    // this has to be 3 dots and the last 4 characters of the secret
+    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
+  });
+
+  it("should create and get an llm api key", async () => {
+    const secret = "test-secret";
+
+    await caller.llmApiKey.create({
+      projectId,
+      secretKey: secret,
+      provider: ModelProvider.OpenAI,
+    });
+
+    const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
+      where: {
+        projectId,
+      },
+    });
+
+    expect(dbLlmApiKeys.length).toBe(1);
+
+    const llmApiKeys = await caller.llmApiKey.all({
+      projectId,
+    });
+
+    expect(llmApiKeys.data.length).toBe(1);
+    expect(llmApiKeys.data[0].provider).toBe(ModelProvider.OpenAI);
+    // this has to be 3 dots and the last 4 characters of the secret
+    expect(llmApiKeys.data[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
+
+    // response must not contain the secret key itself
+    expect(llmApiKeys.data[0]).not.toHaveProperty("secretKey");
+  });
+});

web/src/__tests__/model-match.servertest.ts

@@ -73,6 +73,15 @@ describe("model match", () => {
           input: "I am a prompt",
           output: "I am a completion",
         },
+        {
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "gpt-4o",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
       ],
     });
 
@@ -86,7 +95,7 @@ describe("model match", () => {
     expect(observation1?.completionTokens).toEqual(3000);
     expect(observation1?.totalTokens).toEqual(0);
 
-    expect(observations.length).toEqual(3);
+    expect(observations.length).toEqual(4);
     observations.forEach((observation) => {
       expect(observation.internalModel).toBeDefined();
       expect(observation.promptTokens).toBeGreaterThan(0);

web/src/__tests__/prompts.v1.servertest.ts

@@ -5,10 +5,10 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4, v4 } from "uuid";
 import { type Prompt } from "@langfuse/shared";
 import {
-  PromptSchema,
+  LegacyPromptSchema,
   PromptType,
-  type ValidatedPrompt,
-} from "@/src/features/prompts/server/validation";
+  type LegacyValidatedPrompt,
+} from "@/src/features/prompts/server/utils/validation";
 
 describe("/api/public/prompts API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -21,7 +21,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         config: {
           temperature: 0.1,
@@ -51,6 +51,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production"]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
     expect(fetchedObservations.body.tags).toEqual([]);
@@ -64,7 +65,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt + name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         config: {
           temperature: 0.1,
@@ -94,6 +95,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production"]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
     expect(fetchedObservations.body.tags).toEqual([]);
@@ -107,7 +109,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: false,
+        labels: [],
         version: 1,
         config: {
           temperature: 0.1,
@@ -137,7 +139,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt-one",
-        isActive: false,
+        labels: [],
         version: 1,
         config: {
           temperature: 0.1,
@@ -154,7 +156,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptTwoId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 2,
         config: {
           temperature: 0.2,
@@ -183,68 +185,192 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.prompt).toBe("prompt-one");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(false);
+    expect(fetchedObservations.body.labels).toEqual([]);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
 
   it("should fetch active prompt when multiple exist", async () => {
-    const promptIdOne = uuidv4();
-    const promptIdTwo = uuidv4();
-
-    await prisma.prompt.create({
-      data: {
-        id: promptIdOne,
-        name: "prompt-name",
-        prompt: "prompt",
-        isActive: false,
-        version: 1,
-        config: {
-          temperature: 0.1,
-        },
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
+    // First prompt is activated
+    const prompt1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt1",
+      isActive: true,
+      version: 1,
+      config: {
+        temperature: 0.1,
       },
+      createdBy: "user-1",
     });
 
-    await prisma.prompt.create({
-      data: {
-        id: promptIdTwo,
-        name: "prompt-name",
-        prompt: "prompt",
-        isActive: true,
-        version: 2,
-        config: {
-          temperature: 0.2,
-        },
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
+    // Second prompt also activated
+    const prompt2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt2",
+      labels: ["production"],
+      isActive: true,
+      version: 2,
+      config: {
+        temperature: 0.2,
       },
+      createdBy: "user-1",
     });
 
-    const fetchedObservations = await makeAPICall(
+    // Third prompt is deactivated
+    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt3",
+      labels: [], // This should be ignored
+      isActive: false,
+      version: 3,
+      config: {
+        temperature: 0.3,
+      },
+      createdBy: "user-1",
+    });
+
+    // Expect the second prompt to be fetched
+    const fetchedProductionPrompt = await makeAPICall(
       "GET",
       "/api/public/prompts?name=prompt-name",
       undefined,
     );
 
-    expect(fetchedObservations.status).toBe(200);
+    expect(fetchedProductionPrompt.status).toBe(200);
 
-    if (!isPrompt(fetchedObservations.body)) {
+    if (!isPrompt(fetchedProductionPrompt.body)) {
       throw new Error("Expected body to be a prompt");
     }
 
-    expect(fetchedObservations.body.id).toBe(promptIdTwo);
-    expect(fetchedObservations.body.name).toBe("prompt-name");
-    expect(fetchedObservations.body.prompt).toBe("prompt");
-    expect(fetchedObservations.body.type).toBe("text");
-    expect(fetchedObservations.body.version).toBe(2);
-    expect(fetchedObservations.body.isActive).toBe(true);
-    expect(fetchedObservations.body.createdBy).toBe("user-1");
-    expect(fetchedObservations.body.config).toEqual({ temperature: 0.2 });
+    // @ts-expect-error
+    expect(fetchedProductionPrompt.body.id).toBe(prompt2.body.id);
+    expect(fetchedProductionPrompt.body.name).toBe("prompt-name");
+    expect(fetchedProductionPrompt.body.prompt).toBe("prompt2");
+    expect(fetchedProductionPrompt.body.type).toBe("text");
+    expect(fetchedProductionPrompt.body.version).toBe(2);
+    expect(fetchedProductionPrompt.body.isActive).toBe(true);
+    expect(fetchedProductionPrompt.body.labels).toEqual(["production"]);
+    expect(fetchedProductionPrompt.body.createdBy).toBe("API");
+    expect(fetchedProductionPrompt.body.config).toEqual({ temperature: 0.2 });
+
+    // Expect the first prompt to be deactivated
+    const fetchedOldProductionPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedOldProductionPrompt.status).toBe(200);
+
+    if (!isPrompt(fetchedOldProductionPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedOldProductionPrompt.body.id).toBe(prompt1.body.id);
+    expect(fetchedOldProductionPrompt.body.name).toBe("prompt-name");
+    expect(fetchedOldProductionPrompt.body.prompt).toBe("prompt1");
+    expect(fetchedOldProductionPrompt.body.type).toBe("text");
+    expect(fetchedOldProductionPrompt.body.version).toBe(1);
+    expect(fetchedOldProductionPrompt.body.isActive).toBe(false);
+    expect(fetchedOldProductionPrompt.body.labels).toEqual([]);
+    expect(fetchedOldProductionPrompt.body.createdBy).toBe("API");
+    expect(fetchedOldProductionPrompt.body.config).toEqual({
+      temperature: 0.1,
+    });
+  });
+
+  it("should correctly handle overwriting labels", async () => {
+    // First prompt has multiple labels
+    const prompt1 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt1",
+      labels: ["production", "staging", "development"],
+      isActive: true,
+      version: 1,
+      config: {
+        temperature: 0.1,
+      },
+      createdBy: "user-1",
+    });
+
+    // Second prompt overwrites production and staging label
+    const prompt2 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt2",
+      labels: ["production", "production", "staging"], // Should be deduped
+      isActive: true,
+      version: 2,
+      config: {
+        temperature: 0.2,
+      },
+      createdBy: "user-1",
+    });
+
+    // Third prompt overwrites staging label
+    const prompt3 = await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      prompt: "prompt3",
+      labels: ["staging"],
+      isActive: false,
+      version: 3,
+      config: {
+        temperature: 0.3,
+      },
+      createdBy: "user-1",
+    });
+
+    // Expect the second prompt to be fetched as default production prompt
+    const fetchedProductionPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name",
+      undefined,
+    );
+    expect(fetchedProductionPrompt.status).toBe(200);
+    if (!isPrompt(fetchedProductionPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+    // @ts-expect-error
+    expect(fetchedProductionPrompt.body.id).toBe(prompt2.body.id);
+    expect(fetchedProductionPrompt.body.labels).toEqual(["production"]); // Only production label should be present
+
+    // Expect the first prompt to have only development label
+    const fetchedFirstPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedFirstPrompt.status).toBe(200);
+    if (!isPrompt(fetchedFirstPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedFirstPrompt.body.id).toBe(prompt1.body.id);
+    expect(fetchedFirstPrompt.body.labels).toEqual(["development"]);
+
+    // Expect the third prompt to have only staging label
+    const fetchedThirdPrompt = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=3",
+      undefined,
+    );
+
+    expect(fetchedThirdPrompt.status).toBe(200);
+    if (!isPrompt(fetchedThirdPrompt.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    // @ts-expect-error
+    expect(fetchedThirdPrompt.body.id).toBe(prompt3.body.id);
+    expect(fetchedThirdPrompt.body.labels).toEqual(["staging", "latest"]);
   });
 
   it("should create and fetch a prompt", async () => {
@@ -275,6 +401,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production", "latest"]);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
@@ -290,7 +417,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
@@ -353,7 +480,7 @@ describe("/api/public/prompts API Endpoint", () => {
         id: promptId,
         name: "prompt-name",
         prompt: "prompt",
-        isActive: true,
+        labels: ["production"],
         version: 1,
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
@@ -428,6 +555,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.type).toBe("text");
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.labels).toEqual(["production", "latest"]);
     expect(fetchedObservations.body.createdBy).toBe("API");
     expect(fetchedObservations.body.config).toEqual({});
   });
@@ -461,6 +589,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(validatedPrompt.type).toBe("chat");
     expect(validatedPrompt.version).toBe(1);
     expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.labels).toEqual(["production", "latest"]);
     expect(validatedPrompt.createdBy).toBe("API");
     expect(validatedPrompt.config).toEqual({});
   });
@@ -484,7 +613,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -512,7 +641,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -535,7 +664,7 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(status).toBe(404);
     expect(body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
@@ -589,6 +718,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(validatedPrompt.type).toBe("chat");
     expect(validatedPrompt.version).toBe(1);
     expect(validatedPrompt.isActive).toBe(true);
+    expect(validatedPrompt.labels).toEqual(["production", "latest"]);
     expect(validatedPrompt.createdBy).toBe("API");
     expect(validatedPrompt.config).toEqual({});
 
@@ -600,15 +730,17 @@ describe("/api/public/prompts API Endpoint", () => {
     );
     expect(getResponse2.status).toBe(404);
     expect(getResponse2.body).toEqual({
-      error: "NotFoundError",
+      error: "LangfuseNotFoundError",
       message: "Prompt not found",
     });
   });
 });
 
-const isPrompt = (x: unknown): x is Prompt => {
+type PromptWithIsActive = Prompt & { isActive: boolean };
+
+const isPrompt = (x: unknown): x is PromptWithIsActive => {
   if (typeof x !== "object" || x === null) return false;
-  const prompt = x as Prompt;
+  const prompt = x as PromptWithIsActive;
   return (
     typeof prompt.id === "string" &&
     typeof prompt.name === "string" &&
@@ -622,7 +754,9 @@ const isPrompt = (x: unknown): x is Prompt => {
   );
 };
 
-const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
+const validatePrompt = (
+  obj: Record<string, unknown>,
+): LegacyValidatedPrompt => {
   Object.keys(obj).forEach((key) => {
     obj[key] =
       key === "createdAt" || key === "updatedAt"
@@ -630,5 +764,5 @@ const validatePrompt = (obj: Record<string, unknown>): ValidatedPrompt => {
         : obj[key];
   });
 
-  return PromptSchema.parse(obj);
+  return LegacyPromptSchema.parse(obj);
 };

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id  WHERE  t."project_id" = ',
+          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
           ";",
         ],
       } as const,

web/src/__tests__/scores.servertest.ts

@@ -60,6 +60,7 @@ describe("/api/public/scores API Endpoint", () => {
     expect(dbScore?.observationId).toBeNull();
     expect(dbScore?.comment).toBe("comment");
     expect(dbScore?.source).toBe("API");
+    expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
   });
 
   it("should create score for a trace with int", async () => {

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -23,6 +23,7 @@ describe("Traces TRPC Router", () => {
       ],
       featureFlags: {
         templateFlag: true,
+        evals: true,
       },
       admin: true,
     },

web/src/__tests__/test-utils.ts

@@ -12,6 +12,7 @@ export const pruneDatabase = async () => {
   await prisma.prompt.deleteMany();
   await prisma.events.deleteMany();
   await prisma.model.deleteMany();
+  await prisma.llmApiKeys.deleteMany();
 };
 
 export function createBasicAuthHeader(
@@ -89,7 +90,7 @@ export const setupUserAndProject = async () => {
           },
         ],
       },
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,

web/src/__tests__/usage.servertest.ts

@@ -33,6 +33,7 @@ describe("Token Count Functions", () => {
       { model: "claude", tokenizer: "claude", tokens: 118 },
       { model: "claude-instant-1.2", tokenizer: "claude", tokens: 118 },
       { model: "gpt-3.5-turbo-1106", tokenizer: "openai", tokens: 114 },
+      { model: "gpt-4o-2024-05-13", tokenizer: "openai", tokens: 112 },
     ].forEach(({ model, tokens, tokenizer }) => {
       it(`should return token count ${tokens} for ${model}`, () => {
         const result = tokenCount({

web/src/components/ChatMessages/ChatMessageComponent.tsx

@@ -5,7 +5,7 @@ import { ChatMessageRole, type ChatMessageWithId } from "@langfuse/shared";
 import { Button } from "@/src/components/ui/button";
 import { Card, CardContent } from "@/src/components/ui/card";
 import { Textarea } from "@/src/components/ui/textarea";
-import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
+import type { MessagesContext } from "./types";
 
 type ChatMessageProps = Pick<
   MessagesContext,

web/src/components/ChatMessages/index.tsx

@@ -1,10 +1,12 @@
 import { PlusCircleIcon } from "lucide-react";
+import { useEffect, useRef } from "react";
 
 import { Button } from "@/src/components/ui/button";
-import { ChatMessageComponent } from "@/src/features/playground/client/components/ChatMessageComponent";
-import type { MessagesContext } from "@/src/features/playground/client/components/Messages";
 import { ChatMessageRole } from "@langfuse/shared";
-import { useRef, useEffect } from "react";
+
+import { ChatMessageComponent } from "./ChatMessageComponent";
+
+import type { MessagesContext } from "./types";
 
 type ChatMessagesProps = MessagesContext;
 export const ChatMessages: React.FC<ChatMessagesProps> = (props) => {

web/src/components/ChatMessages/types.ts

@@ -0,0 +1,13 @@
+import type { ChatMessageRole, ChatMessageWithId } from "@langfuse/shared";
+
+
+export type MessagesContext = {
+  messages: ChatMessageWithId[];
+  addMessage: (role: ChatMessageRole, content?: string) => ChatMessageWithId;
+  deleteMessage: (id: string) => void;
+  updateMessage: <Key extends keyof ChatMessageWithId>(
+    id: string,
+    key: Key,
+    value: ChatMessageWithId[Key]
+  ) => void;
+};

web/src/components/LangfuseLogo.tsx

@@ -39,7 +39,7 @@ export const LangfuseLogo = ({
     {showEnvLabel && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && (
       <div
         className={cn(
-          "flex items-center gap-2 self-stretch rounded-md px-3 py-2 text-xs ring-1 lg:-mx-2",
+          "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
           env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
             ? "bg-blue-100 text-blue-500 ring-blue-500"
             : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
@@ -48,9 +48,9 @@ export const LangfuseLogo = ({
         )}
       >
         {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
-          <Check size={16} />
+          <Check size={16} className="hidden sm:block" />
         ) : (
-          <AlertTriangle size={16} />
+          <AlertTriangle size={16} className="hidden sm:block" />
         )}
         <span className="whitespace-nowrap">
           {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)

web/src/components/ModelParameters.tsx

@@ -1,8 +1,3 @@
-import {
-  supportedModels,
-  ModelProvider,
-  type UIModelParams,
-} from "@langfuse/shared";
 import { Input } from "@/src/components/ui/input";
 import {
   Select,
@@ -12,23 +7,31 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
+import { Switch } from "@/src/components/ui/switch";
+import { cn } from "@/src/utils/tailwind";
+import {
+  ModelProvider,
+  supportedModels,
+  type UIModelParams,
+} from "@langfuse/shared";
 
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
-  updateModelParam: <Key extends keyof UIModelParams>(
+  updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
-    value: UIModelParams[Key],
+    value: UIModelParams[Key]["value"],
   ) => void;
-  updateModelParams: <UIModelParams>(params: UIModelParams) => void;
-  disabled?: boolean;
+  setModelParamEnabled?: (key: keyof UIModelParams, enabled: boolean) => void;
+  formDisabled?: boolean;
 };
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
   availableModels,
-  updateModelParam,
-  disabled,
+  updateModelParamValue,
+  setModelParamEnabled,
+  formDisabled = false,
 }) => {
   return (
     <div className="flex flex-col space-y-4">
@@ -37,61 +40,69 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
-          disabled={disabled}
-          value={modelParams.provider}
+          disabled={formDisabled}
+          value={modelParams.provider.value}
           options={
             availableModels
-              ? [...new Set(availableModels.map((m) => m.provider))]
+              ? [...new Set(availableModels.map((m) => m.provider.value))]
               : Object.values(ModelProvider)
           }
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSelect
           title="Model name"
           modelParamsKey="model"
-          disabled={disabled}
-          value={modelParams.model}
+          disabled={formDisabled}
+          value={modelParams.model.value}
           options={Object.values(
             availableModels
               ? availableModels
-                  .filter((m) => m.provider === modelParams.provider)
-                  .map((m) => m.model)
-              : supportedModels[modelParams.provider],
+                  .filter(
+                    (m) => m.provider.value === modelParams.provider.value,
+                  )
+                  .map((m) => m.model.value)
+              : supportedModels[modelParams.provider.value],
           )}
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Temperature"
           modelParamsKey="temperature"
-          disabled={disabled}
-          value={modelParams.temperature}
+          formDisabled={formDisabled}
+          enabled={modelParams.temperature.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.temperature.value}
           min={0}
-          max={modelParams.maxTemperature}
+          max={modelParams.maxTemperature.value}
           step={0.01}
-          tooltip="The sampling temperature. Higher values will make the output more random, while lower values like will make it more focused and deterministic."
-          updateModelParam={updateModelParam}
+          tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Output token limit"
           modelParamsKey="max_tokens"
-          disabled={disabled}
-          value={modelParams.max_tokens}
+          formDisabled={formDisabled}
+          enabled={modelParams.max_tokens.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.max_tokens.value}
           min={1}
           max={4096}
           step={1}
           tooltip="The maximum number of tokens that can be generated in the chat completion."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Top P"
           modelParamsKey="top_p"
-          disabled={disabled}
-          value={modelParams.top_p}
+          formDisabled={formDisabled}
+          enabled={modelParams.top_p.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.top_p.value}
           min={0}
           max={1}
           step={0.01}
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
       </div>
     </div>
@@ -103,7 +114,7 @@ type ModelParamsSelectProps = {
   modelParamsKey: keyof UIModelParams;
   value: string;
   options: string[];
-  updateModelParam: ModelParamsContext["updateModelParam"];
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
   disabled?: boolean;
 };
 const ModelParamsSelect = ({
@@ -116,7 +127,9 @@ const ModelParamsSelect = ({
 }: ModelParamsSelectProps) => {
   return (
     <div className="space-y-2">
-      <p className="text-xs font-semibold">{title}</p>
+      <p className={cn("text-xs font-semibold", disabled && "text-gray-400")}>
+        {title}
+      </p>
       <Select
         disabled={disabled}
         onValueChange={(value) =>
@@ -150,8 +163,10 @@ type ModelParamsSliderProps = {
   min: number;
   max: number;
   step: number;
-  updateModelParam: ModelParamsContext["updateModelParam"];
-  disabled?: boolean;
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
+  enabled?: boolean;
+  formDisabled?: boolean;
+  setModelParamEnabled?: ModelParamsContext["setModelParamEnabled"];
 };
 const ModelParamsSlider = ({
   title,
@@ -162,30 +177,51 @@ const ModelParamsSlider = ({
   max,
   step,
   updateModelParam,
-  disabled,
+  setModelParamEnabled,
+  enabled,
+  formDisabled,
 }: ModelParamsSliderProps) => {
   return (
     <div className="space-y-3" title={tooltip}>
       <div className="flex flex-row">
-        <p className="flex-1 text-xs font-semibold">{title}</p>
-        <Input
-          className="h-6 w-14 appearance-none px-2 text-right"
-          type="number"
-          disabled={disabled}
-          min={min}
-          max={max}
-          step={step}
-          value={value}
-          onChange={(event) => {
-            updateModelParam(
-              modelParamsKey,
-              Math.max(Math.min(parseFloat(event.target.value), max), min),
-            );
-          }}
-        />
+        <p
+          className={cn(
+            "flex-1 text-xs font-semibold",
+            (!enabled || formDisabled) && "text-gray-400",
+          )}
+        >
+          {title}
+        </p>
+        <div className="flex flex-row space-x-3">
+          <Input
+            className="h-6 w-14 appearance-none px-2 text-right"
+            type="number"
+            disabled={!enabled || formDisabled}
+            min={min}
+            max={max}
+            step={step}
+            value={value}
+            onChange={(event) => {
+              updateModelParam(
+                modelParamsKey,
+                Math.max(Math.min(parseFloat(event.target.value), max), min),
+              );
+            }}
+          />
+          {setModelParamEnabled ? (
+            <Switch
+              title={`Control sending the ${title} parameter`}
+              disabled={formDisabled}
+              checked={enabled}
+              onCheckedChange={(checked) => {
+                setModelParamEnabled(modelParamsKey, checked);
+              }}
+            />
+          ) : null}
+        </div>
       </div>
       <Slider
-        disabled={disabled}
+        disabled={!enabled || formDisabled}
         min={min}
         max={max}
         step={step}

web/src/components/deleteButton.tsx

@@ -7,10 +7,10 @@ import {
 } from "@/src/components/ui/popover";
 import { Button } from "@/src/components/ui/button";
 import { TrashIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type Scope } from "@/src/features/rbac/constants/roleAccessRights";
 import { api } from "@/src/utils/api";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DeleteButtonProps {
   itemId: string;
@@ -33,7 +33,7 @@ export function DeleteButton({
 }: DeleteButtonProps) {
   const [isDeleted, setIsDeleted] = useState(false);
   const router = useRouter();
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasAccess = useHasAccess({ projectId, scope: scope });
   const traceMutation = api.traces.deleteMany.useMutation({
@@ -63,6 +63,15 @@ export function DeleteButton({
         <Button
           variant={isTableAction ? "ghost" : "outline"}
           size={isTableAction ? "xs" : "icon"}
+          onClick={() =>
+            type === "trace"
+              ? capture("trace:delete_form_open", {
+                  source: isTableAction ? "table-single-row" : "trace detail",
+                })
+              : capture("datasets:delete_form_open", {
+                  source: "dataset",
+                })
+          }
         >
           <TrashIcon className="h-4 w-4" />
         </Button>
@@ -84,7 +93,7 @@ export function DeleteButton({
                   traceIds: [itemId],
                   projectId,
                 });
-                posthog.capture("trace:delete", {
+                capture("trace:delete", {
                   source: isTableAction ? "table-single-row" : "trace",
                 });
               }}
@@ -101,7 +110,7 @@ export function DeleteButton({
                   projectId,
                   datasetId: itemId,
                 });
-                posthog.capture("dataset:delete", {
+                capture("datasets:delete_dataset_button_click", {
                   source: isTableAction ? "table-single-row" : "dataset",
                 });
               }}

web/src/components/error-page.tsx

@@ -14,6 +14,7 @@ export const ErrorPage = ({
 }) => {
   const session = useSession();
   const router = useRouter();
+  const newTargetPath = router.asPath;
 
   return (
     <div className="flex h-screen flex-col items-center justify-center">
@@ -21,7 +22,15 @@ export const ErrorPage = ({
       <h1 className="mb-4 text-xl font-bold">{title}</h1>
       <p className="mb-8 text-center">{message}</p>
       {session.status === "unauthenticated" ? (
-        <Button onClick={() => void router.push("/")}>Sign In</Button>
+        <Button
+          onClick={() =>
+            void router.push(
+              `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+            )
+          }
+        >
+          Sign In
+        </Button>
       ) : null}
     </div>
   );

web/src/components/json-editor.tsx

@@ -1,6 +1,7 @@
 import CodeMirror, { EditorView } from "@uiw/react-codemirror";
 import { githubLight } from "@uiw/codemirror-theme-github";
 import { json } from "@codemirror/lang-json";
+import { cn } from "@/src/utils/tailwind";
 
 // todo: add json linting
 
@@ -9,11 +10,13 @@ export function JsonEditor({
   onChange,
   editable = true,
   lineWrapping = true,
+  className,
 }: {
   defaultValue: string;
   onChange?: (value: string) => void;
   editable?: boolean;
   lineWrapping?: boolean;
+  className?: string;
 }) {
   return (
     <CodeMirror
@@ -26,7 +29,7 @@ export function JsonEditor({
       extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
       defaultValue={defaultValue}
       onChange={onChange}
-      className="overflow-hidden rounded-md border"
+      className={cn("overflow-hidden rounded-md border", className)}
       editable={editable}
     />
   );

web/src/components/layouts/doc-popup.tsx

@@ -3,9 +3,9 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
-import { usePostHog } from "posthog-js/react";
 
 export type DocPopupProps = {
   description: React.ReactNode;
@@ -26,14 +26,17 @@ export default function DocPopup({
     md: "w-6 h-6",
     lg: "w-8 h-8",
   };
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   return (
     <HoverCard
       openDelay={200}
       onOpenChange={(open) => {
         if (open) {
-          posthog.capture("help-popup:opened", { href, description });
+          capture("help_popup:opened", {
+            hfref: href,
+            description: description,
+          });
         }
       }}
     >
@@ -44,6 +47,12 @@ export default function DocPopup({
             rel="noopener"
             target="_blank"
             className="inline-block whitespace-nowrap text-gray-500 sm:pl-0"
+            onClick={() => {
+              capture("help_popup:href_clicked", {
+                href: href,
+                description: description,
+              });
+            }}
           >
             {
               {

web/src/components/layouts/full-screen-page.tsx

@@ -0,0 +1,9 @@
+export const FullScreenPage: React.FC<React.PropsWithChildren<{}>> = ({
+  children,
+}) => {
+  return (
+    <div className="flex h-[calc(100dvh-6rem)] flex-col overflow-hidden lg:h-[calc(100dvh-2rem)]">
+      {children}
+    </div>
+  );
+};

web/src/components/layouts/layout.tsx

@@ -14,7 +14,6 @@ import {
   AvatarFallback,
   AvatarImage,
 } from "@/src/components/ui/avatar";
-import { api } from "@/src/utils/api";
 import { NewProjectButton } from "@/src/features/projects/components/NewProjectButton";
 import { FeedbackButtonWrapper } from "@/src/features/feedback/component/FeedbackButton";
 import { Button } from "@/src/components/ui/button";
@@ -31,6 +30,7 @@ import {
 import { ChevronDownIcon } from "@heroicons/react/20/solid";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { ProjectNavigation } from "@/src/components/projectNavigation";
+import DOMPurify from "dompurify";
 
 const userNavigation = [
   {
@@ -61,7 +61,7 @@ export default function Layout(props: PropsWithChildren) {
   useCheckNotification(NOTIFICATIONS, session.status === "authenticated");
 
   const enableExperimentalFeatures =
-    api.environment.enableExperimentalFeatures.useQuery().data ?? false;
+    session.data?.environment.enableExperimentalFeatures ?? false;
 
   const projectId = router.query.projectId as string | undefined;
 
@@ -147,7 +147,14 @@ export default function Layout(props: PropsWithChildren) {
     !publishablePaths.includes(router.pathname) &&
     !router.pathname.startsWith("/public/")
   ) {
-    void router.replace("/auth/sign-in");
+    const newTargetPath = router.asPath;
+    if (newTargetPath && newTargetPath !== "/") {
+      void router.replace(
+        `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+      );
+    } else {
+      void router.replace(`/auth/sign-in`);
+    }
     return <Spinner message="Redirecting" />;
   }
 
@@ -155,13 +162,18 @@ export default function Layout(props: PropsWithChildren) {
     session.status === "authenticated" &&
     unauthenticatedPaths.includes(router.pathname)
   ) {
-    void router.replace("/");
+    const targetPath = router.query.targetPath as string | undefined;
+
+    const sanitizedTargetPath = targetPath
+      ? DOMPurify.sanitize(targetPath)
+      : undefined;
+
+    void router.replace(sanitizedTargetPath ?? "/");
     return <Spinner message="Redirecting" />;
   }
 
   const hideNavigation =
     session.status === "unauthenticated" ||
-    projects.length === 0 ||
     pathsWithoutNavigation.includes(router.pathname) ||
     router.pathname.startsWith("/public/");
   if (hideNavigation)

web/src/components/layouts/routes.ts

@@ -63,19 +63,25 @@ export const ROUTES: Route[] = [
     name: "Evaluation",
     icon: Lightbulb,
     cloudOnly: true,
-    featureFlag: "evals",
+    label: "Beta",
     children: [
       {
         name: "Templates",
         pathname: `/project/[projectId]/evals/templates`,
         cloudOnly: true,
-        featureFlag: "evals",
+        rbacScope: "evalTemplate:read",
       },
       {
         name: "Configs",
         pathname: `/project/[projectId]/evals/configs`,
         cloudOnly: true,
-        featureFlag: "evals",
+        rbacScope: "evalJob:read",
+      },
+      {
+        name: "Log",
+        pathname: `/project/[projectId]/evals/log`,
+        cloudOnly: true,
+        rbacScope: "evalJobExecution:read",
       },
     ],
   },
@@ -94,7 +100,6 @@ export const ROUTES: Route[] = [
     name: "Playground",
     pathname: "/project/[projectId]/playground",
     icon: TerminalIcon,
-    featureFlag: "playground",
     cloudOnly: true,
     label: "Beta",
   },

web/src/components/layouts/status-badge.tsx

@@ -1,42 +1,59 @@
 import { cn } from "@/src/utils/tailwind";
 
 const statusCategories = {
-  active: ["production", "live", "active"],
+  active: ["production", "live", "active", "pending"],
   inactive: ["disabled", "inactive"],
+  completed: ["completed", "done", "finished"],
+  error: ["error", "failed"],
 };
 
 export type Status =
   (typeof statusCategories)[keyof typeof statusCategories][number];
 
 export const StatusBadge = (props: { className?: string; type: Status }) => {
+  let badgeColor = "bg-gray-100 text-gray-800";
+  let dotColor = "bg-gray-500";
+  let dotPingColor = "bg-gray-600";
+  let showDot = true;
+
+  if (statusCategories.active.includes(props.type)) {
+    badgeColor = "bg-green-100 text-green-600";
+    dotColor = "animate-ping bg-green-500";
+    dotPingColor = "bg-green-600";
+  } else if (statusCategories.error.includes(props.type)) {
+    badgeColor = "bg-red-100 text-red-600";
+    dotColor = "bg-red-500";
+    dotPingColor = "bg-red-600";
+    showDot = false;
+  } else if (statusCategories.completed.includes(props.type)) {
+    badgeColor = "bg-green-100 text-green-600";
+    showDot = false;
+  }
+
   return (
     <div
       className={cn(
         "inline-flex items-center gap-2 rounded-sm px-2 py-1 text-xs",
-        statusCategories.active.includes(props.type)
-          ? " bg-green-100 text-green-600"
-          : "bg-gray-100 text-gray-800",
+        badgeColor,
         props.className,
       )}
     >
-      <span className="relative inline-flex h-2 w-2">
-        <span
-          className={cn(
-            "absolute inline-flex h-full w-full  rounded-full opacity-75",
-            statusCategories.active.includes(props.type)
-              ? "animate-ping bg-green-500"
-              : "bg-gray-500",
-          )}
-        ></span>
-        <span
-          className={cn(
-            "relative inline-flex h-2 w-2 rounded-full ",
-            statusCategories.active.includes(props.type)
-              ? "bg-green-600"
-              : "bg-gray-600",
-          )}
-        ></span>
-      </span>
+      {showDot && (
+        <span className="relative inline-flex h-2 w-2">
+          <span
+            className={cn(
+              "absolute inline-flex h-full w-full  rounded-full opacity-75",
+              dotColor,
+            )}
+          ></span>
+          <span
+            className={cn(
+              "relative inline-flex h-2 w-2 rounded-full ",
+              dotPingColor,
+            )}
+          ></span>
+        </span>
+      )}
       <span>{props.type}</span>
     </div>
   );

web/src/components/publish-object-switch.tsx

@@ -1,8 +1,8 @@
 import { Switch } from "@/src/components/ui/switch";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { Link, LockIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useState } from "react";
 
 export const PublishTraceSwitch = (props: {
@@ -10,7 +10,7 @@ export const PublishTraceSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -30,7 +30,7 @@ export const PublishTraceSwitch = (props: {
           traceId: props.traceId,
           public: val,
         });
-        posthog.capture("trace_detail:publish_trace_button_click");
+        capture("trace_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}
@@ -43,7 +43,7 @@ export const PublishSessionSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -63,7 +63,7 @@ export const PublishSessionSwitch = (props: {
           sessionId: props.sessionId,
           public: val,
         });
-        posthog.capture("session_detail:publish_session_button_click");
+        capture("session_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}

web/src/components/session/index.tsx

@@ -13,13 +13,32 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
+import { useWindowVirtualizer } from "@tanstack/react-virtual";
+import { type RouterOutput } from "@/src/utils/types";
+import {
+  DataTableRowHeightSwitch,
+  type RowHeight,
+  useRowHeightLocalStorage,
+} from "@/src/components/table/data-table-row-height-switch";
+import { ScrollArea } from "@/src/components/ui/scroll-area";
+
+// do not use the usual table row heights here
+const rowHeightMapping: Record<RowHeight, number> = {
+  s: 200,
+  m: 350,
+  l: 700,
+};
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "single-session",
+    "m",
+  );
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -77,6 +96,11 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
+          <DataTableRowHeightSwitch
+            rowHeight={rowHeight}
+            setRowHeight={setRowHeight}
+            key="height"
+          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -97,36 +121,95 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
-        {session.data?.traces.map((trace) => (
-          <Card
-            className="border-border-gray-150 group grid gap-3 p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
-            key={trace.id}
-          >
-            <SessionIO traceId={trace.id} />
-            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
-              <Link
-                href={`/project/${projectId}/traces/${trace.id}`}
-                className="text-xs hover:underline"
-              >
-                Trace: {trace.name} ({trace.id})&nbsp;↗
-              </Link>
-              <div className="text-xs text-gray-500">
-                {trace.timestamp.toLocaleString()}
-              </div>
-              <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
-              <div className="flex flex-wrap content-start items-start gap-1">
-                <GroupedScoreBadges scores={trace.scores} />
-              </div>
-              <ManualScoreButton
-                projectId={projectId}
-                traceId={trace.id}
-                scores={trace.scores}
-                variant="badge"
-              />
-            </div>
-          </Card>
-        ))}
+      {session.data && (
+        <TraceCardList
+          session={session.data}
+          projectId={projectId}
+          rowHeight={rowHeightMapping[rowHeight]}
+        />
+      )}
+    </div>
+  );
+};
+
+const TraceCardList = ({
+  session,
+  projectId,
+  rowHeight,
+}: {
+  session: RouterOutput["sessions"]["byId"];
+  projectId: string;
+  rowHeight: number;
+}) => {
+  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
+
+  const virtualizer = useWindowVirtualizer({
+    count: session.traces.length,
+    estimateSize: () => rowHeight,
+    overscan: 5,
+    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
+    gap: 10,
+  });
+  useEffect(() => {
+    // re-measure when rowHeight changes to update the virtualizer
+    virtualizer.measure();
+  }, [rowHeight, virtualizer]);
+  return (
+    <div className="mt-5 border-t pt-5">
+      <div
+        ref={listVirtualizationRef}
+        style={{
+          height: `${virtualizer.getTotalSize()}px`,
+          position: "relative",
+        }}
+      >
+        {virtualizer
+          .getVirtualItems()
+          .map((virtualItem) => ({
+            virtualItem,
+            trace: session.traces[virtualItem.index],
+          }))
+          .map(({ virtualItem, trace }) => (
+            <Card
+              className="border-border-gray-150 group grid w-full gap-3 overflow-hidden p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
+              key={virtualItem.key}
+              data-index={virtualItem.index}
+              style={{
+                position: "absolute",
+                top: 0,
+                left: 0,
+                height: `${rowHeight}px`,
+                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
+              }}
+            >
+              <ScrollArea className="col-span-2 pr-3">
+                <SessionIO traceId={trace.id} />
+              </ScrollArea>
+              <ScrollArea>
+                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
+                  <Link
+                    href={`/project/${projectId}/traces/${trace.id}`}
+                    className="text-xs hover:underline"
+                  >
+                    Trace: {trace.name} ({trace.id})&nbsp;↗
+                  </Link>
+                  <div className="text-xs text-gray-500">
+                    {trace.timestamp.toLocaleString()}
+                  </div>
+                  <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
+                  <div className="flex flex-wrap content-start items-start gap-1">
+                    <GroupedScoreBadges scores={trace.scores} />
+                  </div>
+                  <ManualScoreButton
+                    projectId={projectId}
+                    traceId={trace.id}
+                    scores={trace.scores}
+                    variant="badge"
+                  />
+                </div>
+              </ScrollArea>
+            </Card>
+          ))}
       </div>
     </div>
   );
@@ -145,8 +228,9 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
+
   return (
-    <div className="col-span-2 flex flex-col gap-2 p-0">
+    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"

web/src/components/star-toggle.tsx

@@ -6,6 +6,7 @@ import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { cn } from "@/src/utils/tailwind";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function StarToggle({
   value,
@@ -53,7 +54,7 @@ export function StarTraceToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -109,13 +110,18 @@ export function StarTraceToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "traces",
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -133,7 +139,7 @@ export function StarTraceDetailsToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -180,13 +186,17 @@ export function StarTraceDetailsToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("trace_detail:bookmark_button_click", {
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -204,6 +214,7 @@ export function StarSessionToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
+  const capture = usePostHogClientCapture();
   const mutBookmarkSession = api.sessions.bookmark.useMutation({
     onSuccess: () => {
       void utils.sessions.invalidate();
@@ -216,13 +227,18 @@ export function StarSessionToggle({
       size={size}
       isLoading={mutBookmarkSession.isLoading}
       disabled={!hasAccess}
-      onClick={(value) =>
-        mutBookmarkSession.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "sessions",
+          id: sessionId,
+          value: value,
+        });
+        return mutBookmarkSession.mutateAsync({
           projectId,
           sessionId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }

web/src/components/table/data-table-column-visibility-filter.tsx

@@ -14,6 +14,7 @@ import {
 import { type VisibilityState } from "@tanstack/react-table";
 import { ChevronDown, Columns } from "lucide-react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTableColumnVisibilityFilterProps<TData, TValue> {
   columns: LangfuseColumnDef<TData, TValue>[];
@@ -27,14 +28,24 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
   setColumnVisibility,
 }: DataTableColumnVisibilityFilterProps<TData, TValue>) {
   const [isOpen, setIsOpen] = useState(false);
-
+  const capture = usePostHogClientCapture();
   const toggleColumn = useCallback(
     (columnId: string) => {
-      setColumnVisibility((old) => ({
-        ...old,
-        [columnId]: !old[columnId],
-      }));
+      setColumnVisibility((old) => {
+        const newColumnVisibility = {
+          ...old,
+          [columnId]: !old[columnId],
+        };
+        const selectedColumns = Object.keys(newColumnVisibility).filter(
+          (key) => newColumnVisibility[key],
+        );
+        capture("table:column_visibility_changed", {
+          selectedColumns: selectedColumns,
+        });
+        return newColumnVisibility;
+      });
     },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
     [setColumnVisibility],
   );
 

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -9,7 +9,6 @@ import {
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
-import { usePostHog } from "posthog-js/react";
 
 import {
   Dialog,
@@ -20,6 +19,7 @@ import {
   DialogTitle,
 } from "@/src/components/ui/dialog";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function TraceTableMultiSelectAction({
   selectedTraceIds,
@@ -32,7 +32,7 @@ export function TraceTableMultiSelectAction({
 }) {
   const utils = api.useUtils();
   const [open, setOpen] = useState(false);
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasDeleteAccess = useHasAccess({ projectId, scope: "traces:delete" });
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
@@ -55,6 +55,10 @@ export function TraceTableMultiSelectAction({
           <DropdownMenuItem
             disabled={!hasDeleteAccess}
             onClick={() => {
+              capture("trace:delete_form_open", {
+                count: selectedTraceIds.length,
+                source: "table-multi-select",
+              });
               setOpen(true);
             }}
           >
@@ -94,7 +98,7 @@ export function TraceTableMultiSelectAction({
                   .then(() => {
                     setOpen(false);
                   });
-                posthog.capture("trace:delete", {
+                capture("trace:delete_form_submit", {
                   count: selectedTraceIds.length,
                   source: "table-multi-select",
                 });

web/src/components/table/data-table-pagination.tsx

@@ -14,6 +14,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/src/components/ui/select";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTablePaginationProps<TData> {
   table: Table<TData>;
@@ -24,18 +25,27 @@ export function DataTablePagination<TData>({
   table,
   paginationOptions = [10, 20, 30, 40, 50],
 }: DataTablePaginationProps<TData>) {
+  const capture = usePostHogClientCapture();
   return (
-    <div className="mt-3 flex items-center justify-between overflow-x-auto px-2">
+    <div className="flex items-center justify-between">
       <div className="flex-1 text-sm text-muted-foreground">
         {/* {table.getFilteredSelectedRowModel().rows.length} of{" "}
         {table.getFilteredRowModel().rows.length} row(s) selected. */}
       </div>
-      <div className="flex items-center space-x-6 lg:space-x-8">
+      <div className="flex flex-wrap items-center space-x-6 lg:space-x-8">
         <div className="flex items-center space-x-2">
-          <p className="whitespace-nowrap text-sm font-medium">Rows per page</p>
+          <p className="whitespace-nowrap text-sm font-medium md:hidden">
+            Rows
+          </p>
+          <p className="hidden whitespace-nowrap text-sm font-medium md:block">
+            Rows per page
+          </p>
           <Select
             value={`${table.getState().pagination.pageSize}`}
             onValueChange={(value) => {
+              capture("table:pagination_page_size_select", {
+                pageSize: value,
+              });
               table.setPageSize(Number(value));
             }}
           >
@@ -51,7 +61,7 @@ export function DataTablePagination<TData>({
             </SelectContent>
           </Select>
         </div>
-        <div className="flex w-[100px] items-center justify-center whitespace-nowrap text-sm font-medium">
+        <div className="flex items-center justify-center whitespace-nowrap text-sm font-medium">
           Page {table.getState().pagination.pageIndex + 1} of{" "}
           {table.getPageCount()}
         </div>
@@ -59,7 +69,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(0)}
+            onClick={() => {
+              table.setPageIndex(0);
+              capture("table:pagination_button_click", {
+                type: "firstPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to first page</span>
@@ -68,7 +83,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.previousPage()}
+            onClick={() => {
+              table.previousPage();
+              capture("table:pagination_button_click", {
+                type: "previousPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to previous page</span>
@@ -77,7 +97,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.nextPage()}
+            onClick={() => {
+              table.nextPage();
+              capture("table:pagination_button_click", {
+                type: "nextPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to next page</span>
@@ -86,7 +111,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(table.getPageCount() - 1)}
+            onClick={() => {
+              table.setPageIndex(table.getPageCount() - 1);
+              capture("table:pagination_button_click", {
+                type: "lastPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to last page</span>

web/src/components/table/data-table-row-height-switch.tsx

@@ -1,5 +1,6 @@
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import {
   MdDensityLarge,
   MdDensityMedium,
@@ -35,25 +36,33 @@ export const DataTableRowHeightSwitch = ({
 }: {
   rowHeight: RowHeight;
   setRowHeight: (e: RowHeight) => void;
-}) => (
-  <Tabs
-    //defaultValue={height}
-    value={rowHeight}
-    onValueChange={(e) => setRowHeight(e as any)}
-    key="height"
-  >
-    <TabsList className="gap-1 border bg-transparent px-2">
-      {heightOptions.map(({ id, label, icon }) => (
-        <TabsTrigger
-          key={id}
-          value={id}
-          className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
-        >
-          <span role="img" aria-label={`${label} size`}>
-            {icon}
-          </span>
-        </TabsTrigger>
-      ))}
-    </TabsList>
-  </Tabs>
-);
+}) => {
+  const capture = usePostHogClientCapture();
+  return (
+    <Tabs
+      //defaultValue={height}
+      value={rowHeight}
+      onValueChange={(e) => {
+        capture("table:row_height_switch_select", {
+          rowHeight: e,
+        });
+        setRowHeight(e as any);
+      }}
+      key="height"
+    >
+      <TabsList className="gap-1 border bg-transparent px-2">
+        {heightOptions.map(({ id, label, icon }) => (
+          <TabsTrigger
+            key={id}
+            value={id}
+            className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
+          >
+            <span role="img" aria-label={`${label} size`}>
+              {icon}
+            </span>
+          </TabsTrigger>
+        ))}
+      </TabsList>
+    </Tabs>
+  );
+};

web/src/components/table/data-table-toolbar.tsx

@@ -12,6 +12,7 @@ import {
   type RowHeight,
 } from "@/src/components/table/data-table-row-height-switch";
 import { Search } from "lucide-react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface SearchConfig {
   placeholder: string;
@@ -47,9 +48,10 @@ export function DataTableToolbar<TData, TValue>({
   const [searchString, setSearchString] = useState(
     searchConfig?.currentQuery ?? "",
   );
+  const capture = usePostHogClientCapture();
 
   return (
-    <div className="my-2 flex flex-1 flex-wrap items-center gap-2">
+    <div className="my-2 flex flex-wrap items-center gap-2 @container">
       {searchConfig && (
         <div className="flex max-w-md items-center">
           <Input
@@ -59,14 +61,18 @@ export function DataTableToolbar<TData, TValue>({
             onChange={(event) => setSearchString(event.currentTarget.value)}
             onKeyDown={(event) => {
               if (event.key === "Enter") {
+                capture("table:search_submit");
                 searchConfig.updateQuery(searchString);
               }
             }}
-            className="h-10 w-[200px] rounded-r-none lg:w-[250] 2xl:w-[350px]"
+            className="h-10 w-[150px] rounded-r-none @6xl:w-[250px]"
           />
           <Button
             variant="outline"
-            onClick={() => searchConfig.updateQuery(searchString)}
+            onClick={() => {
+              capture("table:search_submit");
+              searchConfig.updateQuery(searchString);
+            }}
             className="rounded-l-none border-l-0 p-3"
           >
             <Search className="h-4 w-4" />
@@ -80,7 +86,7 @@ export function DataTableToolbar<TData, TValue>({
           onChange={setFilterState}
         />
       )}
-      <div className="flex flex-row flex-wrap gap-2 lg:ml-auto">
+      <div className="flex flex-row flex-wrap gap-2 pr-0.5 @6xl:ml-auto">
         {!!columnVisibility && !!setColumnVisibility && (
           <DataTableColumnVisibilityFilter
             columns={columns}