chore: release v2.44.0

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

.env.prod.example

@@ -80,9 +80,23 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # ENABLE_EVENT_LOG=false
 
 
+
+### START Envs to be deprecated in Langfuse v3.0
+
+# Disable the expensive analytics queries and related features
+# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
+
+# Customize the default datetime offset filter used in for the UI tables
+# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
+
+### END Envs to be deprecated in Langfuse v3.0
+
+
+
+
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
-# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
 
 # NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
 # NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -12,10 +12,24 @@ body:
   - type: textarea
     attributes:
       label: To reproduce
-      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+      description: Describe how to reproduce your bug. Please provide detailed steps, code snippets, reproduction repos etc.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: SDK and container versions
+      description: If you're experiencing an issue with an integration or SDK, please ensure you're using the latest version. If you're self-hosting Langfuse, check that you're running the most recent version. If updating isn't an option, please provide the specific versions you're currently using.
   - type: textarea
     attributes:
       label: Additional information
       description: Add any other information related to the bug here, screenshots if applicable.
+  - type: dropdown
+    id: contribute
+    attributes:
+      label: Are you interested to contribute a fix for this bug?
+      description: If this is a confirmed bug, the maintainers are happy to support with guidance and review.
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true

.github/workflows/pipeline.yml

@@ -109,9 +109,6 @@ jobs:
     strategy:
       matrix:
         node-version: [20]
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }}
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master

CONTRIBUTING.md

@@ -290,6 +290,16 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Using secrets stored in 1Password
+
+When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
+
+Example:
+
+```bash
+op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
+```
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2023 Finto Technologies GmbH
+Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 

careers

@@ -0,0 +1,14 @@
+_We are Hiring_
+Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+
+_Open Roles_
+
+- Backend Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-100k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
+_More Info_
+
+- https://langfuse.com/careers
+- https://langfuse.com/docs
+- https://langfuse.com/changelog

docker-compose.yml

@@ -4,7 +4,8 @@ services:
   langfuse-server:
     image: ghcr.io/langfuse/langfuse:latest
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     ports:
       - "3000:3000"
     environment:
@@ -18,6 +19,11 @@ services:
   db:
     image: postgres
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres

ee/LICENSE

@@ -0,0 +1,37 @@
+Langfuse Enterprise License (the “Enterprise License”)
+Copyright (c) 2024 Langfuse GmbH ('Langfuse')
+
+With regard to the Langfuse Software:
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Langfuse Terms of Service, available
+at https://langfuse.com/terms (the “Enterprise Terms”), or other
+agreement governing the use of the Software, as agreed by you and Langfuse,
+and otherwise have a valid Langfuse Enterprise license.
+
+Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that Langfuse
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Langfuse Enterprise license.
+
+Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription. You agree that Langfuse and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications. You are not
+granted any other rights beyond what is expressly stated herein. Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the Langfuse Software, those
+components are licensed under the original license provided by the owner of the 
+applicable component.

ee/package.json

@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "zod": "^3.22.4"
   },

fern/apis/client/generators.yml

@@ -15,7 +15,7 @@ groups:
         version: 0.0.26
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-client
+          path: ../../../web/public/generated/api-client
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true

fern/apis/server/definition/api.yml

@@ -9,8 +9,8 @@ docs: |
 
   ## Exports
 
-  - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
-  - Postman collection: https://cloud.langfuse.com/postman-collection.json
+  - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/postman/collection.json
 
 error-discrimination:
   strategy: status-code

fern/apis/server/definition/dataset-items.yml

@@ -32,3 +32,6 @@ types:
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id
+      status:
+        type: optional<commons.DatasetStatus>
+        docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/metrics.yml

@@ -31,10 +31,10 @@ service:
             docs: Optional filter for metrics where traces include all of these tags
           fromTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces on or after a certain timestamp
+            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
           toTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces before a certain timestamp
+            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:

fern/apis/server/definition/observations.yml

@@ -35,7 +35,7 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this timestamp.
+            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/score.yml

@@ -29,7 +29,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this timestamp.
+            docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
             type: commons.ScoreSource
             docs: Retrieve only scores from a specific source.

fern/apis/server/definition/trace.yml

@@ -32,7 +32,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this timestamp.
+            docs: Retrieve only traces newer than this datetime (ISO 8601).
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.0.28
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-server
+          path: ../../../web/public/generated/api
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true
@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../web/generated/postman
+          path: ../../../web/public/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.37.2",
+  "version": "2.44.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -30,8 +30,8 @@
     "@release-it/bumper": "^6.0.1",
     "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.0",
-    "turbo": "^1.13.2"
+    "release-it": "^17.2.1",
+    "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -11,9 +11,9 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.7.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.2",
+    "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -44,14 +44,14 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.28",
-    "@prisma/client": "^5.12.1",
+    "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -71,12 +71,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.0"
+    "vitest": "^1.5.3"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,13 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const MembershipRole = {
+export const ProjectRole = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
     VIEWER: "VIEWER"
 } as const;
-export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -96,7 +96,7 @@ export type AuditLog = {
     updated_at: Generated<Timestamp>;
     user_id: string;
     project_id: string;
-    user_project_role: MembershipRole;
+    user_project_role: ProjectRole;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -208,17 +208,10 @@ export type LlmApiKeys = {
     secret_key: string;
     project_id: string;
 };
-export type Membership = {
-    project_id: string;
-    user_id: string;
-    role: MembershipRole;
-    created_at: Generated<Timestamp>;
-    updated_at: Generated<Timestamp>;
-};
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: MembershipRole;
+    role: ProjectRole;
     project_id: string;
     sender_id: string | null;
     created_at: Generated<Timestamp>;
@@ -300,6 +293,7 @@ export type ObservationView = {
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
     latency: number | null;
+    time_to_first_token: number | null;
 };
 export type PosthogIntegration = {
     project_id: string;
@@ -324,6 +318,13 @@ export type Project = {
     name: string;
     cloud_config: unknown | null;
 };
+export type ProjectMembership = {
+    project_id: string;
+    user_id: string;
+    role: ProjectRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Prompt = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -342,6 +343,7 @@ export type Prompt = {
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
+    project_id: string;
     name: string;
     value: number;
     source: ScoreSource;
@@ -378,6 +380,8 @@ export type Trace = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type TraceSession = {
     id: string;
@@ -437,12 +441,12 @@ export type DB = {
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
     membership_invitations: MembershipInvitation;
-    memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
     pricings: Pricing;
+    project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
     scores: Score;

packages/shared/prisma/migrations/20240503125742_traces_add_createdat_updatedat/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240503130335_traces_index_created_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_created_at_idx" ON "traces"("created_at");

packages/shared/prisma/migrations/20240503130520_traces_index_updated_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_updated_at_idx" ON "traces"("updated_at");

packages/shared/prisma/migrations/20240508132621_scores_add_project_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "project_id" TEXT;

packages/shared/prisma/migrations/20240508132735_scores_add_projectid_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_project_id_idx" ON "scores"("project_id");

packages/shared/prisma/migrations/20240508132736_scores_backfill_project_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill project_id on existing scores based on linked trace_id
+UPDATE scores
+SET project_id = traces.project_id
+FROM traces 
+WHERE scores.trace_id = traces.id AND scores.project_id IS NULL;

packages/shared/prisma/migrations/20240512151529_rename_memberships_to_project_memberships/migration.sql

@@ -0,0 +1,15 @@
+-- Rename table, will cause a downtime as it needs to be locked and synced with TRPC API
+ALTER TABLE "memberships"
+RENAME TO "project_memberships";
+
+-- AlterTable
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_pkey" TO "project_memberships_pkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_project_id_fkey" TO "project_memberships_project_id_fkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_user_id_fkey" TO "project_memberships_user_id_fkey";
+
+-- RenameIndex
+ALTER INDEX "memberships_user_id_idx" RENAME TO "project_memberships_user_id_idx";

packages/shared/prisma/migrations/20240512155020_rename_enum_membership_role_to_project_role/migration.sql

@@ -0,0 +1,37 @@
+-- Goal: Rename the enum `MembershipRole` to `ProjectRole` and update all tables that use it
+
+-- Create a new Enum
+CREATE TYPE "ProjectRole" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER');
+
+-- Add a temporary column to tables (for each table)
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role_temp" text;
+ALTER TABLE "membership_invitations" ADD COLUMN "role_temp" text;
+ALTER TABLE "project_memberships" ADD COLUMN "role_temp" text;
+
+-- Copy data over to temporary columns (for each table)
+UPDATE "audit_logs" SET "user_project_role_temp" = "user_project_role"::text;
+UPDATE "membership_invitations" SET "role_temp" = "role"::text;
+UPDATE "project_memberships" SET "role_temp" = "role"::text;
+
+-- Drop old columns
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "membership_invitations" DROP COLUMN "role";
+ALTER TABLE "project_memberships" DROP COLUMN "role";
+
+-- Rename temporary columns to old column names
+ALTER TABLE "audit_logs" RENAME COLUMN "user_project_role_temp" TO "user_project_role";
+ALTER TABLE "membership_invitations" RENAME COLUMN "role_temp" TO "role";
+ALTER TABLE "project_memberships" RENAME COLUMN "role_temp" TO "role";
+
+-- Convert the text columns to enum columns (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" TYPE "ProjectRole" USING "user_project_role"::"ProjectRole";
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+ALTER TABLE "project_memberships" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+
+-- Make the columns NOT NULL (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" SET NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" SET NOT NULL;
+ALTER TABLE "project_memberships" ALTER COLUMN "role" SET NOT NULL;
+
+-- Now finally, you can drop your old enum
+DROP TYPE "MembershipRole";

packages/shared/prisma/migrations/20240512155021_add_pricing_gpt4o/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o
+  ('b9854a5c92dc496b997d99d20', NULL, 'gpt-4o', '(?i)^(gpt-4o)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-2024-05-13
+  ('b9854a5c92dc496b997d99d21', NULL, 'gpt-4o-2024-05-13', '(?i)^(gpt-4o-2024-05-13)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o-2024-05-13" }')
+  

packages/shared/prisma/migrations/20240512155021_scores_drop_fk_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_trace_id_fkey";

packages/shared/prisma/migrations/20240512155022_scores_non_null_and_add_fk_project_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Made the column `project_id` on table `scores` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "project_id" SET NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240513082203_scores_unique_id_and_projectid_instead_of_id_and_traceid/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "scores_id_trace_id_key";

packages/shared/prisma/migrations/20240513082204_scores_unique_id_and_projectid_instead_of_id_and_traceid_index/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "scores_id_project_id_key" ON "scores"("id", "project_id");

packages/shared/prisma/migrations/20240513082205_observations_view_add_time_to_first_token/migration.sql

@@ -0,0 +1,61 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views"]
+    previewFeatures = ["tracing", "views", "relationJoins"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id            String                 @id @default(cuid())
-    name          String?
-    email         String?                @unique
-    emailVerified DateTime?              @map("email_verified")
-    password      String?
-    image         String?
-    admin         Boolean                @default(false)
-    accounts      Account[]
-    sessions      Session[]
-    memberships   Membership[]
-    invitations   MembershipInvitation[]
-    createdAt     DateTime               @default(now()) @map("created_at")
-    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags  String[]               @default([]) @map("feature_flags")
-    AuditLog      AuditLog[]
+    id                 String                 @id @default(cuid())
+    name               String?
+    email              String?                @unique
+    emailVerified      DateTime?              @map("email_verified")
+    password           String?
+    image              String?
+    admin              Boolean                @default(false)
+    accounts           Account[]
+    sessions           Session[]
+    projectMemberships ProjectMembership[]
+    invitations        MembershipInvitation[]
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags       String[]               @default([]) @map("feature_flags")
+    AuditLog           AuditLog[]
 
     @@map("users")
 }
@@ -97,7 +97,7 @@ model Project {
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
     cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members            Membership[]
+    projectMembers     ProjectMembership[]
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -113,6 +113,7 @@ model Project {
     JobExecution       JobExecution[]
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
+    Score              Score[]
 
     @@map("projects")
 }
@@ -154,37 +155,37 @@ model LlmApiKeys {
     @@map("llm_api_keys")
 }
 
-model Membership {
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String         @map("user_id")
-    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      MembershipRole
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+model ProjectMembership {
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String      @map("user_id")
+    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      ProjectRole
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
-    @@map("memberships")
+    @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String         @id @unique @default(cuid())
+    id        String      @id @unique @default(cuid())
     email     String
-    role      MembershipRole
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?        @map("sender_id")
-    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    role      ProjectRole
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?     @map("sender_id")
+    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum MembershipRole {
+enum ProjectRole {
     OWNER
     ADMIN
     MEMBER
@@ -227,8 +228,9 @@ model Trace {
     output     Json?
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
+    createdAt  DateTime      @default(now()) @map("created_at")
+    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    scores          Score[]
     DatasetRunItems DatasetRunItems[]
     DatasetItem     DatasetItem[]
     JobExecution    JobExecution[]
@@ -241,6 +243,8 @@ model Trace {
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
+    @@index(updatedAt)
+    @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
 }
@@ -272,6 +276,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -301,7 +307,6 @@ model Observation {
     outputCost          Decimal?          @map("output_cost")
     totalCost           Decimal?          @map("total_cost")
     completionStartTime DateTime?         @map("completion_start_time")
-    scores              Score[]
     project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
     datasetRunItems     DatasetRunItems[]
@@ -368,6 +373,7 @@ view ObservationView {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
     latency              Float?   @map("latency")
+    timeToFirstToken     Float?   @map("time_to_first_token")
 
     @@map("observations_view")
 }
@@ -388,19 +394,20 @@ enum ObservationLevel {
 model Score {
     id            String         @id @default(cuid())
     timestamp     DateTime       @default(now())
+    projectId     String         @map("project_id")
+    project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
     value         Float
     source        ScoreSource
     comment       String?
     traceId       String         @map("trace_id")
-    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
     observationId String?        @map("observation_id")
-    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
     JobExecution  JobExecution[]
 
-    @@unique([id, traceId]) // used for upsert
+    @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
+    @@index([projectId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
@@ -594,16 +601,16 @@ model Model {
 }
 
 model AuditLog {
-    id              String         @id @default(cuid())
-    createdAt       DateTime       @default(now()) @map("created_at")
-    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
-    userId          String         @map("user_id")
-    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String         @map("project_id")
-    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole MembershipRole @map("user_project_role")
-    resourceType    String         @map("resource_type")
-    resourceId      String         @map("resource_id")
+    id              String      @id @default(cuid())
+    createdAt       DateTime    @default(now()) @map("created_at")
+    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
+    userId          String      @map("user_id")
+    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String      @map("project_id")
+    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole ProjectRole @map("user_project_role")
+    resourceType    String      @map("resource_type")
+    resourceId      String      @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON

packages/shared/prisma/seed.ts

@@ -51,7 +51,7 @@ async function main() {
     create: {
       id: seedProjectId,
       name: "llm-app",
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,
@@ -72,7 +72,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      labels: ["production"],
+      labels: ["production", "latest"],
       version: 1,
       createdBy: "user-1",
     },
@@ -110,7 +110,7 @@ async function main() {
       create: {
         id: "239ad00f-562f-411d-af14-831c75ddd875",
         name: "demo-app",
-        members: {
+        projectMembers: {
           create: {
             role: "OWNER",
             userId: user.id,
@@ -459,7 +459,7 @@ function createObjects(
     const session =
       Math.random() > 0.3
         ? {
-            id: `session-${i % 10}`,
+            id: `session-${i % 3}`,
             projectId: projectId,
           }
         : undefined;
@@ -501,6 +501,7 @@ function createObjects(
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.REVIEW,
+              projectId,
             },
           ]
         : []),
@@ -512,6 +513,7 @@ function createObjects(
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
               source: ScoreSource.API,
+              projectId,
             },
           ]
         : []),
@@ -526,9 +528,9 @@ function createObjects(
       const spanTsStart = new Date(
         traceTs.getTime() + Math.floor(Math.random() * 30)
       );
-      // random duration of upto 30ms
+      // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 30)
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
       );
 
       const span = {
@@ -572,6 +574,13 @@ function createObjects(
                 (spanTsEnd.getTime() - generationTsStart.getTime())
             )
         );
+        // somewhere in the middle
+        const generationTsCompletionStart = new Date(
+          generationTsStart.getTime() +
+            Math.floor(
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
+            )
+        );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
         const completionTokens = Math.floor(Math.random() * 500) + 100;
@@ -590,7 +599,9 @@ function createObjects(
         const model = models[Math.floor(Math.random() * models.length)];
         const promptId =
           promptIds.get(projectId)![
-            Math.floor(Math.random() * promptIds.get(projectId)!.length)
+            Math.floor(
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+            )
           ];
 
         const generation = {
@@ -598,6 +609,8 @@ function createObjects(
           id: `generation-${v4()}`,
           startTime: generationTsStart,
           endTime: generationTsEnd,
+          completionStartTime:
+            Math.random() > 0.5 ? generationTsCompletionStart : undefined,
           name: `generation-${i}-${j}-${k}`,
           projectId: trace.projectId,
           promptId: promptId,
@@ -693,6 +706,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -701,6 +715,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -764,7 +779,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -773,7 +788,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -782,7 +797,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -791,7 +806,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -859,6 +874,7 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
+      labels: ["production", "latest"],
     },
   ];
 
@@ -908,7 +924,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        labels: i === 20 ? ["production"] : [],
+        labels: i === 20 ? ["production", "latest"] : [],
       },
       update: {
         id: promptId,

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider } from "../..";
+import { ModelProvider, UIModelParams } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,26 +77,38 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels = [
+export const evalLLMModels: UIModelParams[] = [
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-3.5-turbo",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-3.5-turbo", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-4-turbo-preview",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4-turbo-preview", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4o", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
 ] as const;
 
-export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+export const EvalModelNames = z.enum([
+  "gpt-3.5-turbo",
+  "gpt-4-turbo-preview",
+  "gpt-4o",
+]);
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/server/llm/types.ts

@@ -20,11 +20,15 @@ export enum ChatMessageRole {
   Assistant = "assistant",
 }
 export type ModelParams = AnthropicModelParams | OpenAIModelParams;
-export type UIModelParams = Required<
-  AnthropicModelParams | OpenAIModelParams
-> & {
-  maxTemperature: number;
+
+type RecordWithEnabledFlag<T> = {
+  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
+export type UIModelParams = RecordWithEnabledFlag<
+  Required<AnthropicModelParams | OpenAIModelParams> & {
+    maxTemperature: number;
+  }
+>;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
@@ -42,6 +46,8 @@ export type OpenAIModelParams = {
 } & ModelConfig;
 
 export const openAIModels = [
+  "gpt-4o",
+  "gpt-4o-2024-05-13",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

web/package.json

@@ -1,14 +1,13 @@
 {
   "name": "web",
-  "version": "2.37.2",
+  "version": "2.44.0",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml && cp generated/postman/collection.json public/postman-collection.json",
-    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -23,17 +22,17 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
     "@headlessui/react": "^1.7.19",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.28",
-    "@langfuse/shared": "workspace:*",
     "@langfuse/ee": "workspace:*",
+    "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -56,18 +55,20 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.16",
-    "@react-email/render": "^0.0.12",
+    "@react-email/components": "^0.0.17",
+    "@react-email/render": "^0.0.13",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.110.1",
-    "@sentry/node": "^7.110.1",
-    "@sentry/profiling-node": "^7.110.1",
-    "@sentry/types": "^7.110.1",
+    "@sentry/nextjs": "^7.113.0",
+    "@sentry/node": "^7.113.0",
+    "@sentry/profiling-node": "^7.113.0",
+    "@sentry/types": "^7.113.0",
     "@t3-oss/env-nextjs": "^0.8.0",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.11.1",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
@@ -78,25 +79,26 @@
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
+    "dompurify": "^3.1.2",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.11",
+    "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
@@ -117,9 +119,10 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
     "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^15.0.2",
+    "@testing-library/react": "^15.0.6",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
@@ -133,7 +136,7 @@
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",

web/public/.gitignore

@@ -1,2 +0,0 @@
-openapi*.yml
-postman-collection.json

web/public/generated/api/openapi.yml

@@ -19,9 +19,10 @@ info:
     ## Exports
 
 
-    - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
+    - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
 
-    - Postman collection: https://cloud.langfuse.com/postman-collection.json
+    - Postman collection:
+    https://cloud.langfuse.com/generated/postman/collection.json
 paths:
   /api/public/dataset-items:
     post:
@@ -495,7 +496,7 @@ paths:
           in: query
           description: >-
             Optional filter to only include traces on or after a certain
-            timestamp
+            datetime (ISO 8601)
           required: false
           schema:
             type: string
@@ -503,7 +504,9 @@ paths:
             nullable: true
         - name: toTimestamp
           in: query
-          description: Optional filter to only include traces before a certain timestamp
+          description: >-
+            Optional filter to only include traces before a certain datetime
+            (ISO 8601)
           required: false
           schema:
             type: string
@@ -647,7 +650,7 @@ paths:
           in: query
           description: >-
             Retrieve only observations with a start_time greater than this
-            timestamp.
+            datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -985,7 +988,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only scores newer than this timestamp.
+          description: Retrieve only scores newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1261,7 +1264,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only traces newer than this timestamp.
+          description: Retrieve only traces newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1847,6 +1850,10 @@ components:
           type: string
           nullable: true
           description: Dataset items are upserted on their id
+        status:
+          $ref: '#/components/schemas/DatasetStatus'
+          nullable: true
+          description: Defaults to ACTIVE for newly created items
       required:
         - datasetName
     CreateDatasetRunItemRequest:

web/public/generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml\n- Postman collection: https://cloud.langfuse.com/postman-collection.json"
+    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml\n- Postman collection: https://cloud.langfuse.com/generated/postman/collection.json"
   },
   "variable": [
     {
@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\",\n    \"status\": \"ACTIVE\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -418,12 +418,12 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces on or after a certain timestamp"
+                  "description": "Optional filter to only include traces on or after a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "toTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces before a certain timestamp"
+                  "description": "Optional filter to only include traces before a certain datetime (ISO 8601)"
                 }
               ],
               "variable": []
@@ -528,7 +528,7 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this timestamp."
+                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -788,7 +788,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this timestamp."
+                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "source",
@@ -996,7 +996,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this timestamp."
+                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "orderBy",

web/sentry.server.config.ts

@@ -20,6 +20,15 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       ) {
         return 0.3;
       }
+      if (
+        samplingContext.request &&
+        samplingContext.request.url &&
+        samplingContext.request.url.includes("api/auth") &&
+        samplingContext.transactionContext.status !== "ok" &&
+        samplingContext.transactionContext.status !== "unauthenticated"
+      ) {
+        return 1;
+      }
       return 0.1;
     },
 
@@ -28,6 +37,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),
       new Sentry.Integrations.Prisma({ client: prisma }),
+      Sentry.metrics.metricsAggregatorIntegration(),
     ],
 
     // filter out passwords from the signup request body

web/src/__e2e__/auth.spec.ts

@@ -70,3 +70,38 @@ test("Signup input validation", async ({ page }) => {
 // random email address to be used in tests
 const randomEmailAddress = () =>
   Math.random().toString(36).substring(2, 11) + "@example.com";
+
+test("Unauthenticated user should be redirected to target URL after login", async ({
+  page,
+}) => {
+  await page.goto("/auth/sign-in");
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  // wait 2 seconds
+  await page.waitForTimeout(2000);
+
+  // project id and prompt from seed.ts
+  const promptUrl = "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
+
+  await page.getByRole("button", { name: /Demo User/ }).click();
+
+  await page.getByRole("menuitem", { name: "Sign Out" }).click();
+
+  await expect(page).toHaveURL("/auth/sign-in");
+
+  await page.goto(promptUrl);
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(/targetPath/);
+
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(promptUrl);
+});

web/src/__tests__/datasets.servertest.ts

@@ -210,11 +210,14 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       datasetName: "dataset-name",
       input: { key: "value2" },
       metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
     expect(item2.status).toBe(200);
     expect(item2.body).toMatchObject({
       id: "dataset-item-id",
       input: { key: "value2" },
+      metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
 
     const dbDatasetItem = await prisma.datasetItem.findFirst({
@@ -223,6 +226,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dbDatasetItem).not.toBeNull();
     expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
     expect(dbDatasetItem?.metadata).toMatchObject(["hello-world"]);
+    expect(dbDatasetItem?.status).toBe("ARCHIVED");
   });
 
   it("should create and get a dataset run, include special characters", async () => {

web/src/__tests__/dom.clienttest.ts

@@ -0,0 +1,19 @@
+import * as DOMPurify from "dompurify";
+
+// Create test cases
+describe("should sanitize dom", () => {
+  it("should sanitize urls", () => {
+    const dirty =
+      "http://www.some.site/page.html?default=<script>alert(document.cookie)</script>";
+    const cleam = DOMPurify.sanitize(dirty);
+
+    expect(cleam).toBe("http://www.some.site/page.html?default=");
+  });
+  it("should should not sanitize valid langfuse urls", () => {
+    const validLangfuseUrl =
+      "https://cloud.langfuse.com/project/abjdnksadn/traces?filter=timestamp%3Bdatetime%3B%3B%3E%3B2024-04-19T00%253A00%253A00.000Z%2Cid%3Bstring%3B%3Bcontains%3Ba%2Ctimestamp%3Bdatetime%3B%3B%3C%3B2024-04-30T22%253A00%253A00.000Z%2CinputTokens%3Bnumber%3B%3B%3E%3B1";
+    const cleam = DOMPurify.sanitize(validLangfuseUrl);
+
+    expect(cleam).toBe(validLangfuseUrl);
+  });
+});

web/src/__tests__/ingestion.servertest.ts

@@ -281,6 +281,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expect(dbScore?.name).toBe("score-name");
       expect(dbScore?.value).toBe(100.5);
       expect(dbScore?.observationId).toBeNull();
+      expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
     });
   });
 
@@ -726,6 +727,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             metadata: { key: "value" },
             release: "1.0.0",
             version: "2.0.0",
+            tags: ["tag-1", "tag-2"],
           },
         },
       ],
@@ -742,6 +744,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             id: traceId,
             name: "trace-name",
             userId: "user-2",
+            tags: ["tag-3", "tag-4"],
           },
         },
       ],
@@ -762,6 +765,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace[0]?.externalId).toBeNull();
     expect(dbTrace[0]?.version).toBe("2.0.0");
     expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(dbTrace[0]?.tags).toEqual(["tag-1", "tag-2", "tag-3", "tag-4"]);
   });
 
   it("should fail for wrong event formats", async () => {
@@ -816,57 +820,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace.length).toBe(1);
   });
 
-  it("should fail for auth errors", async () => {
-    const traceId = v4();
-    const scoreId = v4();
-
-    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
-      batch: [
-        {
-          id: v4(),
-          type: "trace-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: traceId,
-            name: "trace-name",
-            userId: "user-1",
-            metadata: { key: "value" },
-            release: "1.0.0",
-            version: "2.0.0",
-          },
-        },
-        {
-          id: v4(),
-          type: "score-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: scoreId,
-            name: "score-name",
-            value: 100.5,
-            traceId: "some-random-id",
-          },
-        },
-      ],
-    });
-
-    expect(responseOne.status).toBe(207);
-
-    expect("errors" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.errors.length).toBe(1);
-    expect("successes" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.successes.length).toBe(1);
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        name: "trace-name",
-      },
-    });
-
-    expect(dbTrace.length).toBe(1);
-  });
-
   it("should fail for resource not found", async () => {
     const traceId = v4();
 
@@ -1653,121 +1606,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
     expect(dbScore?.value).toBe(scoreValue);
   });
 
-  it("should error on Bearer Auth for a trace from different project", async () => {
-    const otherProjectId = "other_project_id";
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    // Seed db with a trace to be scored
-    try {
-      await prisma.project.create({
-        data: {
-          id: otherProjectId,
-          name: "another-project",
-        },
-      });
-
-      await prisma.trace.create({
-        data: {
-          id: traceId,
-          name: "trace-name",
-          project: { connect: { id: otherProjectId } },
-        },
-      });
-
-      const response = await makeAPICall(
-        "POST",
-        "/api/public/ingestion",
-        {
-          batch: [
-            {
-              id: scoreEventId,
-              type: "score-create",
-              timestamp: new Date().toISOString(),
-              body: {
-                id: scoreId,
-                name: scoreName,
-                value: scoreValue,
-                traceId: traceId,
-              },
-            },
-          ],
-        },
-        bearerAuth,
-      );
-
-      expect(response.status).toBe(207);
-      expect(response.body.successes.length).toBe(0);
-      expect(response.body).toHaveProperty("errors");
-      expect(response.body.errors.length).toEqual(1);
-      expect(response.body.errors[0]?.id).toEqual(scoreEventId);
-
-      expect(await prisma.trace.count()).toBe(1);
-      expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
-
-      const dbScore = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-        },
-      });
-
-      expect(dbScore).toBeNull();
-    } finally {
-      await prisma.project.delete({ where: { id: otherProjectId } });
-    }
-  });
-
-  it("should error on Bearer Auth with a trace that does not exist", async () => {
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    const response = await makeAPICall(
-      "POST",
-      "/api/public/ingestion",
-      {
-        batch: [
-          {
-            id: scoreEventId,
-            type: "score-create",
-            timestamp: new Date().toISOString(),
-            body: {
-              id: scoreId,
-              name: scoreName,
-              value: scoreValue,
-              traceId: traceId,
-            },
-          },
-        ],
-      },
-      bearerAuth,
-    );
-
-    expect(response.status).toBe(207);
-    expect(response.body.successes.length).toBe(0);
-    expect(response.body.errors.length).toBe(1);
-    expect(response.body.errors[0]?.id).toBe(scoreEventId);
-
-    expect(await prisma.trace.count()).toBe(0);
-
-    const dbScore = await prisma.score.findUnique({
-      where: {
-        id: scoreId,
-      },
-    });
-
-    expect(dbScore).toBeNull();
-  });
-
   it("should error on wrong input", async () => {
     const traceId = "trace_id";
     const bearerAuth = "Bearer pk-lf-1234567890";

web/src/__tests__/model-match.servertest.ts

@@ -73,6 +73,15 @@ describe("model match", () => {
           input: "I am a prompt",
           output: "I am a completion",
         },
+        {
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "gpt-4o",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
       ],
     });
 
@@ -86,7 +95,7 @@ describe("model match", () => {
     expect(observation1?.completionTokens).toEqual(3000);
     expect(observation1?.totalTokens).toEqual(0);
 
-    expect(observations.length).toEqual(3);
+    expect(observations.length).toEqual(4);
     observations.forEach((observation) => {
       expect(observation.internalModel).toBeDefined();
       expect(observation.promptTokens).toBeGreaterThan(0);

web/src/__tests__/prompts.v2.servertest.ts

@@ -709,7 +709,7 @@ describe("/api/public/v2/prompts API Endpoint", () => {
         create: {
           id: otherProjectId,
           name: "demo-app",
-          members: {
+          projectMembers: {
             create: {
               role: "OWNER",
               userId: "user-test",
@@ -784,10 +784,17 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(promptMeta3.tags).toEqual(["tag-1"]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(3);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(3);
+
+      // Validate pagination backwards compatibility
+      // https://github.com/langfuse/langfuse/issues/2068
+      expect(body.pagination?.page).toBe(1);
+      expect(body.pagination?.limit).toBe(10);
+      expect(body.pagination?.totalPages).toBe(1);
+      expect(body.pagination?.totalItems).toBe(3);
     });
 
     it("should fetch a prompt meta list with name filter", async () => {
@@ -802,10 +809,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(1);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(1);
 
       // Test with a different name
       const response2 = await makeAPICall("GET", `${baseURI}?name=prompt-2`);
@@ -819,10 +826,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body2.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body2.pagination.page).toBe(1);
-      expect(body2.pagination.limit).toBe(10);
-      expect(body2.pagination.totalPages).toBe(1);
-      expect(body2.pagination.totalItems).toBe(1);
+      expect(body2.meta.page).toBe(1);
+      expect(body2.meta.limit).toBe(10);
+      expect(body2.meta.totalPages).toBe(1);
+      expect(body2.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if name does not exist
       const response3 = await makeAPICall(
@@ -846,10 +853,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body.data[0].tags).toEqual(["tag-1"]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(1);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if tag does not exist
       const response3 = await makeAPICall("GET", `${baseURI}?tag=non-existent`);
@@ -875,10 +882,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       ).toBe(true);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(3);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(3);
 
       // Test with a different label
       const response2 = await makeAPICall("GET", `${baseURI}?label=dev`);
@@ -892,10 +899,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body2.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body2.pagination.page).toBe(1);
-      expect(body2.pagination.limit).toBe(10);
-      expect(body2.pagination.totalPages).toBe(1);
-      expect(body2.pagination.totalItems).toBe(1);
+      expect(body2.meta.page).toBe(1);
+      expect(body2.meta.limit).toBe(10);
+      expect(body2.meta.totalPages).toBe(1);
+      expect(body2.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if label does not exist
       const response3 = await makeAPICall(
@@ -914,10 +921,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
     const body = response.body as unknown as PromptsMetaResponse;
 
     expect(body.data).toHaveLength(1);
-    expect(body.pagination.page).toBe(1);
-    expect(body.pagination.limit).toBe(1);
-    expect(body.pagination.totalPages).toBe(3);
-    expect(body.pagination.totalItems).toBe(3);
+    expect(body.meta.page).toBe(1);
+    expect(body.meta.limit).toBe(1);
+    expect(body.meta.totalPages).toBe(3);
+    expect(body.meta.totalItems).toBe(3);
   });
 });
 

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id  WHERE  t."project_id" = ',
+          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
           ";",
         ],
       } as const,

web/src/__tests__/scores.servertest.ts

@@ -60,6 +60,7 @@ describe("/api/public/scores API Endpoint", () => {
     expect(dbScore?.observationId).toBeNull();
     expect(dbScore?.comment).toBe("comment");
     expect(dbScore?.source).toBe("API");
+    expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
   });
 
   it("should create score for a trace with int", async () => {

web/src/__tests__/test-utils.ts

@@ -90,7 +90,7 @@ export const setupUserAndProject = async () => {
           },
         ],
       },
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,

web/src/__tests__/usage.servertest.ts

@@ -33,6 +33,7 @@ describe("Token Count Functions", () => {
       { model: "claude", tokenizer: "claude", tokens: 118 },
       { model: "claude-instant-1.2", tokenizer: "claude", tokens: 118 },
       { model: "gpt-3.5-turbo-1106", tokenizer: "openai", tokens: 114 },
+      { model: "gpt-4o-2024-05-13", tokenizer: "openai", tokens: 112 },
     ].forEach(({ model, tokens, tokenizer }) => {
       it(`should return token count ${tokens} for ${model}`, () => {
         const result = tokenCount({

web/src/components/LangfuseLogo.tsx

@@ -39,7 +39,7 @@ export const LangfuseLogo = ({
     {showEnvLabel && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && (
       <div
         className={cn(
-          "flex items-center gap-2 self-stretch rounded-md px-3 py-2 text-xs ring-1 lg:-mx-2",
+          "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
           env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
             ? "bg-blue-100 text-blue-500 ring-blue-500"
             : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
@@ -48,9 +48,9 @@ export const LangfuseLogo = ({
         )}
       >
         {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
-          <Check size={16} />
+          <Check size={16} className="hidden sm:block" />
         ) : (
-          <AlertTriangle size={16} />
+          <AlertTriangle size={16} className="hidden sm:block" />
         )}
         <span className="whitespace-nowrap">
           {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)

web/src/components/ModelParameters.tsx

@@ -1,8 +1,3 @@
-import {
-  supportedModels,
-  ModelProvider,
-  type UIModelParams,
-} from "@langfuse/shared";
 import { Input } from "@/src/components/ui/input";
 import {
   Select,
@@ -12,22 +7,31 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
+import { Switch } from "@/src/components/ui/switch";
+import { cn } from "@/src/utils/tailwind";
+import {
+  ModelProvider,
+  supportedModels,
+  type UIModelParams,
+} from "@langfuse/shared";
 
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
-  updateModelParam: <Key extends keyof UIModelParams>(
+  updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
-    value: UIModelParams[Key],
+    value: UIModelParams[Key]["value"],
   ) => void;
-  disabled?: boolean;
+  setModelParamEnabled?: (key: keyof UIModelParams, enabled: boolean) => void;
+  formDisabled?: boolean;
 };
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
   availableModels,
-  updateModelParam,
-  disabled,
+  updateModelParamValue,
+  setModelParamEnabled,
+  formDisabled = false,
 }) => {
   return (
     <div className="flex flex-col space-y-4">
@@ -36,61 +40,69 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
-          disabled={disabled}
-          value={modelParams.provider}
+          disabled={formDisabled}
+          value={modelParams.provider.value}
           options={
             availableModels
-              ? [...new Set(availableModels.map((m) => m.provider))]
+              ? [...new Set(availableModels.map((m) => m.provider.value))]
               : Object.values(ModelProvider)
           }
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSelect
           title="Model name"
           modelParamsKey="model"
-          disabled={disabled}
-          value={modelParams.model}
+          disabled={formDisabled}
+          value={modelParams.model.value}
           options={Object.values(
             availableModels
               ? availableModels
-                  .filter((m) => m.provider === modelParams.provider)
-                  .map((m) => m.model)
-              : supportedModels[modelParams.provider],
+                  .filter(
+                    (m) => m.provider.value === modelParams.provider.value,
+                  )
+                  .map((m) => m.model.value)
+              : supportedModels[modelParams.provider.value],
           )}
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Temperature"
           modelParamsKey="temperature"
-          disabled={disabled}
-          value={modelParams.temperature}
+          formDisabled={formDisabled}
+          enabled={modelParams.temperature.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.temperature.value}
           min={0}
-          max={modelParams.maxTemperature}
+          max={modelParams.maxTemperature.value}
           step={0.01}
-          tooltip="The sampling temperature. Higher values will make the output more random, while lower values like will make it more focused and deterministic."
-          updateModelParam={updateModelParam}
+          tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Output token limit"
           modelParamsKey="max_tokens"
-          disabled={disabled}
-          value={modelParams.max_tokens}
+          formDisabled={formDisabled}
+          enabled={modelParams.max_tokens.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.max_tokens.value}
           min={1}
           max={4096}
           step={1}
           tooltip="The maximum number of tokens that can be generated in the chat completion."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Top P"
           modelParamsKey="top_p"
-          disabled={disabled}
-          value={modelParams.top_p}
+          formDisabled={formDisabled}
+          enabled={modelParams.top_p.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.top_p.value}
           min={0}
           max={1}
           step={0.01}
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
       </div>
     </div>
@@ -102,7 +114,7 @@ type ModelParamsSelectProps = {
   modelParamsKey: keyof UIModelParams;
   value: string;
   options: string[];
-  updateModelParam: ModelParamsContext["updateModelParam"];
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
   disabled?: boolean;
 };
 const ModelParamsSelect = ({
@@ -115,7 +127,9 @@ const ModelParamsSelect = ({
 }: ModelParamsSelectProps) => {
   return (
     <div className="space-y-2">
-      <p className="text-xs font-semibold">{title}</p>
+      <p className={cn("text-xs font-semibold", disabled && "text-gray-400")}>
+        {title}
+      </p>
       <Select
         disabled={disabled}
         onValueChange={(value) =>
@@ -149,8 +163,10 @@ type ModelParamsSliderProps = {
   min: number;
   max: number;
   step: number;
-  updateModelParam: ModelParamsContext["updateModelParam"];
-  disabled?: boolean;
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
+  enabled?: boolean;
+  formDisabled?: boolean;
+  setModelParamEnabled?: ModelParamsContext["setModelParamEnabled"];
 };
 const ModelParamsSlider = ({
   title,
@@ -161,30 +177,51 @@ const ModelParamsSlider = ({
   max,
   step,
   updateModelParam,
-  disabled,
+  setModelParamEnabled,
+  enabled,
+  formDisabled,
 }: ModelParamsSliderProps) => {
   return (
     <div className="space-y-3" title={tooltip}>
       <div className="flex flex-row">
-        <p className="flex-1 text-xs font-semibold">{title}</p>
-        <Input
-          className="h-6 w-14 appearance-none px-2 text-right"
-          type="number"
-          disabled={disabled}
-          min={min}
-          max={max}
-          step={step}
-          value={value}
-          onChange={(event) => {
-            updateModelParam(
-              modelParamsKey,
-              Math.max(Math.min(parseFloat(event.target.value), max), min),
-            );
-          }}
-        />
+        <p
+          className={cn(
+            "flex-1 text-xs font-semibold",
+            (!enabled || formDisabled) && "text-gray-400",
+          )}
+        >
+          {title}
+        </p>
+        <div className="flex flex-row space-x-3">
+          <Input
+            className="h-6 w-14 appearance-none px-2 text-right"
+            type="number"
+            disabled={!enabled || formDisabled}
+            min={min}
+            max={max}
+            step={step}
+            value={value}
+            onChange={(event) => {
+              updateModelParam(
+                modelParamsKey,
+                Math.max(Math.min(parseFloat(event.target.value), max), min),
+              );
+            }}
+          />
+          {setModelParamEnabled ? (
+            <Switch
+              title={`Control sending the ${title} parameter`}
+              disabled={formDisabled}
+              checked={enabled}
+              onCheckedChange={(checked) => {
+                setModelParamEnabled(modelParamsKey, checked);
+              }}
+            />
+          ) : null}
+        </div>
       </div>
       <Slider
-        disabled={disabled}
+        disabled={!enabled || formDisabled}
         min={min}
         max={max}
         step={step}

web/src/components/deleteButton.tsx

@@ -7,10 +7,10 @@ import {
 } from "@/src/components/ui/popover";
 import { Button } from "@/src/components/ui/button";
 import { TrashIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type Scope } from "@/src/features/rbac/constants/roleAccessRights";
 import { api } from "@/src/utils/api";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DeleteButtonProps {
   itemId: string;
@@ -33,7 +33,7 @@ export function DeleteButton({
 }: DeleteButtonProps) {
   const [isDeleted, setIsDeleted] = useState(false);
   const router = useRouter();
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasAccess = useHasAccess({ projectId, scope: scope });
   const traceMutation = api.traces.deleteMany.useMutation({
@@ -63,6 +63,15 @@ export function DeleteButton({
         <Button
           variant={isTableAction ? "ghost" : "outline"}
           size={isTableAction ? "xs" : "icon"}
+          onClick={() =>
+            type === "trace"
+              ? capture("trace:delete_form_open", {
+                  source: isTableAction ? "table-single-row" : "trace detail",
+                })
+              : capture("datasets:delete_form_open", {
+                  source: "dataset",
+                })
+          }
         >
           <TrashIcon className="h-4 w-4" />
         </Button>
@@ -84,7 +93,7 @@ export function DeleteButton({
                   traceIds: [itemId],
                   projectId,
                 });
-                posthog.capture("trace:delete", {
+                capture("trace:delete", {
                   source: isTableAction ? "table-single-row" : "trace",
                 });
               }}
@@ -101,7 +110,7 @@ export function DeleteButton({
                   projectId,
                   datasetId: itemId,
                 });
-                posthog.capture("dataset:delete", {
+                capture("datasets:delete_dataset_button_click", {
                   source: isTableAction ? "table-single-row" : "dataset",
                 });
               }}

web/src/components/error-page.tsx

@@ -14,6 +14,7 @@ export const ErrorPage = ({
 }) => {
   const session = useSession();
   const router = useRouter();
+  const newTargetPath = router.asPath;
 
   return (
     <div className="flex h-screen flex-col items-center justify-center">
@@ -21,7 +22,15 @@ export const ErrorPage = ({
       <h1 className="mb-4 text-xl font-bold">{title}</h1>
       <p className="mb-8 text-center">{message}</p>
       {session.status === "unauthenticated" ? (
-        <Button onClick={() => void router.push("/")}>Sign In</Button>
+        <Button
+          onClick={() =>
+            void router.push(
+              `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+            )
+          }
+        >
+          Sign In
+        </Button>
       ) : null}
     </div>
   );

web/src/components/json-editor.tsx

@@ -1,6 +1,7 @@
 import CodeMirror, { EditorView } from "@uiw/react-codemirror";
 import { githubLight } from "@uiw/codemirror-theme-github";
 import { json } from "@codemirror/lang-json";
+import { cn } from "@/src/utils/tailwind";
 
 // todo: add json linting
 
@@ -9,11 +10,13 @@ export function JsonEditor({
   onChange,
   editable = true,
   lineWrapping = true,
+  className,
 }: {
   defaultValue: string;
   onChange?: (value: string) => void;
   editable?: boolean;
   lineWrapping?: boolean;
+  className?: string;
 }) {
   return (
     <CodeMirror
@@ -26,7 +29,7 @@ export function JsonEditor({
       extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
       defaultValue={defaultValue}
       onChange={onChange}
-      className="overflow-hidden rounded-md border"
+      className={cn("overflow-hidden rounded-md border", className)}
       editable={editable}
     />
   );

web/src/components/layouts/doc-popup.tsx

@@ -3,9 +3,9 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
-import { usePostHog } from "posthog-js/react";
 
 export type DocPopupProps = {
   description: React.ReactNode;
@@ -26,14 +26,17 @@ export default function DocPopup({
     md: "w-6 h-6",
     lg: "w-8 h-8",
   };
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   return (
     <HoverCard
       openDelay={200}
       onOpenChange={(open) => {
         if (open) {
-          posthog.capture("help-popup:opened", { href, description });
+          capture("help_popup:opened", {
+            hfref: href,
+            description: description,
+          });
         }
       }}
     >
@@ -44,6 +47,12 @@ export default function DocPopup({
             rel="noopener"
             target="_blank"
             className="inline-block whitespace-nowrap text-gray-500 sm:pl-0"
+            onClick={() => {
+              capture("help_popup:href_clicked", {
+                href: href,
+                description: description,
+              });
+            }}
           >
             {
               {

web/src/components/layouts/full-screen-page.tsx

@@ -0,0 +1,9 @@
+export const FullScreenPage: React.FC<React.PropsWithChildren<{}>> = ({
+  children,
+}) => {
+  return (
+    <div className="flex h-[calc(100dvh-6rem)] flex-col overflow-hidden lg:h-[calc(100dvh-2rem)]">
+      {children}
+    </div>
+  );
+};

web/src/components/layouts/layout.tsx

@@ -14,7 +14,6 @@ import {
   AvatarFallback,
   AvatarImage,
 } from "@/src/components/ui/avatar";
-import { api } from "@/src/utils/api";
 import { NewProjectButton } from "@/src/features/projects/components/NewProjectButton";
 import { FeedbackButtonWrapper } from "@/src/features/feedback/component/FeedbackButton";
 import { Button } from "@/src/components/ui/button";
@@ -31,6 +30,7 @@ import {
 import { ChevronDownIcon } from "@heroicons/react/20/solid";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { ProjectNavigation } from "@/src/components/projectNavigation";
+import DOMPurify from "dompurify";
 
 const userNavigation = [
   {
@@ -61,7 +61,7 @@ export default function Layout(props: PropsWithChildren) {
   useCheckNotification(NOTIFICATIONS, session.status === "authenticated");
 
   const enableExperimentalFeatures =
-    api.environment.enableExperimentalFeatures.useQuery().data ?? false;
+    session.data?.environment.enableExperimentalFeatures ?? false;
 
   const projectId = router.query.projectId as string | undefined;
 
@@ -147,7 +147,14 @@ export default function Layout(props: PropsWithChildren) {
     !publishablePaths.includes(router.pathname) &&
     !router.pathname.startsWith("/public/")
   ) {
-    void router.replace("/auth/sign-in");
+    const newTargetPath = router.asPath;
+    if (newTargetPath && newTargetPath !== "/") {
+      void router.replace(
+        `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+      );
+    } else {
+      void router.replace(`/auth/sign-in`);
+    }
     return <Spinner message="Redirecting" />;
   }
 
@@ -155,7 +162,13 @@ export default function Layout(props: PropsWithChildren) {
     session.status === "authenticated" &&
     unauthenticatedPaths.includes(router.pathname)
   ) {
-    void router.replace("/");
+    const targetPath = router.query.targetPath as string | undefined;
+
+    const sanitizedTargetPath = targetPath
+      ? DOMPurify.sanitize(targetPath)
+      : undefined;
+
+    void router.replace(sanitizedTargetPath ?? "/");
     return <Spinner message="Redirecting" />;
   }
 
@@ -240,7 +253,7 @@ export default function Layout(props: PropsWithChildren) {
                       >
                         <span className="sr-only">Close sidebar</span>
                         <XMarkIcon
-                          className="h-6 w-6 text-white"
+                          className="h-5 w-5 text-white"
                           aria-hidden="true"
                         />
                       </button>
@@ -260,7 +273,7 @@ export default function Layout(props: PropsWithChildren) {
                         <MainNavigation nav={navigation} />
                       </ul>
                       <div className="mb-2 flex flex-row place-content-between items-center">
-                        <div className="text-xs font-semibold leading-6 text-gray-400">
+                        <div className="text-xs font-semibold text-gray-400">
                           Project
                         </div>
                         <NewProjectButton size="xs" />
@@ -278,7 +291,7 @@ export default function Layout(props: PropsWithChildren) {
         </Transition.Root>
 
         {/* Static sidebar for desktop */}
-        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-60 lg:flex-col">
+        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-56 lg:flex-col">
           {/* Sidebar component, swap this element with another sidebar if you like */}
           <div className="flex h-screen grow flex-col border-r border-gray-200 bg-white pt-7">
             <LangfuseLogo
@@ -299,16 +312,16 @@ export default function Layout(props: PropsWithChildren) {
                   description="What do you think about this project? What can be improved?"
                   type="feedback"
                 >
-                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
+                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-1.5 text-sm font-semibold text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
                     <MessageSquarePlus
-                      className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                      className="h-5 w-5 shrink-0 text-gray-400 group-hover:text-indigo-600"
                       aria-hidden="true"
                     />
                     Feedback
                   </li>
                 </FeedbackButtonWrapper>
                 <div className="mb-2 flex flex-row place-content-between items-center">
-                  <div className="text-xs font-semibold leading-6 text-gray-400">
+                  <div className="text-xs font-semibold text-gray-400">
                     Project
                   </div>
                   <NewProjectButton size="xs" />
@@ -321,9 +334,9 @@ export default function Layout(props: PropsWithChildren) {
             </nav>
 
             <Menu as="div" className="relative">
-              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+              <Menu.Button className="flex w-full items-center gap-x-2 overflow-hidden p-1.5 py-3 pl-6 pr-8 text-sm font-semibold text-gray-900 hover:bg-gray-50">
                 <span className="sr-only">Open user menu</span>
-                <Avatar className="h-8 w-8">
+                <Avatar className="h-7 w-7">
                   <AvatarImage src={session.data?.user?.image ?? undefined} />
                   <AvatarFallback>
                     {session.data?.user?.name
@@ -335,7 +348,7 @@ export default function Layout(props: PropsWithChildren) {
                       : null}
                   </AvatarFallback>
                 </Avatar>
-                <span className="flex-shrink truncate text-sm font-semibold leading-6 text-gray-900">
+                <span className="flex-shrink truncate text-sm font-semibold text-gray-900">
                   {session.data?.user?.name}
                 </span>
                 <div className="flex-1" />
@@ -354,7 +367,7 @@ export default function Layout(props: PropsWithChildren) {
                 leaveTo="transform opacity-0 scale-95"
               >
                 <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                  <span className="mb-1 block border-b px-3 pb-2 text-sm text-gray-500">
                     {session.data?.user?.email}
                   </span>
                   {userNavigation.map((item) => (
@@ -364,7 +377,7 @@ export default function Layout(props: PropsWithChildren) {
                           onClick={() => void item.onClick()}
                           className={cn(
                             active ? "bg-gray-50" : "",
-                            "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                            "block cursor-pointer px-3 py-1 text-sm text-gray-900",
                           )}
                         >
                           {item.name}
@@ -385,7 +398,7 @@ export default function Layout(props: PropsWithChildren) {
             onClick={() => setSidebarOpen(true)}
           >
             <span className="sr-only">Open sidebar</span>
-            <Bars3Icon className="h-6 w-6" aria-hidden="true" />
+            <Bars3Icon className="h-5 w-5" aria-hidden="true" />
           </button>
           <LangfuseLogo
             version
@@ -393,7 +406,7 @@ export default function Layout(props: PropsWithChildren) {
             showEnvLabel={session.data?.user?.email?.endsWith("@langfuse.com")}
           />
           <Menu as="div" className="relative">
-            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold leading-6 text-gray-900">
+            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold text-gray-900">
               <span className="sr-only">Open user menu</span>
               <Avatar className="h-8 w-8">
                 <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -418,7 +431,7 @@ export default function Layout(props: PropsWithChildren) {
               leaveTo="transform opacity-0 scale-95"
             >
               <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm text-gray-500">
                   {session.data?.user?.email}
                 </span>
                 {userNavigation.map((item) => (
@@ -428,7 +441,7 @@ export default function Layout(props: PropsWithChildren) {
                         onClick={() => void item.onClick()}
                         className={cn(
                           active ? "bg-gray-50" : "",
-                          "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                          "block cursor-pointer px-3 py-1 text-sm text-gray-900",
                         )}
                       >
                         {item.name}
@@ -440,7 +453,7 @@ export default function Layout(props: PropsWithChildren) {
             </Transition>
           </Menu>
         </div>
-        <div className="lg:pl-60">
+        <div className="lg:pl-56">
           {env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           projectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING" ||
@@ -512,7 +525,7 @@ const MainNavigation: React.FC<{
                   item.current
                     ? "bg-gray-50 text-indigo-600"
                     : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
+                  "group flex gap-x-3 rounded-md p-1.5 text-sm font-semibold",
                 )}
                 onClick={onNavitemClick}
                 target={item.newTab ? "_blank" : undefined}
@@ -523,7 +536,7 @@ const MainNavigation: React.FC<{
                       item.current
                         ? "text-indigo-600"
                         : "text-gray-400 group-hover:text-indigo-600",
-                      "h-6 w-6 shrink-0",
+                      "h-5 w-5 shrink-0",
                     )}
                     aria-hidden="true"
                   />
@@ -532,7 +545,7 @@ const MainNavigation: React.FC<{
                 {item.label && (
                   <span
                     className={cn(
-                      "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                      "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                       item.current
                         ? "border-indigo-600 text-indigo-600"
                         : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
@@ -552,12 +565,12 @@ const MainNavigation: React.FC<{
                 {({ open }) => (
                   <>
                     <Disclosure.Button
-                      className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold leading-6 hover:bg-gray-50 hover:text-indigo-600"
+                      className="group flex w-full items-center gap-x-3 rounded-md p-1.5 text-left text-sm font-semibold hover:bg-gray-50 hover:text-indigo-600"
                       onClick={() => setIsOpen(!isOpen)}
                     >
                       {item.icon && (
                         <item.icon
-                          className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                          className="h-5 w-5 shrink-0 text-gray-400 group-hover:text-indigo-600"
                           aria-hidden="true"
                         />
                       )}
@@ -565,7 +578,7 @@ const MainNavigation: React.FC<{
                       {item.label && (
                         <span
                           className={cn(
-                            "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                            "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                             item.current
                               ? "border-indigo-600 text-indigo-600"
                               : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
@@ -582,7 +595,7 @@ const MainNavigation: React.FC<{
                         aria-hidden="true"
                       />
                     </Disclosure.Button>
-                    <Disclosure.Panel as="ul" className="mt-1 px-2">
+                    <Disclosure.Panel as="ul" className="mt-1 space-y-1 px-2">
                       {item.children?.map((subItem) => (
                         <li key={subItem.name}>
                           {/* 44px */}
@@ -592,7 +605,7 @@ const MainNavigation: React.FC<{
                               subItem.current
                                 ? "bg-gray-50 text-indigo-600"
                                 : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
+                              "ml-0.5 flex w-full items-center gap-x-3 rounded-md p-1.5 pl-7 pr-2 text-sm",
                             )}
                             target={subItem.newTab ? "_blank" : undefined}
                           >

web/src/components/layouts/routes.ts

@@ -75,13 +75,13 @@ export const ROUTES: Route[] = [
         name: "Configs",
         pathname: `/project/[projectId]/evals/configs`,
         cloudOnly: true,
-        rbacScope: "job:read",
+        rbacScope: "evalJob:read",
       },
       {
         name: "Log",
         pathname: `/project/[projectId]/evals/log`,
         cloudOnly: true,
-        rbacScope: "jobExecution:read",
+        rbacScope: "evalJobExecution:read",
       },
     ],
   },

web/src/components/projectNavigation.tsx

@@ -32,7 +32,7 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
         router.push(`/project/${value}`);
       }}
     >
-      <SelectTrigger className="text-gray-700 ring-transparent focus:ring-0 focus:ring-offset-0">
+      <SelectTrigger className="h-8 text-gray-700 ring-transparent focus:ring-0 focus:ring-offset-0">
         <SelectValue
           className="text-sm font-semibold text-gray-700"
           placeholder={currentProjectId}

web/src/components/publish-object-switch.tsx

@@ -1,8 +1,8 @@
 import { Switch } from "@/src/components/ui/switch";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { Link, LockIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useState } from "react";
 
 export const PublishTraceSwitch = (props: {
@@ -10,7 +10,7 @@ export const PublishTraceSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -30,7 +30,7 @@ export const PublishTraceSwitch = (props: {
           traceId: props.traceId,
           public: val,
         });
-        posthog.capture("trace_detail:publish_trace_button_click");
+        capture("trace_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}
@@ -43,7 +43,7 @@ export const PublishSessionSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -63,7 +63,7 @@ export const PublishSessionSwitch = (props: {
           sessionId: props.sessionId,
           public: val,
         });
-        posthog.capture("session_detail:publish_session_button_click");
+        capture("session_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}

web/src/components/session/index.tsx

@@ -13,13 +13,32 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
+import { useWindowVirtualizer } from "@tanstack/react-virtual";
+import { type RouterOutput } from "@/src/utils/types";
+import {
+  DataTableRowHeightSwitch,
+  type RowHeight,
+  useRowHeightLocalStorage,
+} from "@/src/components/table/data-table-row-height-switch";
+import { ScrollArea } from "@/src/components/ui/scroll-area";
+
+// do not use the usual table row heights here
+const rowHeightMapping: Record<RowHeight, number> = {
+  s: 200,
+  m: 350,
+  l: 700,
+};
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "single-session",
+    "m",
+  );
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -77,6 +96,11 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
+          <DataTableRowHeightSwitch
+            rowHeight={rowHeight}
+            setRowHeight={setRowHeight}
+            key="height"
+          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -97,36 +121,95 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
-        {session.data?.traces.map((trace) => (
-          <Card
-            className="border-border-gray-150 group grid gap-3 p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
-            key={trace.id}
-          >
-            <SessionIO traceId={trace.id} />
-            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
-              <Link
-                href={`/project/${projectId}/traces/${trace.id}`}
-                className="text-xs hover:underline"
-              >
-                Trace: {trace.name} ({trace.id})&nbsp;↗
-              </Link>
-              <div className="text-xs text-gray-500">
-                {trace.timestamp.toLocaleString()}
-              </div>
-              <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
-              <div className="flex flex-wrap content-start items-start gap-1">
-                <GroupedScoreBadges scores={trace.scores} />
-              </div>
-              <ManualScoreButton
-                projectId={projectId}
-                traceId={trace.id}
-                scores={trace.scores}
-                variant="badge"
-              />
-            </div>
-          </Card>
-        ))}
+      {session.data && (
+        <TraceCardList
+          session={session.data}
+          projectId={projectId}
+          rowHeight={rowHeightMapping[rowHeight]}
+        />
+      )}
+    </div>
+  );
+};
+
+const TraceCardList = ({
+  session,
+  projectId,
+  rowHeight,
+}: {
+  session: RouterOutput["sessions"]["byId"];
+  projectId: string;
+  rowHeight: number;
+}) => {
+  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
+
+  const virtualizer = useWindowVirtualizer({
+    count: session.traces.length,
+    estimateSize: () => rowHeight,
+    overscan: 5,
+    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
+    gap: 10,
+  });
+  useEffect(() => {
+    // re-measure when rowHeight changes to update the virtualizer
+    virtualizer.measure();
+  }, [rowHeight, virtualizer]);
+  return (
+    <div className="mt-5 border-t pt-5">
+      <div
+        ref={listVirtualizationRef}
+        style={{
+          height: `${virtualizer.getTotalSize()}px`,
+          position: "relative",
+        }}
+      >
+        {virtualizer
+          .getVirtualItems()
+          .map((virtualItem) => ({
+            virtualItem,
+            trace: session.traces[virtualItem.index],
+          }))
+          .map(({ virtualItem, trace }) => (
+            <Card
+              className="border-border-gray-150 group grid w-full gap-3 overflow-hidden p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
+              key={virtualItem.key}
+              data-index={virtualItem.index}
+              style={{
+                position: "absolute",
+                top: 0,
+                left: 0,
+                height: `${rowHeight}px`,
+                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
+              }}
+            >
+              <ScrollArea className="col-span-2 pr-3">
+                <SessionIO traceId={trace.id} />
+              </ScrollArea>
+              <ScrollArea>
+                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
+                  <Link
+                    href={`/project/${projectId}/traces/${trace.id}`}
+                    className="text-xs hover:underline"
+                  >
+                    Trace: {trace.name} ({trace.id})&nbsp;↗
+                  </Link>
+                  <div className="text-xs text-gray-500">
+                    {trace.timestamp.toLocaleString()}
+                  </div>
+                  <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
+                  <div className="flex flex-wrap content-start items-start gap-1">
+                    <GroupedScoreBadges scores={trace.scores} />
+                  </div>
+                  <ManualScoreButton
+                    projectId={projectId}
+                    traceId={trace.id}
+                    scores={trace.scores}
+                    variant="badge"
+                  />
+                </div>
+              </ScrollArea>
+            </Card>
+          ))}
       </div>
     </div>
   );
@@ -145,8 +228,9 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
+
   return (
-    <div className="col-span-2 flex flex-col gap-2 p-0">
+    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"

web/src/components/star-toggle.tsx

@@ -6,6 +6,7 @@ import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { cn } from "@/src/utils/tailwind";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function StarToggle({
   value,
@@ -53,7 +54,7 @@ export function StarTraceToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -109,13 +110,18 @@ export function StarTraceToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "traces",
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -133,7 +139,7 @@ export function StarTraceDetailsToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -180,13 +186,17 @@ export function StarTraceDetailsToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("trace_detail:bookmark_button_click", {
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -204,6 +214,7 @@ export function StarSessionToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
+  const capture = usePostHogClientCapture();
   const mutBookmarkSession = api.sessions.bookmark.useMutation({
     onSuccess: () => {
       void utils.sessions.invalidate();
@@ -216,13 +227,18 @@ export function StarSessionToggle({
       size={size}
       isLoading={mutBookmarkSession.isLoading}
       disabled={!hasAccess}
-      onClick={(value) =>
-        mutBookmarkSession.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "sessions",
+          id: sessionId,
+          value: value,
+        });
+        return mutBookmarkSession.mutateAsync({
           projectId,
           sessionId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }

web/src/components/table/data-table-column-visibility-filter.tsx

@@ -14,6 +14,7 @@ import {
 import { type VisibilityState } from "@tanstack/react-table";
 import { ChevronDown, Columns } from "lucide-react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTableColumnVisibilityFilterProps<TData, TValue> {
   columns: LangfuseColumnDef<TData, TValue>[];
@@ -27,14 +28,24 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
   setColumnVisibility,
 }: DataTableColumnVisibilityFilterProps<TData, TValue>) {
   const [isOpen, setIsOpen] = useState(false);
-
+  const capture = usePostHogClientCapture();
   const toggleColumn = useCallback(
     (columnId: string) => {
-      setColumnVisibility((old) => ({
-        ...old,
-        [columnId]: !old[columnId],
-      }));
+      setColumnVisibility((old) => {
+        const newColumnVisibility = {
+          ...old,
+          [columnId]: !old[columnId],
+        };
+        const selectedColumns = Object.keys(newColumnVisibility).filter(
+          (key) => newColumnVisibility[key],
+        );
+        capture("table:column_visibility_changed", {
+          selectedColumns: selectedColumns,
+        });
+        return newColumnVisibility;
+      });
     },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
     [setColumnVisibility],
   );
 

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -9,7 +9,6 @@ import {
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
-import { usePostHog } from "posthog-js/react";
 
 import {
   Dialog,
@@ -20,6 +19,7 @@ import {
   DialogTitle,
 } from "@/src/components/ui/dialog";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function TraceTableMultiSelectAction({
   selectedTraceIds,
@@ -32,7 +32,7 @@ export function TraceTableMultiSelectAction({
 }) {
   const utils = api.useUtils();
   const [open, setOpen] = useState(false);
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasDeleteAccess = useHasAccess({ projectId, scope: "traces:delete" });
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
@@ -55,6 +55,10 @@ export function TraceTableMultiSelectAction({
           <DropdownMenuItem
             disabled={!hasDeleteAccess}
             onClick={() => {
+              capture("trace:delete_form_open", {
+                count: selectedTraceIds.length,
+                source: "table-multi-select",
+              });
               setOpen(true);
             }}
           >
@@ -94,7 +98,7 @@ export function TraceTableMultiSelectAction({
                   .then(() => {
                     setOpen(false);
                   });
-                posthog.capture("trace:delete", {
+                capture("trace:delete_form_submit", {
                   count: selectedTraceIds.length,
                   source: "table-multi-select",
                 });

web/src/components/table/data-table-pagination.tsx

@@ -14,6 +14,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/src/components/ui/select";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTablePaginationProps<TData> {
   table: Table<TData>;
@@ -24,18 +25,27 @@ export function DataTablePagination<TData>({
   table,
   paginationOptions = [10, 20, 30, 40, 50],
 }: DataTablePaginationProps<TData>) {
+  const capture = usePostHogClientCapture();
   return (
-    <div className="mt-3 flex items-center justify-between overflow-x-auto px-2">
+    <div className="flex items-center justify-between">
       <div className="flex-1 text-sm text-muted-foreground">
         {/* {table.getFilteredSelectedRowModel().rows.length} of{" "}
         {table.getFilteredRowModel().rows.length} row(s) selected. */}
       </div>
-      <div className="flex items-center space-x-6 lg:space-x-8">
+      <div className="flex flex-wrap items-center space-x-6 lg:space-x-8">
         <div className="flex items-center space-x-2">
-          <p className="whitespace-nowrap text-sm font-medium">Rows per page</p>
+          <p className="whitespace-nowrap text-sm font-medium md:hidden">
+            Rows
+          </p>
+          <p className="hidden whitespace-nowrap text-sm font-medium md:block">
+            Rows per page
+          </p>
           <Select
             value={`${table.getState().pagination.pageSize}`}
             onValueChange={(value) => {
+              capture("table:pagination_page_size_select", {
+                pageSize: value,
+              });
               table.setPageSize(Number(value));
             }}
           >
@@ -51,7 +61,7 @@ export function DataTablePagination<TData>({
             </SelectContent>
           </Select>
         </div>
-        <div className="flex w-[100px] items-center justify-center whitespace-nowrap text-sm font-medium">
+        <div className="flex items-center justify-center whitespace-nowrap text-sm font-medium">
           Page {table.getState().pagination.pageIndex + 1} of{" "}
           {table.getPageCount()}
         </div>
@@ -59,7 +69,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(0)}
+            onClick={() => {
+              table.setPageIndex(0);
+              capture("table:pagination_button_click", {
+                type: "firstPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to first page</span>
@@ -68,7 +83,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.previousPage()}
+            onClick={() => {
+              table.previousPage();
+              capture("table:pagination_button_click", {
+                type: "previousPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to previous page</span>
@@ -77,7 +97,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.nextPage()}
+            onClick={() => {
+              table.nextPage();
+              capture("table:pagination_button_click", {
+                type: "nextPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to next page</span>
@@ -86,7 +111,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(table.getPageCount() - 1)}
+            onClick={() => {
+              table.setPageIndex(table.getPageCount() - 1);
+              capture("table:pagination_button_click", {
+                type: "lastPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to last page</span>

web/src/components/table/data-table-row-height-switch.tsx

@@ -1,5 +1,6 @@
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import {
   MdDensityLarge,
   MdDensityMedium,
@@ -35,25 +36,33 @@ export const DataTableRowHeightSwitch = ({
 }: {
   rowHeight: RowHeight;
   setRowHeight: (e: RowHeight) => void;
-}) => (
-  <Tabs
-    //defaultValue={height}
-    value={rowHeight}
-    onValueChange={(e) => setRowHeight(e as any)}
-    key="height"
-  >
-    <TabsList className="gap-1 border bg-transparent px-2">
-      {heightOptions.map(({ id, label, icon }) => (
-        <TabsTrigger
-          key={id}
-          value={id}
-          className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
-        >
-          <span role="img" aria-label={`${label} size`}>
-            {icon}
-          </span>
-        </TabsTrigger>
-      ))}
-    </TabsList>
-  </Tabs>
-);
+}) => {
+  const capture = usePostHogClientCapture();
+  return (
+    <Tabs
+      //defaultValue={height}
+      value={rowHeight}
+      onValueChange={(e) => {
+        capture("table:row_height_switch_select", {
+          rowHeight: e,
+        });
+        setRowHeight(e as any);
+      }}
+      key="height"
+    >
+      <TabsList className="gap-1 border bg-transparent px-2">
+        {heightOptions.map(({ id, label, icon }) => (
+          <TabsTrigger
+            key={id}
+            value={id}
+            className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
+          >
+            <span role="img" aria-label={`${label} size`}>
+              {icon}
+            </span>
+          </TabsTrigger>
+        ))}
+      </TabsList>
+    </Tabs>
+  );
+};

web/src/components/table/data-table-toolbar.tsx

@@ -12,6 +12,7 @@ import {
   type RowHeight,
 } from "@/src/components/table/data-table-row-height-switch";
 import { Search } from "lucide-react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface SearchConfig {
   placeholder: string;
@@ -47,9 +48,10 @@ export function DataTableToolbar<TData, TValue>({
   const [searchString, setSearchString] = useState(
     searchConfig?.currentQuery ?? "",
   );
+  const capture = usePostHogClientCapture();
 
   return (
-    <div className="my-2 flex flex-1 flex-wrap items-center gap-2">
+    <div className="my-2 flex flex-wrap items-center gap-2 @container">
       {searchConfig && (
         <div className="flex max-w-md items-center">
           <Input
@@ -59,14 +61,18 @@ export function DataTableToolbar<TData, TValue>({
             onChange={(event) => setSearchString(event.currentTarget.value)}
             onKeyDown={(event) => {
               if (event.key === "Enter") {
+                capture("table:search_submit");
                 searchConfig.updateQuery(searchString);
               }
             }}
-            className="h-10 w-[200px] rounded-r-none lg:w-[250] 2xl:w-[350px]"
+            className="h-10 w-[150px] rounded-r-none @6xl:w-[250px]"
           />
           <Button
             variant="outline"
-            onClick={() => searchConfig.updateQuery(searchString)}
+            onClick={() => {
+              capture("table:search_submit");
+              searchConfig.updateQuery(searchString);
+            }}
             className="rounded-l-none border-l-0 p-3"
           >
             <Search className="h-4 w-4" />
@@ -80,7 +86,7 @@ export function DataTableToolbar<TData, TValue>({
           onChange={setFilterState}
         />
       )}
-      <div className="flex flex-row flex-wrap gap-2 lg:ml-auto">
+      <div className="flex flex-row flex-wrap gap-2 pr-0.5 @6xl:ml-auto">
         {!!columnVisibility && !!setColumnVisibility && (
           <DataTableColumnVisibilityFilter
             columns={columns}

web/src/components/table/data-table.tsx

@@ -12,11 +12,13 @@ import {
   Table,
   TableBody,
   TableCell,
+  TableFooter,
   TableHead,
   TableHeader,
   TableRow,
 } from "@/src/components/ui/table";
 import { type OrderByState } from "@/src/features/orderBy/types";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { cn } from "@/src/utils/tailwind";
 import {
   flexRender,
@@ -48,6 +50,7 @@ interface DataTableProps<TData, TValue> {
   setOrderBy?: (s: OrderByState) => void;
   help?: { description: string; href: string };
   rowHeight?: RowHeight;
+  className?: string;
 }
 
 export interface AsyncTableData<T> {
@@ -72,6 +75,7 @@ export function DataTable<TData extends object, TValue>({
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight);
+  const capture = usePostHogClientCapture();
 
   const table = useReactTable({
     data: data.data ?? [],
@@ -102,8 +106,8 @@ export function DataTable<TData extends object, TValue>({
 
   return (
     <>
-      <div className="space-y-4">
-        <div className="rounded-md border">
+      <div className="flex w-full max-w-full flex-1 flex-col gap-1 overflow-auto">
+        <div className="w-full overflow-auto rounded-md border">
           <Table>
             <TableHeader>
               {table.getHeaderGroups().map((headerGroup) => (
@@ -129,14 +133,26 @@ export function DataTable<TData extends object, TValue>({
 
                           if (orderBy?.column === columnDef.id) {
                             if (orderBy.order === "DESC") {
+                              capture("table:column_sorting_header_click", {
+                                column: columnDef.id,
+                                order: "ASC",
+                              });
                               setOrderBy({
                                 column: columnDef.id,
                                 order: "ASC",
                               });
                             } else {
+                              capture("table:column_sorting_header_click", {
+                                column: columnDef.id,
+                                order: "Disabled",
+                              });
                               setOrderBy(null);
                             }
                           } else {
+                            capture("table:column_sorting_header_click", {
+                              column: columnDef.id,
+                              order: "DESC",
+                            });
                             setOrderBy({
                               column: columnDef.id,
                               order: "DESC",
@@ -176,10 +192,10 @@ export function DataTable<TData extends object, TValue>({
             </TableHeader>
             <TableBody>
               {data.isLoading || !data.data ? (
-                <TableRow>
+                <TableRow className="h-svh">
                   <TableCell
                     colSpan={columns.length}
-                    className="h-24 text-center"
+                    className="content-start border-b text-center"
                   >
                     Loading...
                   </TableCell>
@@ -190,7 +206,7 @@ export function DataTable<TData extends object, TValue>({
                     {row.getVisibleCells().map((cell) => (
                       <TableCell
                         key={cell.id}
-                        className="overflow-hidden whitespace-nowrap px-2 py-1 text-xs first:pl-2"
+                        className="overflow-hidden whitespace-nowrap border-b px-2 py-1 text-xs first:pl-2"
                       >
                         <div className={cn("flex items-center", rowheighttw)}>
                           {flexRender(
@@ -224,12 +240,15 @@ export function DataTable<TData extends object, TValue>({
             </TableBody>
           </Table>
         </div>
+        <div className="grow"></div>
       </div>
       {pagination !== undefined ? (
-        <DataTablePagination
-          table={table}
-          paginationOptions={pagination.options}
-        />
+        <div className="sticky bottom-0 z-10 flex w-full justify-end bg-white font-medium">
+          <DataTablePagination
+            table={table}
+            paginationOptions={pagination.options}
+          />
+        </div>
       ) : null}
     </>
   );

web/src/components/table/use-cases/generations.tsx

@@ -13,7 +13,6 @@ import {
 } from "@/src/components/ui/dropdown-menu";
 import { Button } from "@/src/components/ui/button";
 import { ChevronDownIcon, Loader } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import {
   NumberParam,
   StringParam,
@@ -29,7 +28,12 @@ import {
 } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { type Prisma, type ObservationLevel } from "@langfuse/shared";
+import {
+  type Prisma,
+  type ObservationLevel,
+  type FilterState,
+  type ObservationOptions,
+} from "@langfuse/shared";
 import { cn } from "@/src/utils/tailwind";
 import { LevelColors } from "@/src/components/level-colors";
 import { usdFormatter } from "@/src/utils/numbers";
@@ -43,6 +47,8 @@ import type Decimal from "decimal.js";
 import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { useSession } from "next-auth/react";
 
 export type GenerationsTableRow = {
   id: string;
@@ -53,6 +59,7 @@ export type GenerationsTableRow = {
   endTime?: string;
   completionStartTime?: Date;
   latency?: number;
+  timeToFirstToken?: number;
   name?: string;
   model?: string;
   // i/o not set explicitly, but fetched from the server from the cell
@@ -76,10 +83,19 @@ export type GenerationsTableRow = {
 
 export type GenerationsTableProps = {
   projectId: string;
+  promptName?: string;
+  promptVersion?: number;
+  omittedFilter?: string[];
 };
 
-export default function GenerationsTable({ projectId }: GenerationsTableProps) {
-  const posthog = usePostHog();
+export default function GenerationsTable({
+  projectId,
+  promptName,
+  promptVersion,
+  omittedFilter = [],
+}: GenerationsTableProps) {
+  const session = useSession();
+  const capture = usePostHogClientCapture();
   const [isExporting, setIsExporting] = useState(false);
   const [searchQuery, setSearchQuery] = useQueryParam(
     "search",
@@ -96,13 +112,15 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
     "s",
   );
 
-  const [filterState, setFilterState] = useQueryFilterState(
+  const [inputFilterState, setInputFilterState] = useQueryFilterState(
     [
       {
         column: "Start Time",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-14),
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+        ),
       },
     ],
     "generations",
@@ -113,6 +131,33 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
     order: "DESC",
   });
 
+  const promptNameFilter: FilterState = promptName
+    ? [
+        {
+          column: "Prompt Name",
+          type: "string",
+          operator: "=",
+          value: promptName,
+        },
+      ]
+    : [];
+
+  const promptVersionFilter: FilterState = promptVersion
+    ? [
+        {
+          column: "Prompt Version",
+          type: "number",
+          operator: "=",
+          value: promptVersion,
+        },
+      ]
+    : [];
+
+  const filterState = inputFilterState.concat([
+    ...promptNameFilter,
+    ...promptVersionFilter,
+  ]);
+
   const generations = api.generations.all.useQuery({
     page: paginationState.pageIndex,
     limit: paginationState.pageSize,
@@ -137,11 +182,19 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
     },
   );
 
+  const transformFilterOptions = (
+    filterOptions: ObservationOptions | undefined,
+  ) => {
+    return observationsTableColsWithOptions(filterOptions).filter(
+      (col) => !omittedFilter?.includes(col.name),
+    );
+  };
+
   const handleExport = async (fileFormat: ExportFileFormats) => {
     if (isExporting) return;
 
     setIsExporting(true);
-    posthog.capture("generations:export", { file_format: fileFormat });
+    capture("generations:export", { file_format: fileFormat });
     if (fileFormat === "OPENAI-JSONL")
       alert(
         "When exporting in OpenAI-JSONL, only generations that exactly match the `ChatML` format will be exported. For any questions, reach out to support.",
@@ -258,21 +311,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
       enableHiding: true,
       enableSorting: true,
       cell: ({ row }) => {
-        const startTime: Date = row.getValue("startTime");
-        const completionStartTime: Date | undefined =
+        const timeToFirstToken: number | undefined =
           row.getValue("timeToFirstToken");
 
-        if (!completionStartTime) {
-          return undefined;
-        }
-
-        const latencyInSeconds =
-          intervalInSeconds(startTime, completionStartTime) || "-";
         return (
           <span>
-            {typeof latencyInSeconds === "number"
-              ? formatIntervalSeconds(latencyInSeconds)
-              : latencyInSeconds}
+            {timeToFirstToken ? formatIntervalSeconds(timeToFirstToken) : "-"}
           </span>
         );
       },
@@ -571,7 +615,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
           traceName: generation.traceName ?? "",
           startTime: generation.startTime,
           endTime: generation.endTime?.toLocaleString() ?? undefined,
-          timeToFirstToken: generation.completionStartTime ?? undefined,
+          timeToFirstToken: generation.timeToFirstToken ?? undefined,
           latency: generation.latency ?? undefined,
           totalCost: generation.calculatedTotalCost ?? undefined,
           inputCost: generation.calculatedInputCost ?? undefined,
@@ -596,14 +640,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
     : [];
 
   return (
-    <div>
+    <>
       <DataTableToolbar
         columns={columns}
-        filterColumnDefinition={observationsTableColsWithOptions(
-          filterOptions.data,
-        )}
-        filterState={filterState}
-        setFilterState={setFilterState}
+        filterColumnDefinition={transformFilterOptions(filterOptions.data)}
+        filterState={inputFilterState}
+        setFilterState={setInputFilterState}
         searchConfig={{
           placeholder: "Search by id, name, traceName, model",
           updateQuery: setSearchQuery,
@@ -617,9 +659,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
           <DropdownMenu>
             <DropdownMenuTrigger asChild>
               <Button variant="outline" className="ml-auto whitespace-nowrap">
-                {filterState.length > 0 || searchQuery
-                  ? "Export selection"
-                  : "Export all"}{" "}
+                <span className="hidden @6xl:inline">
+                  {filterState.length > 0 || searchQuery
+                    ? "Export selection"
+                    : "Export all"}{" "}
+                </span>
+                <span className="@6xl:hidden">Export</span>
                 {isExporting ? (
                   <Loader className="ml-2 h-4 w-4 animate-spin" />
                 ) : (
@@ -669,7 +714,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
         onColumnVisibilityChange={setColumnVisibilityState}
         rowHeight={rowHeight}
       />
-    </div>
+    </>
   );
 }
 

web/src/components/table/use-cases/models.tsx

@@ -2,6 +2,7 @@ import { DataTable } from "@/src/components/table/data-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import { Button } from "@/src/components/ui/button";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
@@ -47,7 +48,6 @@ export default function ModelTable({ projectId }: { projectId: string }) {
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
   });
-
   const models = api.models.all.useQuery({
     page: paginationState.pageIndex,
     limit: paginationState.pageSize,
@@ -256,33 +256,31 @@ export default function ModelTable({ projectId }: { projectId: string }) {
   };
 
   return (
-    <div>
-      <DataTable
-        columns={columns}
-        data={
-          models.isLoading
-            ? { isLoading: true, isError: false }
-            : models.isError
-              ? {
-                  isLoading: false,
-                  isError: true,
-                  error: models.error.message,
-                }
-              : {
-                  isLoading: false,
-                  isError: false,
-                  data: models.data.models.map((t) => convertToTableRow(t)),
-                }
-        }
-        pagination={{
-          pageCount: Math.ceil(totalCount / paginationState.pageSize),
-          onChange: setPaginationState,
-          state: paginationState,
-        }}
-        columnVisibility={columnVisibility}
-        onColumnVisibilityChange={setColumnVisibility}
-      />
-    </div>
+    <DataTable
+      columns={columns}
+      data={
+        models.isLoading
+          ? { isLoading: true, isError: false }
+          : models.isError
+            ? {
+                isLoading: false,
+                isError: true,
+                error: models.error.message,
+              }
+            : {
+                isLoading: false,
+                isError: false,
+                data: models.data.models.map((t) => convertToTableRow(t)),
+              }
+      }
+      pagination={{
+        pageCount: Math.ceil(totalCount / paginationState.pageSize),
+        onChange: setPaginationState,
+        state: paginationState,
+      }}
+      columnVisibility={columnVisibility}
+      onColumnVisibilityChange={setColumnVisibility}
+    />
   );
 }
 
@@ -294,6 +292,7 @@ const DeleteModelButton = ({
   projectId: string;
 }) => {
   const utils = api.useUtils();
+  const capture = usePostHogClientCapture();
   const mut = api.models.delete.useMutation({
     onSuccess: () => {
       void utils.models.invalidate();
@@ -314,14 +313,20 @@ const DeleteModelButton = ({
       size="xs"
       variant="ghost"
       onClick={() => {
-        mut
-          .mutateAsync({
-            projectId,
-            modelId,
-          })
-          .catch((error) => {
-            console.error(error);
-          });
+        const confirmDelete = window.confirm(
+          "Are you sure you want to delete this model?",
+        );
+        if (confirmDelete) {
+          capture("models:delete_button_click");
+          mut
+            .mutateAsync({
+              projectId,
+              modelId,
+            })
+            .catch((error) => {
+              console.error(error);
+            });
+        }
       }}
     >
       <Trash size={14} />

web/src/components/table/use-cases/scores.tsx

@@ -255,7 +255,7 @@ export default function ScoresTable({
   };
 
   return (
-    <div>
+    <>
       <DataTableToolbar
         columns={columns}
         filterColumnDefinition={transformFilterOptions(filterOptions.data)}
@@ -294,6 +294,6 @@ export default function ScoresTable({
         onColumnVisibilityChange={setColumnVisibility}
         rowHeight={rowHeight}
       />
-    </div>
+    </>
   );
 }

web/src/components/table/use-cases/sessions.tsx

@@ -17,6 +17,7 @@ import { type RouterOutput } from "@/src/utils/types";
 import type Decimal from "decimal.js";
 import { useEffect } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
+import { useSession } from "next-auth/react";
 
 export type SessionTableRow = {
   id: string;
@@ -45,6 +46,7 @@ export default function SessionsTable({
   omittedFilter = [],
 }: SessionTableProps) {
   const { setDetailPageList } = useDetailPageLists();
+  const session = useSession();
 
   const [userFilterState, setUserFilterState] = useQueryFilterState(
     [
@@ -52,7 +54,9 @@ export default function SessionsTable({
         column: "Created At",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-14),
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+        ),
       },
     ],
     "sessions",
@@ -331,7 +335,7 @@ export default function SessionsTable({
     useColumnVisibility<SessionTableRow>("sessionsColumnVisibility", columns);
 
   return (
-    <div>
+    <>
       <DataTableToolbar
         filterColumnDefinition={transformFilterOptions()}
         filterState={userFilterState}
@@ -372,6 +376,6 @@ export default function SessionsTable({
           href: "https://langfuse.com/docs/tracing-features/sessions",
         }}
       />
-    </div>
+    </>
   );
 }

web/src/components/table/use-cases/traces.tsx

@@ -38,6 +38,7 @@ import {
 } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import { useSession } from "next-auth/react";
 
 export type TracesTableRow = {
   bookmarked: boolean;
@@ -47,6 +48,7 @@ export type TracesTableRow = {
   userId: string;
   metadata?: string;
   level: ObservationLevel;
+  observationCount: number;
   latency?: number;
   release?: string;
   version?: string;
@@ -80,6 +82,7 @@ export default function TracesTable({
   omittedFilter = [],
 }: TracesTableProps) {
   const utils = api.useUtils();
+  const session = useSession();
   const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
   const { setDetailPageList } = useDetailPageLists();
   const [searchQuery, setSearchQuery] = useQueryParam(
@@ -92,7 +95,9 @@ export default function TracesTable({
         column: "Timestamp",
         type: "datetime",
         operator: ">",
-        value: utcDateOffsetByDays(-14),
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+        ),
       },
     ],
     "traces",
@@ -173,6 +178,7 @@ export default function TracesTable({
       timestamp: trace.timestamp.toLocaleString(),
       name: trace.name ?? "",
       level: trace.level,
+      observationCount: trace.observationCount,
       metadata: JSON.stringify(trace.metadata),
       release: trace.release ?? undefined,
       version: trace.version ?? undefined,
@@ -528,6 +534,13 @@ export default function TracesTable({
       defaultHidden: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "observationCount",
+      id: "observationCount",
+      header: "Observation Count",
+      enableHiding: true,
+      defaultHidden: true,
+    },
     {
       accessorKey: "version",
       id: "version",
@@ -586,7 +599,7 @@ export default function TracesTable({
     useColumnVisibility<TracesTableRow>("tracesColumnVisibility", columns);
 
   return (
-    <div>
+    <>
       <DataTableToolbar
         columns={columns}
         filterColumnDefinition={transformFilterOptions(traceFilterOptions.data)}
@@ -648,7 +661,7 @@ export default function TracesTable({
         onColumnVisibilityChange={setColumnVisibility}
         rowHeight={rowHeight}
       />
-    </div>
+    </>
   );
 }
 

web/src/components/trace/IOPreview.tsx

@@ -6,6 +6,7 @@ import { useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { Fragment } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const IOPreview: React.FC<{
   input?: unknown;
@@ -14,7 +15,7 @@ export const IOPreview: React.FC<{
   hideIfNull?: boolean;
 }> = ({ isLoading = false, hideIfNull = false, ...props }) => {
   const [currentView, setCurrentView] = useState<"pretty" | "json">("pretty");
-
+  const capture = usePostHogClientCapture();
   const input = deepParseJson(props.input);
   const output = deepParseJson(props.output);
 
@@ -54,7 +55,9 @@ export const IOPreview: React.FC<{
       }
     }
   }
-  const outChatMlMessage = ChatMlMessageSchema.safeParse(output);
+  const outChatMlArray = ChatMlArraySchema.safeParse(
+    Array.isArray(output) ? output : [output],
+  );
 
   // Pretty view available
   const isPrettyViewAvailable = inChatMlArray.success;
@@ -65,7 +68,10 @@ export const IOPreview: React.FC<{
       {isPrettyViewAvailable ? (
         <Tabs
           value={currentView}
-          onValueChange={(v) => setCurrentView(v as "pretty" | "json")}
+          onValueChange={(v) => {
+            setCurrentView(v as "pretty" | "json"),
+              capture("trace_detail:io_mode_switch", { view: v });
+          }}
         >
           <TabsList>
             <TabsTrigger value="pretty">Pretty ✨</TabsTrigger>
@@ -75,17 +81,20 @@ export const IOPreview: React.FC<{
       ) : null}
       {isPrettyViewAvailable && currentView === "pretty" ? (
         <OpenAiMessageView
-          messages={inChatMlArray.data.concat(
-            outChatMlMessage.success
-              ? {
-                  ...outChatMlMessage.data,
-                  role: outChatMlMessage.data.role ?? "assistant",
-                }
-              : ChatMlMessageSchema.parse({
-                  role: "assistant",
-                  content: outputClean ? JSON.stringify(outputClean) : null,
-                }),
-          )}
+          messages={[
+            ...inChatMlArray.data,
+            ...(outChatMlArray.success
+              ? outChatMlArray.data.map((m) => ({
+                  ...m,
+                  role: m.role ?? "assistant",
+                }))
+              : [
+                  ChatMlMessageSchema.parse({
+                    role: "assistant",
+                    content: outputClean ? JSON.stringify(outputClean) : null,
+                  }),
+                ]),
+          ]}
         />
       ) : null}
       {currentView === "json" || !isPrettyViewAvailable ? (
@@ -119,12 +128,11 @@ const ChatMlMessageSchema = z
       .optional(),
     name: z.string().optional(),
     content: z
-      .union([z.record(z.any()), z.record(z.any()).array(), z.string()])
+      .union([z.record(z.any()), z.string(), z.array(z.any())])
       .nullish(),
     additional_kwargs: z.record(z.any()).optional(),
   })
   .passthrough()
-
   .refine((value) => value.content !== null || value.role !== undefined)
   .transform(({ additional_kwargs, ...other }) => ({
     ...other,

web/src/components/trace/ObservationPreview.tsx

@@ -67,14 +67,10 @@ export const ObservationPreview = (props: {
                 projectId={preloadedObservation.projectId}
               />
             ) : undefined}
-            {preloadedObservation.completionStartTime ? (
+            {preloadedObservation.timeToFirstToken ? (
               <Badge variant="outline">
                 Time to first token:{" "}
-                {formatIntervalSeconds(
-                  (preloadedObservation.completionStartTime.getTime() -
-                    preloadedObservation.startTime.getTime()) /
-                    1000,
-                )}
+                {formatIntervalSeconds(preloadedObservation.timeToFirstToken)}
               </Badge>
             ) : null}
             {preloadedObservation.endTime ? (
@@ -155,11 +151,11 @@ export const ObservationPreview = (props: {
           />
         ) : null}
 
-        {preloadedObservation.metadata ? (
+        {observationWithInputAndOutput.data?.metadata ? (
           <JSONView
-            key={preloadedObservation.id + "-metadata"}
+            key={observationWithInputAndOutput.data.id + "-metadata"}
             title="Metadata"
-            json={preloadedObservation.metadata}
+            json={observationWithInputAndOutput.data.metadata}
           />
         ) : null}
 

web/src/components/trace/ObservationTree.tsx

@@ -9,6 +9,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { MinusCircle, MinusIcon, PlusCircleIcon, PlusIcon } from "lucide-react";
 import { Toggle } from "@/src/components/ui/toggle";
 import { Button } from "@/src/components/ui/button";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const ObservationTree = (props: {
   observations: ObservationReturnType[];
@@ -120,126 +121,138 @@ const ObservationTreeNode = (props: {
   setCurrentObservationId: (id: string | undefined) => void;
   showMetrics?: boolean;
   showScores?: boolean;
-}) => (
-  <>
-    {props.observations
-      .sort((a, b) => a.startTime.getTime() - b.startTime.getTime())
-      .map((observation) => {
-        const collapsed = props.collapsedObservations.includes(observation.id);
-
-        return (
-          <Fragment key={observation.id}>
-            <div className="flex">
-              {Array.from({ length: props.indentationLevel }, (_, i) => (
-                <div className="mx-2 border-r" key={i} />
-              ))}
-              <div
-                className={cn(
-                  "group my-0.5 flex flex-1 cursor-pointer flex-col gap-1 rounded-sm p-1",
-                  props.currentObservationId === observation.id
-                    ? "bg-gray-100"
-                    : "hover:bg-gray-50",
-                )}
-                onClick={() => props.setCurrentObservationId(observation.id)}
-              >
-                <div className="flex gap-2">
-                  <ColorCodedObservationType
-                    observationType={observation.type}
-                  />
-                  <span className="flex-1 break-all text-sm">
-                    {observation.name}
-                  </span>
-                  {observation.children.length === 0 ? null : (
-                    <Toggle
-                      onClick={(ev) => (
-                        ev.stopPropagation(),
-                        props.toggleCollapsedObservation(observation.id)
-                      )}
-                      variant="default"
-                      pressed={collapsed}
-                      size="xs"
-                      className="w-7"
-                      title={
-                        collapsed ? "Expand children" : "Collapse children"
-                      }
-                    >
-                      {collapsed ? (
-                        <PlusIcon className="h-4 w-4" />
-                      ) : (
-                        <MinusIcon className="h-4 w-4" />
-                      )}
-                    </Toggle>
+}) => {
+  const capture = usePostHogClientCapture();
+  return (
+    <>
+      {props.observations
+        .sort((a, b) => a.startTime.getTime() - b.startTime.getTime())
+        .map((observation) => {
+          const collapsed = props.collapsedObservations.includes(
+            observation.id,
+          );
+          return (
+            <Fragment key={observation.id}>
+              <div className="flex">
+                {Array.from({ length: props.indentationLevel }, (_, i) => (
+                  <div className="mx-2 border-r" key={i} />
+                ))}
+                <div
+                  className={cn(
+                    "group my-0.5 flex flex-1 cursor-pointer flex-col gap-1 rounded-sm p-1",
+                    props.currentObservationId === observation.id
+                      ? "bg-gray-100"
+                      : "hover:bg-gray-50",
                   )}
-                </div>
-                {props.showMetrics &&
-                  (observation.promptTokens ||
-                    observation.completionTokens ||
-                    observation.totalTokens ||
-                    observation.endTime) && (
-                    <div className="flex gap-2">
-                      {observation.endTime ? (
-                        <span className="text-xs text-gray-500">
-                          {formatIntervalSeconds(
-                            (observation.endTime.getTime() -
-                              observation.startTime.getTime()) /
-                              1000,
-                          )}
-                        </span>
-                      ) : null}
-                      {observation.promptTokens ||
-                      observation.completionTokens ||
-                      observation.totalTokens ? (
-                        <span className="text-xs text-gray-500">
-                          {observation.promptTokens} →{" "}
-                          {observation.completionTokens} (∑{" "}
-                          {observation.totalTokens})
-                        </span>
-                      ) : null}
-                    </div>
-                  )}
-                {observation.level !== "DEFAULT" ? (
-                  <div className="flex">
-                    <span
-                      className={cn(
-                        "rounded-sm p-0.5 text-xs",
-                        LevelColors[observation.level].bg,
-                        LevelColors[observation.level].text,
-                      )}
-                    >
-                      {observation.level}
-                    </span>
-                  </div>
-                ) : null}
-                {props.showScores &&
-                props.scores.find((s) => s.observationId === observation.id) ? (
-                  <div className="flex flex-wrap gap-1">
-                    <GroupedScoreBadges
-                      scores={props.scores.filter(
-                        (s) => s.observationId === observation.id,
-                      )}
+                  onClick={() => props.setCurrentObservationId(observation.id)}
+                >
+                  <div className="flex gap-2">
+                    <ColorCodedObservationType
+                      observationType={observation.type}
                     />
+                    <span className="flex-1 break-all text-sm">
+                      {observation.name}
+                    </span>
+                    {observation.children.length === 0 ? null : (
+                      <Toggle
+                        onClick={(ev) => (
+                          ev.stopPropagation(),
+                          props.toggleCollapsedObservation(observation.id),
+                          capture(
+                            collapsed
+                              ? "trace_detail:observation_tree_expand"
+                              : "trace_detail:observation_tree_collapse",
+                            { type: "single" },
+                          )
+                        )}
+                        variant="default"
+                        pressed={collapsed}
+                        size="xs"
+                        className="w-7"
+                        title={
+                          collapsed ? "Expand children" : "Collapse children"
+                        }
+                      >
+                        {collapsed ? (
+                          <PlusIcon className="h-4 w-4" />
+                        ) : (
+                          <MinusIcon className="h-4 w-4" />
+                        )}
+                      </Toggle>
+                    )}
                   </div>
-                ) : null}
+                  {props.showMetrics &&
+                    (observation.promptTokens ||
+                      observation.completionTokens ||
+                      observation.totalTokens ||
+                      observation.endTime) && (
+                      <div className="flex gap-2">
+                        {observation.endTime ? (
+                          <span className="text-xs text-gray-500">
+                            {formatIntervalSeconds(
+                              (observation.endTime.getTime() -
+                                observation.startTime.getTime()) /
+                                1000,
+                            )}
+                          </span>
+                        ) : null}
+                        {observation.promptTokens ||
+                        observation.completionTokens ||
+                        observation.totalTokens ? (
+                          <span className="text-xs text-gray-500">
+                            {observation.promptTokens} →{" "}
+                            {observation.completionTokens} (∑{" "}
+                            {observation.totalTokens})
+                          </span>
+                        ) : null}
+                      </div>
+                    )}
+                  {observation.level !== "DEFAULT" ? (
+                    <div className="flex">
+                      <span
+                        className={cn(
+                          "rounded-sm p-0.5 text-xs",
+                          LevelColors[observation.level].bg,
+                          LevelColors[observation.level].text,
+                        )}
+                      >
+                        {observation.level}
+                      </span>
+                    </div>
+                  ) : null}
+                  {props.showScores &&
+                  props.scores.find(
+                    (s) => s.observationId === observation.id,
+                  ) ? (
+                    <div className="flex flex-wrap gap-1">
+                      <GroupedScoreBadges
+                        scores={props.scores.filter(
+                          (s) => s.observationId === observation.id,
+                        )}
+                      />
+                    </div>
+                  ) : null}
+                </div>
               </div>
-            </div>
-            {!collapsed && (
-              <ObservationTreeNode
-                observations={observation.children}
-                collapsedObservations={props.collapsedObservations}
-                toggleCollapsedObservation={props.toggleCollapsedObservation}
-                scores={props.scores}
-                indentationLevel={props.indentationLevel + 1}
-                currentObservationId={props.currentObservationId}
-                setCurrentObservationId={props.setCurrentObservationId}
-                showMetrics={props.showMetrics}
-                showScores={props.showScores}
-              />
-            )}
-          </Fragment>
-        );
-      })}
-  </>
-);
+              {!collapsed && (
+                <ObservationTreeNode
+                  observations={observation.children}
+                  collapsedObservations={props.collapsedObservations}
+                  toggleCollapsedObservation={props.toggleCollapsedObservation}
+                  scores={props.scores}
+                  indentationLevel={props.indentationLevel + 1}
+                  currentObservationId={props.currentObservationId}
+                  setCurrentObservationId={props.setCurrentObservationId}
+                  showMetrics={props.showMetrics}
+                  showScores={props.showScores}
+                />
+              )}
+            </Fragment>
+          );
+        })}
+    </>
+  );
+};
 
 const ColorCodedObservationType = (props: {
   observationType: $Enums.ObservationType;

web/src/components/trace/index.tsx

@@ -23,6 +23,7 @@ import { usdFormatter } from "@/src/utils/numbers";
 import Decimal from "decimal.js";
 import { useCallback, useState } from "react";
 import { DeleteButton } from "@/src/components/deleteButton";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
@@ -30,6 +31,7 @@ export function Trace(props: {
   scores: Score[];
   projectId: string;
 }) {
+  const capture = usePostHogClientCapture();
   const [currentObservationId, setCurrentObservationId] = useQueryParam(
     "observation",
     StringParam,
@@ -77,17 +79,19 @@ export function Trace(props: {
           .filter((id) => !excludeParentObservations.has(id)),
       );
     } while (newExcludeParentObservations.size > 0);
-
+    capture("trace_detail:observation_tree_collapse", { type: "all" });
     setCollapsedObservations(
       props.observations
         .map((o) => o.id)
         .filter((id) => !excludeParentObservations.has(id)),
     );
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [props.observations, currentObservationId]);
 
   const expandAll = useCallback(() => {
+    capture("trace_detail:observation_tree_expand", { type: "all" });
     setCollapsedObservations([]);
-  }, [setCollapsedObservations]);
+  }, []);
 
   return (
     <div className="grid gap-4 md:h-full md:grid-cols-5 lg:grid-cols-6 xl:grid-cols-7">
@@ -115,6 +119,9 @@ export function Trace(props: {
           <Toggle
             pressed={scoresOnObservationTree}
             onPressedChange={(e) => {
+              capture("trace_detail:observation_tree_toggle_scores", {
+                show: e,
+              });
               setScoresOnObservationTree(e);
             }}
             size="sm"
@@ -125,6 +132,9 @@ export function Trace(props: {
           <Toggle
             pressed={metricsOnObservationTree}
             onPressedChange={(e) => {
+              capture("trace_detail:observation_tree_toggle_metrics", {
+                show: e,
+              });
               setMetricsOnObservationTree(e);
             }}
             size="sm"

web/src/components/ui/accordion.tsx

@@ -0,0 +1,58 @@
+"use client"
+
+import * as React from "react"
+import * as AccordionPrimitive from "@radix-ui/react-accordion"
+import { ChevronDown } from "lucide-react"
+
+import { cn } from "@/src/utils/tailwind"
+
+const Accordion = AccordionPrimitive.Root
+
+const AccordionItem = React.forwardRef<
+  React.ElementRef<typeof AccordionPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Item>
+>(({ className, ...props }, ref) => (
+  <AccordionPrimitive.Item
+    ref={ref}
+    className={cn("border-b", className)}
+    {...props}
+  />
+))
+AccordionItem.displayName = "AccordionItem"
+
+const AccordionTrigger = React.forwardRef<
+  React.ElementRef<typeof AccordionPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Trigger>
+>(({ className, children, ...props }, ref) => (
+  <AccordionPrimitive.Header className="flex">
+    <AccordionPrimitive.Trigger
+      ref={ref}
+      className={cn(
+        "flex flex-1 items-center justify-between py-4 font-medium transition-all hover:underline [&[data-state=open]>svg]:rotate-180",
+        className
+      )}
+      {...props}
+    >
+      {children}
+      <ChevronDown className="h-4 w-4 shrink-0 transition-transform duration-200" />
+    </AccordionPrimitive.Trigger>
+  </AccordionPrimitive.Header>
+))
+AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName
+
+const AccordionContent = React.forwardRef<
+  React.ElementRef<typeof AccordionPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <AccordionPrimitive.Content
+    ref={ref}
+    className="overflow-hidden text-sm transition-all data-[state=closed]:animate-accordion-up data-[state=open]:animate-accordion-down"
+    {...props}
+  >
+    <div className={cn("pb-4 pt-0", className)}>{children}</div>
+  </AccordionPrimitive.Content>
+))
+
+AccordionContent.displayName = AccordionPrimitive.Content.displayName
+
+export { Accordion, AccordionItem, AccordionTrigger, AccordionContent }

web/src/components/ui/slider.tsx

@@ -18,9 +18,19 @@ const Slider = React.forwardRef<
     {...props}
   >
     <SliderPrimitive.Track className="relative h-2 w-full grow overflow-hidden rounded-full bg-secondary">
-      <SliderPrimitive.Range className="absolute h-full bg-primary" />
+      <SliderPrimitive.Range
+        className={cn(
+          "absolute h-full",
+          props.disabled ? "bg-input" : "bg-primary",
+        )}
+      />
     </SliderPrimitive.Track>
-    <SliderPrimitive.Thumb className="block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50" />
+    <SliderPrimitive.Thumb
+      className={cn(
+        "block h-5 w-5 rounded-full border-2 bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
+        props.disabled ? "border-secondary" : "border-primary",
+      )}
+    />
   </SliderPrimitive.Root>
 ));
 Slider.displayName = SliderPrimitive.Root.displayName;

web/src/components/ui/table.tsx

@@ -6,13 +6,14 @@ const Table = React.forwardRef<
   HTMLTableElement,
   React.HTMLAttributes<HTMLTableElement>
 >(({ className, ...props }, ref) => (
-  <div className="relative w-full overflow-auto">
-    <table
-      ref={ref}
-      className={cn("w-full caption-bottom text-sm", className)}
-      {...props}
-    />
-  </div>
+  <table
+    ref={ref}
+    className={cn(
+      "w-full table-auto caption-bottom border-separate border-spacing-0 space-y-4 overflow-auto text-sm",
+      className,
+    )}
+    {...props}
+  />
 ));
 Table.displayName = "Table";
 
@@ -73,7 +74,7 @@ const TableHead = React.forwardRef<
   <th
     ref={ref}
     className={cn(
-      "h-12 px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
+      "sticky top-0 z-10 border-b bg-white px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
       className,
     )}
     {...props}
@@ -87,7 +88,10 @@ const TableCell = React.forwardRef<
 >(({ className, ...props }, ref) => (
   <td
     ref={ref}
-    className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)}
+    className={cn(
+      "whitespace-nowrap p-4 align-middle [&:has([role=checkbox])]:pr-0",
+      className,
+    )}
     {...props}
   />
 ));

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.37.2";
+export const VERSION = "v2.44.0";

web/src/ee/features/evals/components/eval-config-detail.tsx

@@ -21,6 +21,7 @@ import {
 import { Label } from "@/src/components/ui/label";
 import TableLink from "@/src/components/table/table-link";
 import EvalLogTable from "@/src/ee/features/evals/components/eval-log";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const EvalConfigDetail = () => {
   const router = useRouter();
@@ -122,8 +123,9 @@ export function DeactivateConfig({
   isLoading: boolean;
 }) {
   const utils = api.useUtils();
-  const hasAccess = useHasAccess({ projectId, scope: "job:CUD" });
+  const hasAccess = useHasAccess({ projectId, scope: "evalJob:CUD" });
   const [isOpen, setIsOpen] = useState(false);
+  const capture = usePostHogClientCapture();
 
   const mutEvalConfig = api.evals.updateEvalJob.useMutation({
     onSuccess: () => {
@@ -141,6 +143,7 @@ export function DeactivateConfig({
       evalConfigId: config?.id ?? "",
       updatedStatus: "INACTIVE",
     });
+    capture("eval_config:delete");
     setIsOpen(false);
   };
 

web/src/ee/features/evals/components/eval-config-form.tsx

@@ -1,4 +1,3 @@
-import { usePostHog } from "posthog-js/react";
 import { useFieldArray, useForm } from "react-hook-form";
 import { Input } from "@/src/components/ui/input";
 import { Button } from "@/src/components/ui/button";
@@ -42,6 +41,7 @@ import { Card } from "@/src/components/ui/card";
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { Label } from "@/src/components/ui/label";
 import DocPopup from "@/src/components/layouts/doc-popup";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 const formSchema = z.object({
   scoreName: z.string(),
@@ -113,7 +113,7 @@ export const InnerEvalConfigForm = (props: {
   onFormSuccess?: () => void;
 }) => {
   const [formError, setFormError] = useState<string | null>(null);
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const form = useForm<z.infer<typeof formSchema>>({
     resolver: zodResolver(formSchema),
@@ -182,7 +182,7 @@ export const InnerEvalConfigForm = (props: {
   });
 
   function onSubmit(values: z.infer<typeof formSchema>) {
-    posthog.capture("evals:new_config_form");
+    capture("eval_config:new_form_submit");
 
     const validatedFilter = z.array(singleFilter).safeParse(values.filter);
 

web/src/ee/features/evals/components/eval-config-table.tsx

@@ -92,32 +92,30 @@ export default function EvalConfigTable({ projectId }: { projectId: string }) {
   };
 
   return (
-    <div>
-      <DataTable
-        columns={columns}
-        data={
-          templates.isLoading
-            ? { isLoading: true, isError: false }
-            : templates.isError
-              ? {
-                  isLoading: false,
-                  isError: true,
-                  error: templates.error.message,
-                }
-              : {
-                  isLoading: false,
-                  isError: false,
-                  data: templates.data.configs.map((t) => convertToTableRow(t)),
-                }
-        }
-        pagination={{
-          pageCount: Math.ceil(totalCount / paginationState.pageSize),
-          onChange: setPaginationState,
-          state: paginationState,
-        }}
-        columnVisibility={columnVisibility}
-        onColumnVisibilityChange={setColumnVisibility}
-      />
-    </div>
+    <DataTable
+      columns={columns}
+      data={
+        templates.isLoading
+          ? { isLoading: true, isError: false }
+          : templates.isError
+            ? {
+                isLoading: false,
+                isError: true,
+                error: templates.error.message,
+              }
+            : {
+                isLoading: false,
+                isError: false,
+                data: templates.data.configs.map((t) => convertToTableRow(t)),
+              }
+      }
+      pagination={{
+        pageCount: Math.ceil(totalCount / paginationState.pageSize),
+        onChange: setPaginationState,
+        state: paginationState,
+      }}
+      columnVisibility={columnVisibility}
+      onColumnVisibilityChange={setColumnVisibility}
+    />
   );
 }

web/src/ee/features/evals/components/eval-template-detail.tsx

@@ -16,6 +16,7 @@ import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { Button } from "@/src/components/ui/button";
 import { Pencil } from "lucide-react";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export const EvalTemplateDetail = () => {
   const router = useRouter();
@@ -92,12 +93,14 @@ export function EvalVersionDropdown(props: {
   defaultOption?: EvalTemplate;
   onSelect?: (template: EvalTemplate) => void;
 }) {
+  const capture = usePostHogClientCapture();
   const handleSelect = (value: string) => {
     const selectedTemplate = props.options?.find(
       (template) => template.id === value,
     );
     if (selectedTemplate && props.onSelect) {
       props.onSelect(selectedTemplate);
+      capture("eval_templates:view_version");
     }
   };
 
@@ -133,9 +136,11 @@ export function UpdateTemplate({
   setIsEditing: (isEditing: boolean) => void;
 }) {
   const hasAccess = useHasAccess({ projectId, scope: "evalTemplate:create" });
+  const capture = usePostHogClientCapture();
 
   const handlePromptEdit = () => {
     setIsEditing(true);
+    capture("eval_templates:update_form_open");
   };
 
   return (

web/src/ee/features/evals/components/eval-templates-table.tsx

@@ -87,34 +87,30 @@ export default function EvalsTemplateTable({
   };
 
   return (
-    <div>
-      <DataTable
-        columns={columns}
-        data={
-          templates.isLoading
-            ? { isLoading: true, isError: false }
-            : templates.isError
-              ? {
-                  isLoading: false,
-                  isError: true,
-                  error: templates.error.message,
-                }
-              : {
-                  isLoading: false,
-                  isError: false,
-                  data: templates.data.templates.map((t) =>
-                    convertToTableRow(t),
-                  ),
-                }
-        }
-        pagination={{
-          pageCount: Math.ceil(totalCount / paginationState.pageSize),
-          onChange: setPaginationState,
-          state: paginationState,
-        }}
-        columnVisibility={columnVisibility}
-        onColumnVisibilityChange={setColumnVisibility}
-      />
-    </div>
+    <DataTable
+      columns={columns}
+      data={
+        templates.isLoading
+          ? { isLoading: true, isError: false }
+          : templates.isError
+            ? {
+                isLoading: false,
+                isError: true,
+                error: templates.error.message,
+              }
+            : {
+                isLoading: false,
+                isError: false,
+                data: templates.data.templates.map((t) => convertToTableRow(t)),
+              }
+      }
+      pagination={{
+        pageCount: Math.ceil(totalCount / paginationState.pageSize),
+        onChange: setPaginationState,
+        state: paginationState,
+      }}
+      columnVisibility={columnVisibility}
+      onColumnVisibilityChange={setColumnVisibility}
+    />
   );
 }

web/src/ee/features/evals/components/template-form.tsx

@@ -1,5 +1,4 @@
-import { usePostHog } from "posthog-js/react";
-import { useEffect, useState } from "react";
+import { useEffect, useState, useMemo } from "react";
 import { useForm } from "react-hook-form";
 import * as z from "zod";
 import { Input } from "@/src/components/ui/input";
@@ -31,6 +30,7 @@ import {
   ModelProvider,
   type OpenAIModel,
   type OpenAIModelParams,
+  type ModelParams,
 } from "@langfuse/shared";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
 import Link from "next/dist/client/link";
@@ -45,6 +45,8 @@ import {
 import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
 import { Label } from "@/src/components/ui/label";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 
 export const EvalTemplateForm = (props: {
   projectId: string;
@@ -186,29 +188,28 @@ export const InnerEvalTemplateForm = (props: {
   isEditing?: boolean;
   setIsEditing?: (isEditing: boolean) => void;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const [formError, setFormError] = useState<string | null>(null);
+  const preFilledModel = useMemo(
+    () => getModelParamsWithEnabledFlag(props.preFilledFormValues),
+    [props.preFilledFormValues],
+  );
 
   // updates the model params based on the pre-filled data
   // either form update or from langfuse-generated template
-  const [modelParams, setModelParams] = useState<UIModelParams>({
-    model: props.preFilledFormValues?.model ?? "gpt-3.5-turbo",
-    provider:
-      props.preFilledFormValues?.modelParams.provider ?? ModelProvider.OpenAI,
-    max_tokens: props.preFilledFormValues?.modelParams.max_tokens ?? 100,
-    maxTemperature:
-      props.preFilledFormValues?.modelParams.provider === ModelProvider.OpenAI
-        ? 2
-        : 1,
-    top_p: props.preFilledFormValues?.modelParams.top_p ?? 1,
-    temperature: props.preFilledFormValues?.modelParams.temperature ?? 1,
-  });
-
-  const updateModelParam: ModelParamsContext["updateModelParam"] = (
+  const [modelParams, setModelParams] = useState<UIModelParams>(preFilledModel);
+  const updateModelParamValue: ModelParamsContext["updateModelParamValue"] = (
     key,
     value,
   ) => {
-    setModelParams((prev) => ({ ...prev, [key]: value }));
+    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], value } }));
+  };
+
+  const setModelParamEnabled: ModelParamsContext["setModelParamEnabled"] = (
+    key,
+    enabled,
+  ) => {
+    setModelParams((prev) => ({ ...prev, [key]: { ...prev[key], enabled } }));
   };
 
   // updates the form based on the pre-filled data
@@ -249,26 +250,30 @@ export const InnerEvalTemplateForm = (props: {
 
       // state for the model params is outside of the form, hence needs to be handled individually
       // also set the context for the playground
-      const model = EvalModelNames.parse(props.preFilledFormValues.model);
-      updateModelParam("model", model);
+      const model = EvalModelNames.parse(preFilledModel.model.value);
+      updateModelParamValue("model", model);
       setModelParams((prev) => ({
         ...prev,
-        ...(props.preFilledFormValues?.modelParams as UIModelParams),
+        ...preFilledModel,
       }));
 
-      const modelProvider = evalLLMModels.find(
-        (m) => m.model === model,
-      )?.provider;
+      const modelProvider = evalLLMModels.find((m) => m.model.value === model)
+        ?.provider.value;
 
       if (modelProvider) {
-        updateModelParam("provider", modelProvider); // updating the provider based on the model
-        updateModelParam(
+        updateModelParamValue("provider", modelProvider); // updating the provider based on the model
+        updateModelParamValue(
           "maxTemperature",
           modelProvider === ModelProvider.OpenAI ? 2 : 1,
         ); // setting the max value of the slider based on the provider
       }
     }
-  }, [props.preFilledFormValues, form, props.existingEvalTemplateName]);
+  }, [
+    props.preFilledFormValues,
+    preFilledModel,
+    form,
+    props.existingEvalTemplateName,
+  ]);
 
   const extractedVariables = form.watch("prompt")
     ? extractVariables(form.watch("prompt")).filter(getIsCharOrUnderscore)
@@ -281,9 +286,13 @@ export const InnerEvalTemplateForm = (props: {
   });
 
   function onSubmit(values: z.infer<typeof formSchema>) {
-    posthog.capture("evals:new_template_form");
+    capture(
+      props.isEditing
+        ? "eval_templates:update_form_submit"
+        : "eval_templates:new_form_submit",
+    );
 
-    const model = EvalModelNames.safeParse(modelParams.model);
+    const model = EvalModelNames.safeParse(modelParams.model.value);
 
     if (!model.success) {
       setFormError("Please select a model.");
@@ -296,7 +305,7 @@ export const InnerEvalTemplateForm = (props: {
         projectId: props.projectId,
         prompt: values.prompt,
         model: model.data,
-        modelParams: modelParams,
+        modelParams: getFinalModelParams(modelParams),
         vars: extractedVariables ?? [],
         outputSchema: {
           score: values.outputScore,
@@ -422,9 +431,13 @@ export const InnerEvalTemplateForm = (props: {
         <div className="col-span-1 row-span-3">
           <div className="flex flex-col gap-6">
             <ModelParameters
-              {...{ modelParams, updateModelParam }}
+              {...{
+                modelParams,
+                updateModelParamValue: updateModelParamValue,
+                setModelParamEnabled,
+              }}
               availableModels={[...evalLLMModels]}
-              disabled={!props.isEditing}
+              formDisabled={!props.isEditing}
             />
             <LLMApiKeyComponent
               projectId={props.projectId}
@@ -486,7 +499,7 @@ export const LLMApiKeyComponent = (p: {
   }
 
   const getModelProvider = (model: string) => {
-    return evalLLMModels.find((m) => m.model === model)?.provider;
+    return evalLLMModels.find((m) => m.model.value === model)?.provider.value;
   };
 
   const getApiKeyForModel = (model: string) => {
@@ -498,14 +511,14 @@ export const LLMApiKeyComponent = (p: {
     <div>
       <Label>API key</Label>
       <div>
-        {getApiKeyForModel(p.modelParams.model) ? (
+        {getApiKeyForModel(p.modelParams.model.value) ? (
           <span className="mr-2 rounded-sm bg-gray-200 p-1 text-xs">
-            {getApiKeyForModel(p.modelParams.model)?.displaySecretKey}
+            {getApiKeyForModel(p.modelParams.model.value)?.displaySecretKey}
           </span>
         ) : undefined}
       </div>
       {/* Custom form message to include a link to the already existing prompt */}
-      {!getApiKeyForModel(p.modelParams.model) ? (
+      {!getApiKeyForModel(p.modelParams.model.value) ? (
         <div className="flex flex-col text-sm font-medium text-destructive">
           {"No LLM API key found."}
 
@@ -523,3 +536,33 @@ export const LLMApiKeyComponent = (p: {
     </div>
   );
 };
+
+function getModelParamsWithEnabledFlag(
+  evalPreFill?: EvalTemplateFormPreFill,
+): UIModelParams {
+  const defaultModelParams: ModelParams & { maxTemperature: number } = {
+    model: evalPreFill?.model ?? "gpt-3.5-turbo",
+    provider: ModelProvider.OpenAI,
+    max_tokens: 100,
+    maxTemperature:
+      evalPreFill?.modelParams?.provider === ModelProvider.OpenAI ? 2 : 1,
+    top_p: 1,
+    temperature: 1,
+  };
+
+  return Object.entries({
+    ...defaultModelParams,
+    ...evalPreFill?.modelParams,
+  }).reduce(
+    (params, [key, value]) => ({
+      ...params,
+      [key]: {
+        enabled: Boolean(
+          !evalPreFill || evalPreFill.modelParams[key as keyof UIModelParams],
+        ),
+        value,
+      },
+    }),
+    {} as UIModelParams,
+  );
+}

web/src/ee/features/evals/pages/configs.tsx

@@ -5,26 +5,33 @@ import Link from "next/link";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { Lock } from "lucide-react";
 import EvalConfigTable from "@/src/ee/features/evals/components/eval-config-table";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
 
 export default function ConfigsPage() {
   const router = useRouter();
   const projectId = router.query.projectId as string;
-
+  const capture = usePostHogClientCapture();
   const hasWriteAccess = useHasAccess({
     projectId,
-    scope: "job:CUD",
+    scope: "evalJob:CUD",
   });
 
   return (
-    <div>
+    <FullScreenPage>
       <Header
         title="Eval configs"
         help={{
-          description: "XXX",
-          href: "https://langfuse.com/docs/evals",
+          description:
+            "Eval configs let you define how your evaluations templates are applied to incoming traces in Langfuse.",
+          href: "https://langfuse.com/docs/scores/model-based-evals",
         }}
         actionButtons={
-          <Button disabled={!hasWriteAccess} asChild>
+          <Button
+            disabled={!hasWriteAccess}
+            onClick={() => capture("eval_config:new_form_open")}
+            asChild
+          >
             <Link
               href={
                 hasWriteAccess ? `/project/${projectId}/evals/configs/new` : "#"
@@ -37,6 +44,6 @@ export default function ConfigsPage() {
         }
       />
       <EvalConfigTable projectId={projectId} />
-    </div>
+    </FullScreenPage>
   );
 }

web/src/ee/features/evals/pages/new-configs.tsx

@@ -16,7 +16,14 @@ export default function NewConfigsPage() {
 
   return (
     <div>
-      <Header title="Create eval config" />
+      <Header
+        title="Create eval config"
+        help={{
+          description:
+            "Eval configs let you define how your evaluations templates are applied to incoming traces in Langfuse.",
+          href: "https://langfuse.com/docs/scores/model-based-evals",
+        }}
+      />
       <EvalConfigForm
         projectId={projectId}
         evalTemplates={evalTemplates.data?.templates ?? []}

web/src/ee/features/evals/pages/new-template.tsx

@@ -16,7 +16,14 @@ export default function NewTemplatesPage() {
 
   return (
     <div className="md:container">
-      <Header title="Create eval template" />
+      <Header
+        title="Create eval template"
+        help={{
+          description:
+            "Create an evaluation template. Choose from one of the pre-defined templates or create your own.",
+          href: "https://langfuse.com/docs/scores/model-based-evals",
+        }}
+      />
       <EvalTemplateForm projectId={projectId} isEditing={true} />
     </div>
   );

web/src/ee/features/evals/pages/templates.tsx

@@ -5,26 +5,32 @@ import Link from "next/link";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { Lock } from "lucide-react";
 import EvalsTemplateTable from "@/src/ee/features/evals/components/eval-templates-table";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export default function TemplatesPage() {
   const router = useRouter();
   const projectId = router.query.projectId as string;
-
+  const capture = usePostHogClientCapture();
   const hasWriteAccess = useHasAccess({
     projectId,
     scope: "evalTemplate:create",
   });
 
   return (
-    <div>
+    <div className="flex h-[calc(100vh-6rem)] flex-col overflow-hidden md:h-[calc(100vh-2rem)]">
       <Header
         title="Eval Templates"
         help={{
-          description: "XXX",
-          href: "https://langfuse.com/docs/evals",
+          description:
+            "Create an evaluation template. Choose from one of the pre-defined templates or create your own.",
+          href: "https://langfuse.com/docs/scores/model-based-evals",
         }}
         actionButtons={
-          <Button disabled={!hasWriteAccess} asChild>
+          <Button
+            disabled={!hasWriteAccess}
+            onClick={() => capture("eval_templates:new_form_open")}
+            asChild
+          >
             <Link
               href={
                 hasWriteAccess

web/src/ee/features/evals/server/router.ts

@@ -45,7 +45,7 @@ export const evalRouter = createTRPCRouter({
       throwIfNoAccess({
         session: ctx.session,
         projectId: input.projectId,
-        scope: "job:read",
+        scope: "evalJob:read",
       });
 
       const configs = await ctx.prisma.jobConfiguration.findMany({
@@ -86,7 +86,7 @@ export const evalRouter = createTRPCRouter({
       throwIfNoAccess({
         session: ctx.session,
         projectId: input.projectId,
-        scope: "job:read",
+        scope: "evalJob:read",
       });
 
       const config = await ctx.prisma.jobConfiguration.findUnique({
@@ -249,7 +249,7 @@ export const evalRouter = createTRPCRouter({
         filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
         mapping: z.array(variableMapping),
         sampling: z.number().gte(0).lte(1),
-        delay: z.number().gte(0).default(10_000),
+        delay: z.number().gte(0).default(DEFAULT_TRACE_JOB_DELAY), // 10 seconds default
       }),
     )
     .mutation(async ({ input, ctx }) => {
@@ -260,7 +260,7 @@ export const evalRouter = createTRPCRouter({
         throwIfNoAccess({
           session: ctx.session,
           projectId: input.projectId,
-          scope: "job:CUD",
+          scope: "evalJob:CUD",
         });
 
         const evalTemplate = await ctx.prisma.evalTemplate.findUnique({
@@ -287,7 +287,7 @@ export const evalRouter = createTRPCRouter({
             filter: input.filter ?? [],
             variableMapping: input.mapping,
             sampling: input.sampling,
-            delay: DEFAULT_TRACE_JOB_DELAY, // 10 seconds default
+            delay: input.delay,
             status: "ACTIVE",
           },
         });
@@ -359,7 +359,7 @@ export const evalRouter = createTRPCRouter({
       throwIfNoAccess({
         session: ctx.session,
         projectId: input.projectId,
-        scope: "job:CUD",
+        scope: "evalJob:CUD",
       });
 
       await ctx.prisma.jobConfiguration.update({
@@ -396,7 +396,7 @@ export const evalRouter = createTRPCRouter({
       throwIfNoAccess({
         session: ctx.session,
         projectId: input.projectId,
-        scope: "job:read",
+        scope: "evalJob:read",
       });
 
       const jobExecutions = await ctx.prisma.jobExecution.findMany({

web/src/ee/features/playground/page/components/PromptVariableComponent.tsx

@@ -1,10 +1,11 @@
 import { type ChangeEvent } from "react";
-import { usePlaygroundContext } from "../context";
-import { Input } from "@/src/components/ui/input";
 import { CheckCircle2, Circle, Trash2Icon } from "lucide-react";
 import { Button } from "@/src/components/ui/button";
+import { Textarea } from "@/src/components/ui/textarea";
 import { type PromptVariable } from "@langfuse/shared";
 
+import { usePlaygroundContext } from "../context";
+
 export const PromptVariableComponent: React.FC<{
   promptVariable: PromptVariable;
 }> = ({ promptVariable }) => {
@@ -12,7 +13,7 @@ export const PromptVariableComponent: React.FC<{
     usePlaygroundContext();
   const { name, value, isUsed } = promptVariable;
 
-  const handleInputChange = (event: ChangeEvent<HTMLInputElement>) => {
+  const handleInputChange = (event: ChangeEvent<HTMLTextAreaElement>) => {
     updatePromptVariableValue(name, event.target.value);
   };
   const handleDeleteVariable = () => {
@@ -29,22 +30,31 @@ export const PromptVariableComponent: React.FC<{
     : "Variable is not in use";
 
   return (
-    <div className="my-2 flex flex-row items-center justify-center space-x-2 text-xs">
-      <p title={isUsedTooltip}>{isUsedIcon}</p>
-      <p className="min-w-[90px] font-mono" title={name}>
-        {displayName}
-      </p>
-      <Input value={value} onChange={handleInputChange} placeholder={name} />
-      <Button
-        variant="ghost"
-        size="icon"
-        title="Delete variable"
-        disabled={isUsed}
-        onClick={handleDeleteVariable}
-        className="p-0"
-      >
-        {!isUsed && <Trash2Icon size={16} />}
-      </Button>
+    <div className="p-1">
+      <div className="mb-1 flex flex-row items-center">
+        <span className="flex flex-1 flex-row space-x-2 text-xs">
+          <p title={isUsedTooltip}>{isUsedIcon}</p>
+          <p className="min-w-[90px] font-mono" title={name}>
+            {displayName}
+          </p>
+        </span>
+        <Button
+          variant="ghost"
+          size="icon"
+          title="Delete variable"
+          disabled={isUsed}
+          onClick={handleDeleteVariable}
+          className="p-0"
+        >
+          {!isUsed && <Trash2Icon size={16} />}
+        </Button>
+      </div>
+      <Textarea
+        className="max-h-[10rem] min-h-[3rem] w-full resize-y pt-3 font-mono text-xs focus:outline-none"
+        value={value}
+        onChange={handleInputChange}
+        placeholder={name}
+      />
     </div>
   );
 };