chore: release v2.46.0

Fixes #2107

.env.prod.example

@@ -58,6 +58,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -80,9 +85,23 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # ENABLE_EVENT_LOG=false
 
 
+
+### START Envs to be deprecated in Langfuse v3.0
+
+# Disable the expensive analytics queries and related features
+# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
+
+# Customize the default datetime offset filter used in for the UI tables
+# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
+
+### END Envs to be deprecated in Langfuse v3.0
+
+
+
+
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
-# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
 
 # NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
 # NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -12,10 +12,24 @@ body:
   - type: textarea
     attributes:
       label: To reproduce
-      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+      description: Describe how to reproduce your bug. Please provide detailed steps, code snippets, reproduction repos etc.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: SDK and container versions
+      description: If you're experiencing an issue with an integration or SDK, please ensure you're using the latest version. If you're self-hosting Langfuse, check that you're running the most recent version. If updating isn't an option, please provide the specific versions you're currently using.
   - type: textarea
     attributes:
       label: Additional information
       description: Add any other information related to the bug here, screenshots if applicable.
+  - type: dropdown
+    id: contribute
+    attributes:
+      label: Are you interested to contribute a fix for this bug?
+      description: If this is a confirmed bug, the maintainers are happy to support with guidance and review.
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true

.github/workflows/pipeline.yml

@@ -109,9 +109,6 @@ jobs:
     strategy:
       matrix:
         node-version: [20]
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }}
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master

.github/workflows/porter_stack_worker.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

CONTRIBUTING.md

@@ -290,6 +290,84 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Theming
+
+At Langfuse, we utilize CSS variables to manage our theme settings across the platform.
+
+Our approach leverages separate CSS variables for backgrounds (--background) and foregrounds (--foreground), fully adhering to the [shadcn/ui](https://ui.shadcn.com/docs/theming) color conventions. The background suffix can be omitted if the variable is used for the background color of the component. We recommend using HSL values for these colors to enhance consistency and customization. There is no need to manually handle dark mode styling with "dark:" prefixes, as next-themes automatically manages the theme switching.
+
+Given the following CSS variables:
+
+```
+--primary: 222.2 47.4% 11.2%; // e.g. background-color
+--primary-foreground: 210 40% 98%; // e.g. text-color
+```
+
+The background color of the following component will be `hsl(var(--primary))` and the foreground color will be `hsl(var(--primary-foreground))`.
+
+```
+<div class="bg-primary text-primary-foreground">Hello</div>
+```
+
+### Color Variables
+
+| Variable                 | Description                                                        | Examples                         |
+| ------------------------ | ------------------------------------------------------------------ | -------------------------------- |
+| --background             | Background color                                                   | Default background color of body |
+| --foreground             | Foreground color                                                   | Default text color of body       |
+| --muted                  | Muted background color                                             | TabsList, Skeleton and Switch    |
+| --muted-foreground       | Muted foreground color                                             |                                  |
+| --popover                | Popover background color                                           | DropdownMenu, HoverCard, Popover |
+| --popover-foreground     | Popover foreground color                                           |                                  |
+| --card                   | Card background color                                              | Card                             |
+| --card-foreground        | Card foreground color                                              |                                  |
+| --border                 | Border color                                                       | Default border color             |
+| --input                  | Input field border color                                           | Input, Select, Textarea          |
+| --primary                | Primary button background colors                                   | Button variant="primary"         |
+| --primary-foreground     | Primary button foreground color                                    |                                  |
+| --secondary              | Secondary button background color                                  | Button variant="secondary"       |
+| --secondary-foreground   | Secondary button foreground color                                  |                                  |
+| --accent                 | Used for accents such as hover effects                             | DropdownMenuItem, SelectItem     |
+| --accent-foreground      | Used for texts on hover effects                                    | DropdownMenuItem, SelectItem     |
+| --destructive            | Destructive action color for background                            | Button variant="destructive"     |
+| --destructive-foreground | Destructive action color for text                                  |                                  |
+| --ring                   | Focus ring color                                                   | MultiSelect                      |
+| --primary-accent         | Primary accent color used for branding                             | Layout                           |
+| --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
+| --muted-green            | Muted green for Event label                                        | ObservationTree                  |
+| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
+| --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
+| --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
+| --accent-dark-green      | Dark green accent for border of output and assistant messages      | CodeJsonViewer and IOPReview     |
+| --light-red              | Light red for error background                                     | level-color and StatusBadge      |
+| --dark-red               | Dark red for error text and error badge dot color                  | level-color and ErrorPage        |
+| --light-yellow           | Light yellow for warning background                                | LevelColor                       |
+| --dark-yellow            | Dark yellow for warning text                                       | LevelColor                       |
+| --light-green            | Light green for success status badge background                    | StatusBadge                      |
+| --dark-green             | Dark green for success status badge text and dot                   | StatusBadge                      |
+| --light-blue             | Light blue for background of Staging label                         | LangfuseLogo                     |
+| --dark-blue              | Dark blue for text and border of Staging label                     | LangfuseLogo                     |
+| --accent-light-blue      | Light blue accent for table link hover effect                      | TableLink                        |
+| --accent-dark-blue       | Dark blue accent for table link text                               | TableLink                        |
+
+### Adding New Colors
+
+1. Global Definitions: Add new CSS variable definitions in the global.css file.
+2. Tailwind Configuration: Reflect these new colors in the tailwind.config.js to maintain alignment with Tailwind's utility classes.
+
+By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
+
+## Using secrets stored in 1Password
+
+When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
+
+Example:
+
+```bash
+op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
+```
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2023 Finto Technologies GmbH
+Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 

careers

@@ -0,0 +1,14 @@
+_We are Hiring_
+Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+
+_Open Roles_
+
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
+_More Info_
+
+- https://langfuse.com/careers
+- https://langfuse.com/docs
+- https://langfuse.com/changelog

docker-compose.yml

@@ -4,7 +4,8 @@ services:
   langfuse-server:
     image: ghcr.io/langfuse/langfuse:latest
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     ports:
       - "3000:3000"
     environment:
@@ -18,6 +19,11 @@ services:
   db:
     image: postgres
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres

ee/LICENSE

@@ -0,0 +1,37 @@
+Langfuse Enterprise License (the “Enterprise License”)
+Copyright (c) 2024 Langfuse GmbH ('Langfuse')
+
+With regard to the Langfuse Software:
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Langfuse Terms of Service, available
+at https://langfuse.com/terms (the “Enterprise Terms”), or other
+agreement governing the use of the Software, as agreed by you and Langfuse,
+and otherwise have a valid Langfuse Enterprise license.
+
+Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that Langfuse
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Langfuse Enterprise license.
+
+Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription. You agree that Langfuse and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications. You are not
+granted any other rights beyond what is expressly stated herein. Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the Langfuse Software, those
+components are licensed under the original license provided by the owner of the 
+applicable component.

ee/package.json

@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "zod": "^3.22.4"
   },

ee/src/sso/index.ts

@@ -2,6 +2,7 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
@@ -144,6 +145,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,24 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +89,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/client/generators.yml

@@ -15,7 +15,7 @@ groups:
         version: 0.0.26
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-client
+          path: ../../../web/public/generated/api-client
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true

fern/apis/server/definition/api.yml

@@ -9,8 +9,8 @@ docs: |
 
   ## Exports
 
-  - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
-  - Postman collection: https://cloud.langfuse.com/postman-collection.json
+  - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/postman/collection.json
 
 error-discrimination:
   strategy: status-code

fern/apis/server/definition/commons.yml

@@ -209,8 +209,8 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
 
 errors:

fern/apis/server/definition/dataset-items.yml

@@ -32,3 +32,6 @@ types:
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id
+      status:
+        type: optional<commons.DatasetStatus>
+        docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/metrics.yml

@@ -31,10 +31,10 @@ service:
             docs: Optional filter for metrics where traces include all of these tags
           fromTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces on or after a certain timestamp
+            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
           toTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces before a certain timestamp
+            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:

fern/apis/server/definition/observations.yml

@@ -35,7 +35,7 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this timestamp.
+            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/score.yml

@@ -29,7 +29,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this timestamp.
+            docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
             type: commons.ScoreSource
             docs: Retrieve only scores from a specific source.

fern/apis/server/definition/trace.yml

@@ -32,7 +32,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this timestamp.
+            docs: Retrieve only traces newer than this datetime (ISO 8601).
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.0.28
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-server
+          path: ../../../web/public/generated/api
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true
@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../web/generated/postman
+          path: ../../../web/public/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.37.2",
+  "version": "2.46.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -28,10 +28,10 @@
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
-    "dotenv-cli": "^7.4.1",
+    "dotenv-cli": "^7.4.2",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.0",
-    "turbo": "^1.13.2"
+    "release-it": "^17.2.1",
+    "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -11,9 +11,9 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.7.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.2",
+    "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -44,14 +44,14 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
-    "@langchain/openai": "^0.0.28",
-    "@prisma/client": "^5.12.1",
+    "@langchain/anthropic": "^0.1.21",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.33",
+    "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -59,9 +59,9 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.5",
+    "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.7.0",
     "eslint": "^8.57.0",
@@ -71,12 +71,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.0"
+    "vitest": "^1.5.3"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,13 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const MembershipRole = {
+export const ProjectRole = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
     VIEWER: "VIEWER"
 } as const;
-export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -25,11 +25,16 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC"
+} as const;
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const PricingUnit = {
     PER_1000_TOKENS: "PER_1000_TOKENS",
     PER_1000_CHARS: "PER_1000_CHARS"
@@ -96,7 +101,7 @@ export type AuditLog = {
     updated_at: Generated<Timestamp>;
     user_id: string;
     project_id: string;
-    user_project_role: MembershipRole;
+    user_project_role: ProjectRole;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -208,17 +213,10 @@ export type LlmApiKeys = {
     secret_key: string;
     project_id: string;
 };
-export type Membership = {
-    project_id: string;
-    user_id: string;
-    role: MembershipRole;
-    created_at: Generated<Timestamp>;
-    updated_at: Generated<Timestamp>;
-};
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: MembershipRole;
+    role: ProjectRole;
     project_id: string;
     sender_id: string | null;
     created_at: Generated<Timestamp>;
@@ -300,6 +298,7 @@ export type ObservationView = {
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
     latency: number | null;
+    time_to_first_token: number | null;
 };
 export type PosthogIntegration = {
     project_id: string;
@@ -324,6 +323,13 @@ export type Project = {
     name: string;
     cloud_config: unknown | null;
 };
+export type ProjectMembership = {
+    project_id: string;
+    user_id: string;
+    role: ProjectRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Prompt = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -342,12 +348,30 @@ export type Prompt = {
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
+    project_id: string;
     name: string;
     value: number;
     source: ScoreSource;
+    author_user_id: string | null;
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -378,6 +402,8 @@ export type Trace = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type TraceSession = {
     id: string;
@@ -437,14 +463,15 @@ export type DB = {
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
     membership_invitations: MembershipInvitation;
-    memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
     pricings: Pricing;
+    project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240503125742_traces_add_createdat_updatedat/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240503130335_traces_index_created_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_created_at_idx" ON "traces"("created_at");

packages/shared/prisma/migrations/20240503130520_traces_index_updated_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_updated_at_idx" ON "traces"("updated_at");

packages/shared/prisma/migrations/20240508132621_scores_add_project_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "project_id" TEXT;

packages/shared/prisma/migrations/20240508132735_scores_add_projectid_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_project_id_idx" ON "scores"("project_id");

packages/shared/prisma/migrations/20240508132736_scores_backfill_project_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill project_id on existing scores based on linked trace_id
+UPDATE scores
+SET project_id = traces.project_id
+FROM traces 
+WHERE scores.trace_id = traces.id AND scores.project_id IS NULL;

packages/shared/prisma/migrations/20240512151529_rename_memberships_to_project_memberships/migration.sql

@@ -0,0 +1,15 @@
+-- Rename table, will cause a downtime as it needs to be locked and synced with TRPC API
+ALTER TABLE "memberships"
+RENAME TO "project_memberships";
+
+-- AlterTable
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_pkey" TO "project_memberships_pkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_project_id_fkey" TO "project_memberships_project_id_fkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_user_id_fkey" TO "project_memberships_user_id_fkey";
+
+-- RenameIndex
+ALTER INDEX "memberships_user_id_idx" RENAME TO "project_memberships_user_id_idx";

packages/shared/prisma/migrations/20240512155020_rename_enum_membership_role_to_project_role/migration.sql

@@ -0,0 +1,37 @@
+-- Goal: Rename the enum `MembershipRole` to `ProjectRole` and update all tables that use it
+
+-- Create a new Enum
+CREATE TYPE "ProjectRole" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER');
+
+-- Add a temporary column to tables (for each table)
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role_temp" text;
+ALTER TABLE "membership_invitations" ADD COLUMN "role_temp" text;
+ALTER TABLE "project_memberships" ADD COLUMN "role_temp" text;
+
+-- Copy data over to temporary columns (for each table)
+UPDATE "audit_logs" SET "user_project_role_temp" = "user_project_role"::text;
+UPDATE "membership_invitations" SET "role_temp" = "role"::text;
+UPDATE "project_memberships" SET "role_temp" = "role"::text;
+
+-- Drop old columns
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "membership_invitations" DROP COLUMN "role";
+ALTER TABLE "project_memberships" DROP COLUMN "role";
+
+-- Rename temporary columns to old column names
+ALTER TABLE "audit_logs" RENAME COLUMN "user_project_role_temp" TO "user_project_role";
+ALTER TABLE "membership_invitations" RENAME COLUMN "role_temp" TO "role";
+ALTER TABLE "project_memberships" RENAME COLUMN "role_temp" TO "role";
+
+-- Convert the text columns to enum columns (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" TYPE "ProjectRole" USING "user_project_role"::"ProjectRole";
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+ALTER TABLE "project_memberships" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+
+-- Make the columns NOT NULL (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" SET NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" SET NOT NULL;
+ALTER TABLE "project_memberships" ALTER COLUMN "role" SET NOT NULL;
+
+-- Now finally, you can drop your old enum
+DROP TYPE "MembershipRole";

packages/shared/prisma/migrations/20240512155021_add_pricing_gpt4o/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o
+  ('b9854a5c92dc496b997d99d20', NULL, 'gpt-4o', '(?i)^(gpt-4o)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-2024-05-13
+  ('b9854a5c92dc496b997d99d21', NULL, 'gpt-4o-2024-05-13', '(?i)^(gpt-4o-2024-05-13)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o-2024-05-13" }')
+  

packages/shared/prisma/migrations/20240512155021_scores_drop_fk_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_trace_id_fkey";

packages/shared/prisma/migrations/20240512155022_scores_non_null_and_add_fk_project_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Made the column `project_id` on table `scores` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "project_id" SET NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240513082203_scores_unique_id_and_projectid_instead_of_id_and_traceid/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "scores_id_trace_id_key";

packages/shared/prisma/migrations/20240513082204_scores_unique_id_and_projectid_instead_of_id_and_traceid_index/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "scores_id_project_id_key" ON "scores"("id", "project_id");

packages/shared/prisma/migrations/20240513082205_observations_view_add_time_to_first_token/migration.sql

@@ -0,0 +1,61 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240522081254_scores_add_author_user_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "author_user_id" TEXT;

packages/shared/prisma/migrations/20240522095738_scores_add_author_user_id_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_author_user_id_idx" ON "scores"("author_user_id");

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views"]
+    previewFeatures = ["tracing", "views", "relationJoins"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id            String                 @id @default(cuid())
-    name          String?
-    email         String?                @unique
-    emailVerified DateTime?              @map("email_verified")
-    password      String?
-    image         String?
-    admin         Boolean                @default(false)
-    accounts      Account[]
-    sessions      Session[]
-    memberships   Membership[]
-    invitations   MembershipInvitation[]
-    createdAt     DateTime               @default(now()) @map("created_at")
-    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags  String[]               @default([]) @map("feature_flags")
-    AuditLog      AuditLog[]
+    id                 String                 @id @default(cuid())
+    name               String?
+    email              String?                @unique
+    emailVerified      DateTime?              @map("email_verified")
+    password           String?
+    image              String?
+    admin              Boolean                @default(false)
+    accounts           Account[]
+    sessions           Session[]
+    projectMemberships ProjectMembership[]
+    invitations        MembershipInvitation[]
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags       String[]               @default([]) @map("feature_flags")
+    AuditLog           AuditLog[]
 
     @@map("users")
 }
@@ -97,7 +97,7 @@ model Project {
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
     cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members            Membership[]
+    projectMembers     ProjectMembership[]
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -113,6 +113,8 @@ model Project {
     JobExecution       JobExecution[]
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
+    Score              Score[]
+    scoreConfig        ScoreConfig[]
 
     @@map("projects")
 }
@@ -154,37 +156,37 @@ model LlmApiKeys {
     @@map("llm_api_keys")
 }
 
-model Membership {
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String         @map("user_id")
-    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      MembershipRole
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+model ProjectMembership {
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String      @map("user_id")
+    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      ProjectRole
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
-    @@map("memberships")
+    @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String         @id @unique @default(cuid())
+    id        String      @id @unique @default(cuid())
     email     String
-    role      MembershipRole
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?        @map("sender_id")
-    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    role      ProjectRole
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?     @map("sender_id")
+    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum MembershipRole {
+enum ProjectRole {
     OWNER
     ADMIN
     MEMBER
@@ -227,8 +229,9 @@ model Trace {
     output     Json?
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
+    createdAt  DateTime      @default(now()) @map("created_at")
+    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    scores          Score[]
     DatasetRunItems DatasetRunItems[]
     DatasetItem     DatasetItem[]
     JobExecution    JobExecution[]
@@ -241,6 +244,8 @@ model Trace {
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
+    @@index(updatedAt)
+    @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
 }
@@ -272,6 +277,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -301,7 +308,6 @@ model Observation {
     outputCost          Decimal?          @map("output_cost")
     totalCost           Decimal?          @map("total_cost")
     completionStartTime DateTime?         @map("completion_start_time")
-    scores              Score[]
     project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
     datasetRunItems     DatasetRunItems[]
@@ -368,6 +374,7 @@ view ObservationView {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
     latency              Float?   @map("latency")
+    timeToFirstToken     Float?   @map("time_to_first_token")
 
     @@map("observations_view")
 }
@@ -388,19 +395,27 @@ enum ObservationLevel {
 model Score {
     id            String         @id @default(cuid())
     timestamp     DateTime       @default(now())
+    projectId     String         @map("project_id")
+    project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
     value         Float
     source        ScoreSource
+    authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
-    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
     observationId String?        @map("observation_id")
-    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
     JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
 
-    @@unique([id, traceId]) // used for upsert
+    @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
+    @@index([projectId])
+    @@index([authorUserId])
+    @@index([configId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
@@ -408,11 +423,39 @@ model Score {
 }
 
 enum ScoreSource {
+    ANNOTATION
     API
-    REVIEW
     EVAL
 }
 
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@map("score_configs")
+}
+
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+}
+
 enum PricingUnit {
     PER_1000_TOKENS
     PER_1000_CHARS
@@ -594,16 +637,16 @@ model Model {
 }
 
 model AuditLog {
-    id              String         @id @default(cuid())
-    createdAt       DateTime       @default(now()) @map("created_at")
-    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
-    userId          String         @map("user_id")
-    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String         @map("project_id")
-    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole MembershipRole @map("user_project_role")
-    resourceType    String         @map("resource_type")
-    resourceId      String         @map("resource_id")
+    id              String      @id @default(cuid())
+    createdAt       DateTime    @default(now()) @map("created_at")
+    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
+    userId          String      @map("user_id")
+    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String      @map("project_id")
+    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole ProjectRole @map("user_project_role")
+    resourceType    String      @map("resource_type")
+    resourceId      String      @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
@@ -703,6 +746,10 @@ model JobExecution {
     @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
     @@map("job_executions")
 }
 

packages/shared/prisma/seed.ts

@@ -12,6 +12,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -51,7 +52,7 @@ async function main() {
     create: {
       id: seedProjectId,
       name: "llm-app",
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,
@@ -72,7 +73,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      labels: ["production"],
+      labels: ["production", "latest"],
       version: 1,
       createdBy: "user-1",
     },
@@ -110,7 +111,7 @@ async function main() {
       create: {
         id: "239ad00f-562f-411d-af14-831c75ddd875",
         name: "demo-app",
-        members: {
+        projectMembers: {
           create: {
             role: "OWNER",
             userId: user.id,
@@ -165,6 +166,24 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
     // add eval objects
     const evalTemplate = await prisma.evalTemplate.upsert({
       where: {
@@ -459,7 +478,7 @@ function createObjects(
     const session =
       Math.random() > 0.3
         ? {
-            id: `session-${i % 10}`,
+            id: `session-${i % 3}`,
             projectId: projectId,
           }
         : undefined;
@@ -500,7 +519,9 @@ function createObjects(
               name: "manual-score",
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
+              projectId,
+              authorUserId: `user-${i}`,
             },
           ]
         : []),
@@ -512,6 +533,7 @@ function createObjects(
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
               source: ScoreSource.API,
+              projectId,
             },
           ]
         : []),
@@ -526,9 +548,9 @@ function createObjects(
       const spanTsStart = new Date(
         traceTs.getTime() + Math.floor(Math.random() * 30)
       );
-      // random duration of upto 30ms
+      // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 30)
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
       );
 
       const span = {
@@ -572,6 +594,13 @@ function createObjects(
                 (spanTsEnd.getTime() - generationTsStart.getTime())
             )
         );
+        // somewhere in the middle
+        const generationTsCompletionStart = new Date(
+          generationTsStart.getTime() +
+            Math.floor(
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
+            )
+        );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
         const completionTokens = Math.floor(Math.random() * 500) + 100;
@@ -590,7 +619,9 @@ function createObjects(
         const model = models[Math.floor(Math.random() * models.length)];
         const promptId =
           promptIds.get(projectId)![
-            Math.floor(Math.random() * promptIds.get(projectId)!.length)
+            Math.floor(
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+            )
           ];
 
         const generation = {
@@ -598,6 +629,8 @@ function createObjects(
           id: `generation-${v4()}`,
           startTime: generationTsStart,
           endTime: generationTsEnd,
+          completionStartTime:
+            Math.random() > 0.5 ? generationTsCompletionStart : undefined,
           name: `generation-${i}-${j}-${k}`,
           projectId: trace.projectId,
           promptId: promptId,
@@ -693,6 +726,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -701,6 +735,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -764,7 +799,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -773,7 +808,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -782,7 +817,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -791,7 +826,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -859,6 +894,7 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
+      labels: ["production", "latest"],
     },
   ];
 
@@ -908,7 +944,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        labels: i === 20 ? ["production"] : [],
+        labels: i === 20 ? ["production", "latest"] : [],
       },
       update: {
         id: promptId,

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider } from "../..";
+import { ModelProvider, UIModelParams } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,26 +77,38 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels = [
+export const evalLLMModels: UIModelParams[] = [
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-3.5-turbo",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-3.5-turbo", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-4-turbo-preview",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4-turbo-preview", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4o", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
 ] as const;
 
-export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+export const EvalModelNames = z.enum([
+  "gpt-3.5-turbo",
+  "gpt-4-turbo-preview",
+  "gpt-4o",
+]);
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/server/llm/types.ts

@@ -20,19 +20,23 @@ export enum ChatMessageRole {
   Assistant = "assistant",
 }
 export type ModelParams = AnthropicModelParams | OpenAIModelParams;
-export type UIModelParams = Required<
-  AnthropicModelParams | OpenAIModelParams
-> & {
-  maxTemperature: number;
+
+type RecordWithEnabledFlag<T> = {
+  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
+export type UIModelParams = RecordWithEnabledFlag<
+  Required<AnthropicModelParams | OpenAIModelParams> & {
+    maxTemperature: number;
+  }
+>;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
 
 export const ZodModelConfig = z.object({
-  max_tokens: z.number().optional(),
-  temperature: z.number().optional(),
-  top_p: z.number().optional(),
+  max_tokens: z.coerce.number().optional(),
+  temperature: z.coerce.number().optional(),
+  top_p: z.coerce.number().optional(),
 });
 
 // OpenAI
@@ -42,6 +46,8 @@ export type OpenAIModelParams = {
 } & ModelConfig;
 
 export const openAIModels = [
+  "gpt-4o",
+  "gpt-4o-2024-05-13",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

web/package.json

@@ -1,14 +1,13 @@
 {
   "name": "web",
-  "version": "2.37.2",
+  "version": "2.46.0",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml && cp generated/postman/collection.json public/postman-collection.json",
-    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -23,17 +22,18 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
-    "@headlessui/react": "^1.7.19",
+    "@headlessui/react": "1.7.18",
+    "@headlessui/tailwindcss": "0.2.0",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
-    "@langchain/openai": "^0.0.28",
-    "@langfuse/shared": "workspace:*",
+    "@langchain/anthropic": "^0.1.21",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.33",
     "@langfuse/ee": "workspace:*",
+    "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -56,52 +56,57 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.16",
-    "@react-email/render": "^0.0.12",
+    "@react-email/components": "^0.0.18",
+    "@react-email/render": "^0.0.14",
+    "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.110.1",
-    "@sentry/node": "^7.110.1",
-    "@sentry/profiling-node": "^7.110.1",
-    "@sentry/types": "^7.110.1",
-    "@t3-oss/env-nextjs": "^0.8.0",
-    "@tailwindcss/forms": "^0.5.7",
+    "@sentry/nextjs": "^7.113.0",
+    "@sentry/node": "^7.113.0",
+    "@sentry/profiling-node": "^7.113.0",
+    "@sentry/types": "^7.113.0",
+    "@t3-oss/env-nextjs": "^0.10.1",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
-    "@tremor/react": "3.11.1",
+    "@tanstack/react-virtual": "^3.5.0",
+    "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
+    "dompurify": "^3.1.4",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.11",
+    "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
+    "next-themes": "^0.3.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.3",
-    "react-icons": "^5.0.1",
+    "react-hook-form": "^7.51.5",
+    "react-icons": "^5.2.1",
     "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",
@@ -116,24 +121,26 @@
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
-    "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^15.0.2",
+    "@tailwindcss/forms": "^0.5.7",
+    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "20.10.5",
-    "@types/nodemailer": "^6.4.14",
+    "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.7.0",
     "autoprefixer": "^10.4.19",
-    "dotenv-cli": "^7.3.0",
+    "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",

web/public/.gitignore

@@ -1,2 +0,0 @@
-openapi*.yml
-postman-collection.json

web/public/generated/api/openapi.yml

@@ -19,9 +19,10 @@ info:
     ## Exports
 
 
-    - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
+    - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
 
-    - Postman collection: https://cloud.langfuse.com/postman-collection.json
+    - Postman collection:
+    https://cloud.langfuse.com/generated/postman/collection.json
 paths:
   /api/public/dataset-items:
     post:
@@ -495,7 +496,7 @@ paths:
           in: query
           description: >-
             Optional filter to only include traces on or after a certain
-            timestamp
+            datetime (ISO 8601)
           required: false
           schema:
             type: string
@@ -503,7 +504,9 @@ paths:
             nullable: true
         - name: toTimestamp
           in: query
-          description: Optional filter to only include traces before a certain timestamp
+          description: >-
+            Optional filter to only include traces before a certain datetime
+            (ISO 8601)
           required: false
           schema:
             type: string
@@ -647,7 +650,7 @@ paths:
           in: query
           description: >-
             Retrieve only observations with a start_time greater than this
-            timestamp.
+            datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -985,7 +988,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only scores newer than this timestamp.
+          description: Retrieve only scores newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1261,7 +1264,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only traces newer than this timestamp.
+          description: Retrieve only traces newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1822,8 +1825,8 @@ components:
       title: ScoreSource
       type: string
       enum:
+        - ANNOTATION
         - API
-        - REVIEW
         - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
@@ -1847,6 +1850,10 @@ components:
           type: string
           nullable: true
           description: Dataset items are upserted on their id
+        status:
+          $ref: '#/components/schemas/DatasetStatus'
+          nullable: true
+          description: Defaults to ACTIVE for newly created items
       required:
         - datasetName
     CreateDatasetRunItemRequest:

web/public/generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml\n- Postman collection: https://cloud.langfuse.com/postman-collection.json"
+    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml\n- Postman collection: https://cloud.langfuse.com/generated/postman/collection.json"
   },
   "variable": [
     {
@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\",\n    \"status\": \"ACTIVE\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -418,12 +418,12 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces on or after a certain timestamp"
+                  "description": "Optional filter to only include traces on or after a certain datetime (ISO 8601)"
                 },
                 {
                   "key": "toTimestamp",
                   "value": "",
-                  "description": "Optional filter to only include traces before a certain timestamp"
+                  "description": "Optional filter to only include traces before a certain datetime (ISO 8601)"
                 }
               ],
               "variable": []
@@ -528,7 +528,7 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this timestamp."
+                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -788,7 +788,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this timestamp."
+                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "source",
@@ -996,7 +996,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this timestamp."
+                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "orderBy",

web/sentry.server.config.ts

@@ -20,6 +20,15 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       ) {
         return 0.3;
       }
+      if (
+        samplingContext.request &&
+        samplingContext.request.url &&
+        samplingContext.request.url.includes("api/auth") &&
+        samplingContext.transactionContext.status !== "ok" &&
+        samplingContext.transactionContext.status !== "unauthenticated"
+      ) {
+        return 1;
+      }
       return 0.1;
     },
 
@@ -28,6 +37,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),
       new Sentry.Integrations.Prisma({ client: prisma }),
+      Sentry.metrics.metricsAggregatorIntegration(),
     ],
 
     // filter out passwords from the signup request body

web/src/__e2e__/auth.spec.ts

@@ -70,3 +70,38 @@ test("Signup input validation", async ({ page }) => {
 // random email address to be used in tests
 const randomEmailAddress = () =>
   Math.random().toString(36).substring(2, 11) + "@example.com";
+
+test("Unauthenticated user should be redirected to target URL after login", async ({
+  page,
+}) => {
+  await page.goto("/auth/sign-in");
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  // wait 2 seconds
+  await page.waitForTimeout(2000);
+
+  // project id and prompt from seed.ts
+  const promptUrl = "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
+
+  await page.getByRole("button", { name: /Demo User/ }).click();
+
+  await page.getByRole("menuitem", { name: "Sign Out" }).click();
+
+  await expect(page).toHaveURL("/auth/sign-in");
+
+  await page.goto(promptUrl);
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(/targetPath/);
+
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(promptUrl);
+});

web/src/__tests__/datasets.servertest.ts

@@ -210,11 +210,14 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       datasetName: "dataset-name",
       input: { key: "value2" },
       metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
     expect(item2.status).toBe(200);
     expect(item2.body).toMatchObject({
       id: "dataset-item-id",
       input: { key: "value2" },
+      metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
 
     const dbDatasetItem = await prisma.datasetItem.findFirst({
@@ -223,6 +226,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     expect(dbDatasetItem).not.toBeNull();
     expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
     expect(dbDatasetItem?.metadata).toMatchObject(["hello-world"]);
+    expect(dbDatasetItem?.status).toBe("ARCHIVED");
   });
 
   it("should create and get a dataset run, include special characters", async () => {

web/src/__tests__/dom.clienttest.ts

@@ -0,0 +1,19 @@
+import * as DOMPurify from "dompurify";
+
+// Create test cases
+describe("should sanitize dom", () => {
+  it("should sanitize urls", () => {
+    const dirty =
+      "http://www.some.site/page.html?default=<script>alert(document.cookie)</script>";
+    const cleam = DOMPurify.sanitize(dirty);
+
+    expect(cleam).toBe("http://www.some.site/page.html?default=");
+  });
+  it("should should not sanitize valid langfuse urls", () => {
+    const validLangfuseUrl =
+      "https://cloud.langfuse.com/project/abjdnksadn/traces?filter=timestamp%3Bdatetime%3B%3B%3E%3B2024-04-19T00%253A00%253A00.000Z%2Cid%3Bstring%3B%3Bcontains%3Ba%2Ctimestamp%3Bdatetime%3B%3B%3C%3B2024-04-30T22%253A00%253A00.000Z%2CinputTokens%3Bnumber%3B%3B%3E%3B1";
+    const cleam = DOMPurify.sanitize(validLangfuseUrl);
+
+    expect(cleam).toBe(validLangfuseUrl);
+  });
+});

web/src/__tests__/ingestion.servertest.ts

@@ -281,6 +281,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expect(dbScore?.name).toBe("score-name");
       expect(dbScore?.value).toBe(100.5);
       expect(dbScore?.observationId).toBeNull();
+      expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
     });
   });
 
@@ -726,6 +727,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             metadata: { key: "value" },
             release: "1.0.0",
             version: "2.0.0",
+            tags: ["tag-1", "tag-2", "tag-2"],
           },
         },
       ],
@@ -742,6 +744,7 @@ describe("/api/public/ingestion API Endpoint", () => {
             id: traceId,
             name: "trace-name",
             userId: "user-2",
+            tags: ["tag-1", "tag-4", "tag-3"],
           },
         },
       ],
@@ -762,6 +765,8 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace[0]?.externalId).toBeNull();
     expect(dbTrace[0]?.version).toBe("2.0.0");
     expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(dbTrace[0]?.tags).toEqual(["tag-1", "tag-2", "tag-3", "tag-4"]);
+    expect(dbTrace[0]?.tags.length).toBe(4);
   });
 
   it("should fail for wrong event formats", async () => {
@@ -816,57 +821,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace.length).toBe(1);
   });
 
-  it("should fail for auth errors", async () => {
-    const traceId = v4();
-    const scoreId = v4();
-
-    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
-      batch: [
-        {
-          id: v4(),
-          type: "trace-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: traceId,
-            name: "trace-name",
-            userId: "user-1",
-            metadata: { key: "value" },
-            release: "1.0.0",
-            version: "2.0.0",
-          },
-        },
-        {
-          id: v4(),
-          type: "score-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: scoreId,
-            name: "score-name",
-            value: 100.5,
-            traceId: "some-random-id",
-          },
-        },
-      ],
-    });
-
-    expect(responseOne.status).toBe(207);
-
-    expect("errors" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.errors.length).toBe(1);
-    expect("successes" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.successes.length).toBe(1);
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        name: "trace-name",
-      },
-    });
-
-    expect(dbTrace.length).toBe(1);
-  });
-
   it("should fail for resource not found", async () => {
     const traceId = v4();
 
@@ -1653,121 +1607,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
     expect(dbScore?.value).toBe(scoreValue);
   });
 
-  it("should error on Bearer Auth for a trace from different project", async () => {
-    const otherProjectId = "other_project_id";
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    // Seed db with a trace to be scored
-    try {
-      await prisma.project.create({
-        data: {
-          id: otherProjectId,
-          name: "another-project",
-        },
-      });
-
-      await prisma.trace.create({
-        data: {
-          id: traceId,
-          name: "trace-name",
-          project: { connect: { id: otherProjectId } },
-        },
-      });
-
-      const response = await makeAPICall(
-        "POST",
-        "/api/public/ingestion",
-        {
-          batch: [
-            {
-              id: scoreEventId,
-              type: "score-create",
-              timestamp: new Date().toISOString(),
-              body: {
-                id: scoreId,
-                name: scoreName,
-                value: scoreValue,
-                traceId: traceId,
-              },
-            },
-          ],
-        },
-        bearerAuth,
-      );
-
-      expect(response.status).toBe(207);
-      expect(response.body.successes.length).toBe(0);
-      expect(response.body).toHaveProperty("errors");
-      expect(response.body.errors.length).toEqual(1);
-      expect(response.body.errors[0]?.id).toEqual(scoreEventId);
-
-      expect(await prisma.trace.count()).toBe(1);
-      expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
-
-      const dbScore = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-        },
-      });
-
-      expect(dbScore).toBeNull();
-    } finally {
-      await prisma.project.delete({ where: { id: otherProjectId } });
-    }
-  });
-
-  it("should error on Bearer Auth with a trace that does not exist", async () => {
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    const response = await makeAPICall(
-      "POST",
-      "/api/public/ingestion",
-      {
-        batch: [
-          {
-            id: scoreEventId,
-            type: "score-create",
-            timestamp: new Date().toISOString(),
-            body: {
-              id: scoreId,
-              name: scoreName,
-              value: scoreValue,
-              traceId: traceId,
-            },
-          },
-        ],
-      },
-      bearerAuth,
-    );
-
-    expect(response.status).toBe(207);
-    expect(response.body.successes.length).toBe(0);
-    expect(response.body.errors.length).toBe(1);
-    expect(response.body.errors[0]?.id).toBe(scoreEventId);
-
-    expect(await prisma.trace.count()).toBe(0);
-
-    const dbScore = await prisma.score.findUnique({
-      where: {
-        id: scoreId,
-      },
-    });
-
-    expect(dbScore).toBeNull();
-  });
-
   it("should error on wrong input", async () => {
     const traceId = "trace_id";
     const bearerAuth = "Bearer pk-lf-1234567890";

web/src/__tests__/model-match.servertest.ts

@@ -73,6 +73,15 @@ describe("model match", () => {
           input: "I am a prompt",
           output: "I am a completion",
         },
+        {
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "gpt-4o",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
       ],
     });
 
@@ -86,7 +95,7 @@ describe("model match", () => {
     expect(observation1?.completionTokens).toEqual(3000);
     expect(observation1?.totalTokens).toEqual(0);
 
-    expect(observations.length).toEqual(3);
+    expect(observations.length).toEqual(4);
     observations.forEach((observation) => {
       expect(observation.internalModel).toBeDefined();
       expect(observation.promptTokens).toBeGreaterThan(0);

web/src/__tests__/prompts.v2.servertest.ts

@@ -709,7 +709,7 @@ describe("/api/public/v2/prompts API Endpoint", () => {
         create: {
           id: otherProjectId,
           name: "demo-app",
-          members: {
+          projectMembers: {
             create: {
               role: "OWNER",
               userId: "user-test",
@@ -784,10 +784,17 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(promptMeta3.tags).toEqual(["tag-1"]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(3);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(3);
+
+      // Validate pagination backwards compatibility
+      // https://github.com/langfuse/langfuse/issues/2068
+      expect(body.pagination?.page).toBe(1);
+      expect(body.pagination?.limit).toBe(10);
+      expect(body.pagination?.totalPages).toBe(1);
+      expect(body.pagination?.totalItems).toBe(3);
     });
 
     it("should fetch a prompt meta list with name filter", async () => {
@@ -802,10 +809,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(1);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(1);
 
       // Test with a different name
       const response2 = await makeAPICall("GET", `${baseURI}?name=prompt-2`);
@@ -819,10 +826,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body2.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body2.pagination.page).toBe(1);
-      expect(body2.pagination.limit).toBe(10);
-      expect(body2.pagination.totalPages).toBe(1);
-      expect(body2.pagination.totalItems).toBe(1);
+      expect(body2.meta.page).toBe(1);
+      expect(body2.meta.limit).toBe(10);
+      expect(body2.meta.totalPages).toBe(1);
+      expect(body2.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if name does not exist
       const response3 = await makeAPICall(
@@ -846,10 +853,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body.data[0].tags).toEqual(["tag-1"]);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(1);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if tag does not exist
       const response3 = await makeAPICall("GET", `${baseURI}?tag=non-existent`);
@@ -875,10 +882,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       ).toBe(true);
 
       // Validate pagination
-      expect(body.pagination.page).toBe(1);
-      expect(body.pagination.limit).toBe(10);
-      expect(body.pagination.totalPages).toBe(1);
-      expect(body.pagination.totalItems).toBe(3);
+      expect(body.meta.page).toBe(1);
+      expect(body.meta.limit).toBe(10);
+      expect(body.meta.totalPages).toBe(1);
+      expect(body.meta.totalItems).toBe(3);
 
       // Test with a different label
       const response2 = await makeAPICall("GET", `${baseURI}?label=dev`);
@@ -892,10 +899,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body2.data[0].tags).toEqual([]);
 
       // Validate pagination
-      expect(body2.pagination.page).toBe(1);
-      expect(body2.pagination.limit).toBe(10);
-      expect(body2.pagination.totalPages).toBe(1);
-      expect(body2.pagination.totalItems).toBe(1);
+      expect(body2.meta.page).toBe(1);
+      expect(body2.meta.limit).toBe(10);
+      expect(body2.meta.totalPages).toBe(1);
+      expect(body2.meta.totalItems).toBe(1);
 
       // Return 200 with empty list if label does not exist
       const response3 = await makeAPICall(
@@ -914,10 +921,10 @@ describe("/api/public/v2/prompts API Endpoint", () => {
     const body = response.body as unknown as PromptsMetaResponse;
 
     expect(body.data).toHaveLength(1);
-    expect(body.pagination.page).toBe(1);
-    expect(body.pagination.limit).toBe(1);
-    expect(body.pagination.totalPages).toBe(3);
-    expect(body.pagination.totalItems).toBe(3);
+    expect(body.meta.page).toBe(1);
+    expect(body.meta.limit).toBe(1);
+    expect(body.meta.totalPages).toBe(3);
+    expect(body.meta.totalItems).toBe(3);
   });
 });
 

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id  WHERE  t."project_id" = ',
+          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
           ";",
         ],
       } as const,

web/src/__tests__/scores.servertest.ts

@@ -60,6 +60,7 @@ describe("/api/public/scores API Endpoint", () => {
     expect(dbScore?.observationId).toBeNull();
     expect(dbScore?.comment).toBe("comment");
     expect(dbScore?.source).toBe("API");
+    expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
   });
 
   it("should create score for a trace with int", async () => {
@@ -488,16 +489,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_2,
           name: scoreName,
           value: 50.5,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator >=", async () => {
@@ -513,16 +514,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
       ]);
     });
     it("test operator !=", async () => {
@@ -538,16 +539,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator =", async () => {

web/src/__tests__/test-utils.ts

@@ -90,7 +90,7 @@ export const setupUserAndProject = async () => {
           },
         ],
       },
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,

web/src/__tests__/traces.servertest.ts

@@ -182,6 +182,32 @@ describe("/api/public/traces API Endpoint", () => {
       endTime: "2021-01-01T00:20:00.000Z",
     });
 
+    // Simulate scores on the trace
+    const scoreId1 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId1,
+      name: "score-1",
+      value: 75.0,
+      traceId: traceId,
+      comment: "First score",
+    });
+    const scoreId2 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId2,
+      name: "score-2",
+      value: 85.5,
+      traceId: traceId,
+      comment: "Second score",
+    });
+    const scoreId3 = uuidv4();
+    await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId3,
+      name: "score-3",
+      value: 95.0,
+      traceId: traceId,
+      comment: "Third score",
+    });
+
     // GET traces
     // Retrieve the trace with totalCost and latency
     const traces = await makeAPICall<GetTracesAPIResponse>(

web/src/__tests__/usage.servertest.ts

@@ -33,6 +33,7 @@ describe("Token Count Functions", () => {
       { model: "claude", tokenizer: "claude", tokens: 118 },
       { model: "claude-instant-1.2", tokenizer: "claude", tokens: 118 },
       { model: "gpt-3.5-turbo-1106", tokenizer: "openai", tokens: 114 },
+      { model: "gpt-4o-2024-05-13", tokenizer: "openai", tokens: 112 },
     ].forEach(({ model, tokens, tokenizer }) => {
       it(`should return token count ${tokens} for ${model}`, () => {
         const result = tokenCount({

web/src/app/api/chatCompletion/route.ts

@@ -0,0 +1,6 @@
+import chatCompletionHandler from "@/src/ee/features/playground/server/chatCompletionHandler";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 60;
+
+export const POST = chatCompletionHandler;

web/src/components/ChatMessages/ChatMessageComponent.tsx

@@ -63,6 +63,7 @@ export const ChatMessageComponent: React.FC<ChatMessageProps> = ({
 
         <Textarea
           ref={textAreaRef}
+          id={message.id}
           className="height-[auto] min-h-8 w-full pt-3  font-mono text-xs focus:outline-none"
           placeholder={placeholder}
           value={message.content}

web/src/components/LangfuseLogo.tsx

@@ -39,18 +39,18 @@ export const LangfuseLogo = ({
     {showEnvLabel && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && (
       <div
         className={cn(
-          "flex items-center gap-2 self-stretch rounded-md px-3 py-2 text-xs ring-1 lg:-mx-2",
+          "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
           env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
-            ? "bg-blue-100 text-blue-500 ring-blue-500"
+            ? "bg-light-blue text-dark-blue ring-dark-blue"
             : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
-              ? "bg-green-100 text-green-500 ring-green-500"
-              : "bg-red-100 text-red-500 ring-red-500",
+              ? "bg-light-green text-dark-green ring-dark-green"
+              : "bg-light-red text-dark-red ring-dark-red",
         )}
       >
         {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
-          <Check size={16} />
+          <Check size={16} className="hidden sm:block" />
         ) : (
-          <AlertTriangle size={16} />
+          <AlertTriangle size={16} className="hidden sm:block" />
         )}
         <span className="whitespace-nowrap">
           {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)
@@ -76,7 +76,7 @@ export const LangfuseLogo = ({
           target="_blank"
           rel="noopener"
           title="View releases on GitHub"
-          className="ml-2 text-xs text-gray-400"
+          className="ml-2 text-xs text-muted-foreground"
         >
           {VERSION}
         </a>

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -0,0 +1,77 @@
+import Link from "next/link";
+
+import { Label } from "@/src/components/ui/label";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+export const LLMApiKeyComponent = (p: {
+  projectId: string;
+  modelParams: UIModelParams;
+}) => {
+  const hasAccess = useHasAccess({
+    projectId: p.projectId,
+    scope: "llmApiKeys:read",
+  });
+
+  if (!hasAccess) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">
+          LLM API Key only visible to Owner and Admin roles.
+        </p>
+      </div>
+    );
+  }
+
+  const apiKeys = api.llmApiKey.all.useQuery({
+    projectId: p.projectId,
+  });
+
+  if (apiKeys.isLoading) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">Loading...</p>
+      </div>
+    );
+  }
+
+  const model = p.modelParams.model.value;
+  const modelProvider = Object.entries(supportedModels).find((providerData) =>
+    (providerData[1] as any as string[]).includes(model),
+  )?.[0];
+
+  const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
+
+  return (
+    <div className="space-y-2 text-xs">
+      <Label className="text-xs font-semibold">API key</Label>
+      <div>
+        {apiKey ? (
+          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+            {apiKey.displaySecretKey}
+          </span>
+        ) : undefined}
+      </div>
+      {/* Custom form message to include a link to the already existing prompt */}
+      {!apiKey ? (
+        <div className="flex flex-col font-medium text-destructive">
+          {`No LLM API key found for provider ${modelProvider}.`}
+
+          <Link
+            href={`/project/${p.projectId}/settings`}
+            className="flex flex-row"
+          >
+            Create a new LLM API key here. <ArrowTopRightIcon />
+          </Link>
+        </div>
+      ) : undefined}
+      <p className="text-muted-foreground">
+        The LLM API key is used for each execution and will incur costs.
+      </p>
+    </div>
+  );
+};

web/src/components/ModelParameters/index.tsx

@@ -1,8 +1,3 @@
-import {
-  supportedModels,
-  ModelProvider,
-  type UIModelParams,
-} from "@langfuse/shared";
 import { Input } from "@/src/components/ui/input";
 import {
   Select,
@@ -12,86 +7,111 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
+import { Switch } from "@/src/components/ui/switch";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
+import { cn } from "@/src/utils/tailwind";
+import {
+  ModelProvider,
+  supportedModels,
+  type UIModelParams,
+} from "@langfuse/shared";
+
+import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
 
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
-  updateModelParam: <Key extends keyof UIModelParams>(
+  updateModelParamValue: <Key extends keyof UIModelParams>(
     key: Key,
-    value: UIModelParams[Key],
+    value: UIModelParams[Key]["value"],
   ) => void;
-  disabled?: boolean;
+  setModelParamEnabled?: (key: keyof UIModelParams, enabled: boolean) => void;
+  formDisabled?: boolean;
 };
 
 export const ModelParameters: React.FC<ModelParamsContext> = ({
   modelParams,
   availableModels,
-  updateModelParam,
-  disabled,
+  updateModelParamValue,
+  setModelParamEnabled,
+  formDisabled = false,
 }) => {
+  const projectId = useProjectIdFromURL();
+
+  if (!projectId) return null;
+
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-6">
+      <div className="space-y-4">
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
-          disabled={disabled}
-          value={modelParams.provider}
+          disabled={formDisabled}
+          value={modelParams.provider.value}
           options={
             availableModels
-              ? [...new Set(availableModels.map((m) => m.provider))]
+              ? [...new Set(availableModels.map((m) => m.provider.value))]
               : Object.values(ModelProvider)
           }
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSelect
           title="Model name"
           modelParamsKey="model"
-          disabled={disabled}
-          value={modelParams.model}
+          disabled={formDisabled}
+          value={modelParams.model.value}
           options={Object.values(
             availableModels
               ? availableModels
-                  .filter((m) => m.provider === modelParams.provider)
-                  .map((m) => m.model)
-              : supportedModels[modelParams.provider],
+                  .filter(
+                    (m) => m.provider.value === modelParams.provider.value,
+                  )
+                  .map((m) => m.model.value)
+              : supportedModels[modelParams.provider.value],
           )}
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Temperature"
           modelParamsKey="temperature"
-          disabled={disabled}
-          value={modelParams.temperature}
+          formDisabled={formDisabled}
+          enabled={modelParams.temperature.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.temperature.value}
           min={0}
-          max={modelParams.maxTemperature}
+          max={modelParams.maxTemperature.value}
           step={0.01}
-          tooltip="The sampling temperature. Higher values will make the output more random, while lower values like will make it more focused and deterministic."
-          updateModelParam={updateModelParam}
+          tooltip="The sampling temperature. Higher values will make the output more random, while lower values will make it more focused and deterministic."
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Output token limit"
           modelParamsKey="max_tokens"
-          disabled={disabled}
-          value={modelParams.max_tokens}
+          formDisabled={formDisabled}
+          enabled={modelParams.max_tokens.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.max_tokens.value}
           min={1}
           max={4096}
           step={1}
           tooltip="The maximum number of tokens that can be generated in the chat completion."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
         <ModelParamsSlider
           title="Top P"
           modelParamsKey="top_p"
-          disabled={disabled}
-          value={modelParams.top_p}
+          formDisabled={formDisabled}
+          enabled={modelParams.top_p.enabled}
+          setModelParamEnabled={setModelParamEnabled}
+          value={modelParams.top_p.value}
           min={0}
           max={1}
           step={0.01}
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
-          updateModelParam={updateModelParam}
+          updateModelParam={updateModelParamValue}
         />
+        <LLMApiKeyComponent {...{ projectId, modelParams }} />
       </div>
     </div>
   );
@@ -102,7 +122,7 @@ type ModelParamsSelectProps = {
   modelParamsKey: keyof UIModelParams;
   value: string;
   options: string[];
-  updateModelParam: ModelParamsContext["updateModelParam"];
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
   disabled?: boolean;
 };
 const ModelParamsSelect = ({
@@ -115,7 +135,14 @@ const ModelParamsSelect = ({
 }: ModelParamsSelectProps) => {
   return (
     <div className="space-y-2">
-      <p className="text-xs font-semibold">{title}</p>
+      <p
+        className={cn(
+          "text-xs font-semibold",
+          disabled && "text-muted-foreground",
+        )}
+      >
+        {title}
+      </p>
       <Select
         disabled={disabled}
         onValueChange={(value) =>
@@ -149,8 +176,10 @@ type ModelParamsSliderProps = {
   min: number;
   max: number;
   step: number;
-  updateModelParam: ModelParamsContext["updateModelParam"];
-  disabled?: boolean;
+  updateModelParam: ModelParamsContext["updateModelParamValue"];
+  enabled?: boolean;
+  formDisabled?: boolean;
+  setModelParamEnabled?: ModelParamsContext["setModelParamEnabled"];
 };
 const ModelParamsSlider = ({
   title,
@@ -161,30 +190,51 @@ const ModelParamsSlider = ({
   max,
   step,
   updateModelParam,
-  disabled,
+  setModelParamEnabled,
+  enabled,
+  formDisabled,
 }: ModelParamsSliderProps) => {
   return (
     <div className="space-y-3" title={tooltip}>
       <div className="flex flex-row">
-        <p className="flex-1 text-xs font-semibold">{title}</p>
-        <Input
-          className="h-6 w-14 appearance-none px-2 text-right"
-          type="number"
-          disabled={disabled}
-          min={min}
-          max={max}
-          step={step}
-          value={value}
-          onChange={(event) => {
-            updateModelParam(
-              modelParamsKey,
-              Math.max(Math.min(parseFloat(event.target.value), max), min),
-            );
-          }}
-        />
+        <p
+          className={cn(
+            "flex-1 text-xs font-semibold",
+            (!enabled || formDisabled) && "text-muted-foreground",
+          )}
+        >
+          {title}
+        </p>
+        <div className="flex flex-row space-x-3">
+          <Input
+            className="h-6 w-14 appearance-none px-2 text-right"
+            type="number"
+            disabled={!enabled || formDisabled}
+            min={min}
+            max={max}
+            step={step}
+            value={value}
+            onChange={(event) => {
+              updateModelParam(
+                modelParamsKey,
+                Math.max(Math.min(parseFloat(event.target.value), max), min),
+              );
+            }}
+          />
+          {setModelParamEnabled ? (
+            <Switch
+              title={`Control sending the ${title} parameter`}
+              disabled={formDisabled}
+              checked={enabled}
+              onCheckedChange={(checked) => {
+                setModelParamEnabled(modelParamsKey, checked);
+              }}
+            />
+          ) : null}
+        </div>
       </div>
       <Slider
-        disabled={disabled}
+        disabled={!enabled || formDisabled}
         min={min}
         max={max}
         step={step}

web/src/components/Slider.tsx

@@ -7,27 +7,23 @@ export const Slider = (props: {
   loading?: boolean;
   onChecked?: (checked: boolean) => void;
   isChecked?: boolean; // whether the slider is active
-}) => {
-  console.log("props", props);
-
-  return (
-    <Switch
-      checked={props.isChecked}
-      disabled={props.loading || props.disabled}
-      onChange={props.onChecked}
+}) => (
+  <Switch
+    checked={props.isChecked}
+    disabled={props.loading || props.disabled}
+    onChange={props.onChecked}
+    className={cn(
+      props.isChecked ? "bg-background" : "bg-input",
+      "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-foreground focus:ring-offset-2",
+    )}
+  >
+    <span className="sr-only">Use setting</span>
+    <span
+      aria-hidden="true"
       className={cn(
-        props.isChecked ? "bg-black" : "bg-gray-200",
-        "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-black focus:ring-offset-2",
+        props.isChecked ? "translate-x-5" : "translate-x-0",
+        "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-background shadow ring-0 transition duration-200 ease-in-out",
       )}
-    >
-      <span className="sr-only">Use setting</span>
-      <span
-        aria-hidden="true"
-        className={cn(
-          props.isChecked ? "translate-x-5" : "translate-x-0",
-          "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out",
-        )}
-      />
-    </Switch>
-  );
-};
+    />
+  </Switch>
+);

web/src/components/deleteButton.tsx

@@ -7,10 +7,10 @@ import {
 } from "@/src/components/ui/popover";
 import { Button } from "@/src/components/ui/button";
 import { TrashIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type Scope } from "@/src/features/rbac/constants/roleAccessRights";
 import { api } from "@/src/utils/api";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DeleteButtonProps {
   itemId: string;
@@ -33,7 +33,7 @@ export function DeleteButton({
 }: DeleteButtonProps) {
   const [isDeleted, setIsDeleted] = useState(false);
   const router = useRouter();
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasAccess = useHasAccess({ projectId, scope: scope });
   const traceMutation = api.traces.deleteMany.useMutation({
@@ -63,6 +63,15 @@ export function DeleteButton({
         <Button
           variant={isTableAction ? "ghost" : "outline"}
           size={isTableAction ? "xs" : "icon"}
+          onClick={() =>
+            type === "trace"
+              ? capture("trace:delete_form_open", {
+                  source: isTableAction ? "table-single-row" : "trace detail",
+                })
+              : capture("datasets:delete_form_open", {
+                  source: "dataset",
+                })
+          }
         >
           <TrashIcon className="h-4 w-4" />
         </Button>
@@ -84,7 +93,7 @@ export function DeleteButton({
                   traceIds: [itemId],
                   projectId,
                 });
-                posthog.capture("trace:delete", {
+                capture("trace:delete", {
                   source: isTableAction ? "table-single-row" : "trace",
                 });
               }}
@@ -101,7 +110,7 @@ export function DeleteButton({
                   projectId,
                   datasetId: itemId,
                 });
-                posthog.capture("dataset:delete", {
+                capture("datasets:delete_dataset_button_click", {
                   source: isTableAction ? "table-single-row" : "dataset",
                 });
               }}

web/src/components/error-page.tsx

@@ -14,14 +14,23 @@ export const ErrorPage = ({
 }) => {
   const session = useSession();
   const router = useRouter();
+  const newTargetPath = router.asPath;
 
   return (
     <div className="flex h-screen flex-col items-center justify-center">
-      <AlertCircle className="mb-4 h-12 w-12 text-red-500" />
+      <AlertCircle className="text-dark-red mb-4 h-12 w-12" />
       <h1 className="mb-4 text-xl font-bold">{title}</h1>
       <p className="mb-8 text-center">{message}</p>
       {session.status === "unauthenticated" ? (
-        <Button onClick={() => void router.push("/")}>Sign In</Button>
+        <Button
+          onClick={() =>
+            void router.push(
+              `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+            )
+          }
+        >
+          Sign In
+        </Button>
       ) : null}
     </div>
   );

web/src/components/grouped-score-badge.tsx

@@ -58,7 +58,7 @@ export const GroupedScoreBadges = ({
           .sort(([a], [b]) => (a < b ? -1 : 1))
           .map(([name, scores]) => (
             <div key={name}>
-              <div className="text-xs text-gray-500">{name}</div>
+              <div className="text-xs text-muted-foreground">{name}</div>
               <ScoresOfGroup scores={scores} />
             </div>
           ))}

web/src/components/json-editor.tsx

@@ -1,6 +1,9 @@
 import CodeMirror, { EditorView } from "@uiw/react-codemirror";
 import { githubLight } from "@uiw/codemirror-theme-github";
+import { tokyoNight } from "@uiw/codemirror-theme-tokyo-night";
 import { json } from "@codemirror/lang-json";
+import { useTheme } from "next-themes";
+import { cn } from "@/src/utils/tailwind";
 
 // todo: add json linting
 
@@ -9,16 +12,20 @@ export function JsonEditor({
   onChange,
   editable = true,
   lineWrapping = true,
+  className,
 }: {
   defaultValue: string;
   onChange?: (value: string) => void;
   editable?: boolean;
   lineWrapping?: boolean;
+  className?: string;
 }) {
+  const { theme } = useTheme();
+  const codeMirrorTheme = theme === "dark" ? tokyoNight : githubLight;
   return (
     <CodeMirror
       value={defaultValue}
-      theme={githubLight}
+      theme={codeMirrorTheme}
       basicSetup={{
         foldGutter: true,
       }}
@@ -26,7 +33,7 @@ export function JsonEditor({
       extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
       defaultValue={defaultValue}
       onChange={onChange}
-      className="overflow-hidden rounded-md border"
+      className={cn("overflow-hidden rounded-md border", className)}
       editable={editable}
     />
   );

web/src/components/layouts/doc-popup.tsx

@@ -3,9 +3,10 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { cn } from "@/src/utils/tailwind";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
-import { usePostHog } from "posthog-js/react";
 
 export type DocPopupProps = {
   description: React.ReactNode;
@@ -26,24 +27,36 @@ export default function DocPopup({
     md: "w-6 h-6",
     lg: "w-8 h-8",
   };
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   return (
     <HoverCard
       openDelay={200}
       onOpenChange={(open) => {
         if (open) {
-          posthog.capture("help-popup:opened", { href, description });
+          capture("help_popup:opened", {
+            hfref: href,
+            description: description,
+          });
         }
       }}
     >
-      <HoverCardTrigger className="mx-1 cursor-pointer" asChild>
+      <HoverCardTrigger
+        className={cn("mx-1", href ? "cursor-pointer" : "cursor-default")}
+        asChild
+      >
         {href ? (
           <Link
             href={href}
             rel="noopener"
             target="_blank"
-            className="inline-block whitespace-nowrap text-gray-500 sm:pl-0"
+            className="inline-block whitespace-nowrap text-muted-foreground sm:pl-0"
+            onClick={() => {
+              capture("help_popup:href_clicked", {
+                href: href,
+                description: description,
+              });
+            }}
           >
             {
               {
@@ -53,7 +66,7 @@ export default function DocPopup({
             }
           </Link>
         ) : (
-          <div className="inline-block whitespace-nowrap text-gray-500 sm:pl-0">
+          <div className="inline-block whitespace-nowrap text-muted-foreground sm:pl-0">
             {
               {
                 question: <HelpCircle className={sizes[size]} />,
@@ -65,7 +78,7 @@ export default function DocPopup({
       </HoverCardTrigger>
       <HoverCardContent>
         {typeof description === "string" ? (
-          <div className="whitespace-break-spaces text-xs font-normal text-gray-800 sm:pl-0">
+          <div className="whitespace-break-spaces text-xs font-normal text-primary sm:pl-0">
             {description}
           </div>
         ) : (
@@ -89,7 +102,7 @@ export function Popup({ triggerContent, description }: PopupProps) {
       </HoverCardTrigger>
       <HoverCardContent>
         {typeof description === "string" ? (
-          <div className="whitespace-break-spaces text-xs font-normal text-gray-800 sm:pl-0">
+          <div className="whitespace-break-spaces text-xs font-normal text-primary sm:pl-0">
             {description}
           </div>
         ) : (

web/src/components/layouts/full-screen-page.tsx

@@ -0,0 +1,9 @@
+export const FullScreenPage: React.FC<React.PropsWithChildren<{}>> = ({
+  children,
+}) => {
+  return (
+    <div className="flex h-[calc(100dvh-6rem)] flex-col overflow-hidden lg:h-[calc(100dvh-2rem)]">
+      {children}
+    </div>
+  );
+};

web/src/components/layouts/header.tsx

@@ -49,10 +49,10 @@ export default function Header({
           <nav className="sm:hidden" aria-label="Back">
             <Link
               href={backHref}
-              className="flex items-center text-sm font-medium text-gray-500 hover:text-gray-700"
+              className="flex items-center text-sm font-medium text-muted-foreground hover:text-primary"
             >
               <ChevronLeftIcon
-                className="-ml-1 mr-1 h-5 w-5 flex-shrink-0 text-gray-400"
+                className="-ml-1 mr-1 h-5 w-5 flex-shrink-0 text-muted-foreground"
                 aria-hidden="true"
               />
               Back
@@ -67,19 +67,19 @@ export default function Header({
                   <div className="flex items-center">
                     {index !== 0 && (
                       <ChevronRightIcon
-                        className="mr-4 h-5 w-5 flex-shrink-0 text-gray-400"
+                        className="mr-4 h-5 w-5 flex-shrink-0 text-muted-foreground"
                         aria-hidden="true"
                       />
                     )}
                     {href ? (
                       <Link
                         href={href}
-                        className="text-sm font-medium text-gray-500 hover:text-gray-700"
+                        className="text-sm font-medium text-muted-foreground hover:text-primary"
                       >
                         {name}
                       </Link>
                     ) : (
-                      <div className="text-sm font-medium text-gray-500">
+                      <div className="text-sm font-medium text-muted-foreground">
                         {name}
                       </div>
                     )}
@@ -94,11 +94,11 @@ export default function Header({
         <div className="flex items-center gap-3 md:gap-5">
           <div className="flex min-w-0 flex-row justify-center align-middle">
             {level === "h2" ? (
-              <h2 className="text-2xl font-bold leading-7 text-gray-900 sm:truncate sm:text-3xl sm:tracking-tight">
+              <h2 className="text-2xl font-bold leading-7 sm:truncate sm:text-3xl sm:tracking-tight">
                 {props.title}
               </h2>
             ) : (
-              <h3 className="text-lg font-bold leading-7 text-gray-900 sm:truncate sm:text-xl sm:tracking-tight">
+              <h3 className="text-lg font-bold leading-7 sm:truncate sm:text-xl sm:tracking-tight">
                 {props.title}
               </h3>
             )}

web/src/components/layouts/layout.tsx

@@ -14,7 +14,6 @@ import {
   AvatarFallback,
   AvatarImage,
 } from "@/src/components/ui/avatar";
-import { api } from "@/src/utils/api";
 import { NewProjectButton } from "@/src/features/projects/components/NewProjectButton";
 import { FeedbackButtonWrapper } from "@/src/features/feedback/component/FeedbackButton";
 import { Button } from "@/src/components/ui/button";
@@ -31,14 +30,27 @@ import {
 import { ChevronDownIcon } from "@heroicons/react/20/solid";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { ProjectNavigation } from "@/src/components/projectNavigation";
+import DOMPurify from "dompurify";
+import { ThemeToggle } from "@/src/features/theming/ThemeToggle";
+
+const signOutUser = async () => {
+  localStorage.clear();
+  sessionStorage.clear();
+
+  await signOut({
+    callbackUrl: "/auth/sign-in",
+  });
+};
 
 const userNavigation = [
+  {
+    name: "Theme",
+    onClick: () => {},
+    content: <ThemeToggle />,
+  },
   {
     name: "Sign out",
-    onClick: () =>
-      signOut({
-        callbackUrl: "/auth/sign-in",
-      }),
+    onClick: signOutUser,
   },
 ];
 
@@ -61,7 +73,7 @@ export default function Layout(props: PropsWithChildren) {
   useCheckNotification(NOTIFICATIONS, session.status === "authenticated");
 
   const enableExperimentalFeatures =
-    api.environment.enableExperimentalFeatures.useQuery().data ?? false;
+    session.data?.environment.enableExperimentalFeatures ?? false;
 
   const projectId = router.query.projectId as string | undefined;
 
@@ -135,9 +147,8 @@ export default function Layout(props: PropsWithChildren) {
     !publishablePaths.includes(router.pathname) &&
     !router.pathname.startsWith("/public/")
   ) {
-    void signOut({
-      callbackUrl: "/auth/sign-in",
-    });
+    signOutUser();
+
     return <Spinner message="Redirecting" />;
   }
 
@@ -147,7 +158,14 @@ export default function Layout(props: PropsWithChildren) {
     !publishablePaths.includes(router.pathname) &&
     !router.pathname.startsWith("/public/")
   ) {
-    void router.replace("/auth/sign-in");
+    const newTargetPath = router.asPath;
+    if (newTargetPath && newTargetPath !== "/") {
+      void router.replace(
+        `/auth/sign-in?targetPath=${encodeURIComponent(newTargetPath)}`,
+      );
+    } else {
+      void router.replace(`/auth/sign-in`);
+    }
     return <Spinner message="Redirecting" />;
   }
 
@@ -155,7 +173,13 @@ export default function Layout(props: PropsWithChildren) {
     session.status === "authenticated" &&
     unauthenticatedPaths.includes(router.pathname)
   ) {
-    void router.replace("/");
+    const targetPath = router.query.targetPath as string | undefined;
+
+    const sanitizedTargetPath = targetPath
+      ? DOMPurify.sanitize(targetPath)
+      : undefined;
+
+    void router.replace(sanitizedTargetPath ?? "/");
     return <Spinner message="Redirecting" />;
   }
 
@@ -165,7 +189,7 @@ export default function Layout(props: PropsWithChildren) {
     router.pathname.startsWith("/public/");
   if (hideNavigation)
     return (
-      <main className="min-h-screen bg-gray-50 px-4 py-4 sm:px-6 lg:px-8">
+      <main className="min-h-screen bg-primary-foreground px-4 py-4 sm:px-6 lg:px-8">
         {props.children}
       </main>
     );
@@ -209,7 +233,7 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="opacity-100"
               leaveTo="opacity-0"
             >
-              <div className="fixed inset-0 bg-gray-900/80" />
+              <div className="fixed inset-0 bg-primary/80" />
             </Transition.Child>
 
             <div className="fixed inset-0 flex">
@@ -240,14 +264,14 @@ export default function Layout(props: PropsWithChildren) {
                       >
                         <span className="sr-only">Close sidebar</span>
                         <XMarkIcon
-                          className="h-6 w-6 text-white"
+                          className="h-5 w-5 text-background"
                           aria-hidden="true"
                         />
                       </button>
                     </div>
                   </Transition.Child>
                   {/* Sidebar component, swap this element with another sidebar if you like */}
-                  <div className="flex grow flex-col gap-y-5 overflow-y-auto bg-white px-6 py-4">
+                  <div className="flex grow flex-col gap-y-5 overflow-y-auto bg-background px-6 py-4">
                     <LangfuseLogo
                       version
                       size="xl"
@@ -260,7 +284,7 @@ export default function Layout(props: PropsWithChildren) {
                         <MainNavigation nav={navigation} />
                       </ul>
                       <div className="mb-2 flex flex-row place-content-between items-center">
-                        <div className="text-xs font-semibold leading-6 text-gray-400">
+                        <div className="text-xs font-semibold text-muted-foreground">
                           Project
                         </div>
                         <NewProjectButton size="xs" />
@@ -278,9 +302,9 @@ export default function Layout(props: PropsWithChildren) {
         </Transition.Root>
 
         {/* Static sidebar for desktop */}
-        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-60 lg:flex-col">
+        <div className="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-56 lg:flex-col">
           {/* Sidebar component, swap this element with another sidebar if you like */}
-          <div className="flex h-screen grow flex-col border-r border-gray-200 bg-white pt-7">
+          <div className="flex h-screen grow flex-col border-r border-border bg-background pt-7">
             <LangfuseLogo
               version
               size="xl"
@@ -299,16 +323,16 @@ export default function Layout(props: PropsWithChildren) {
                   description="What do you think about this project? What can be improved?"
                   type="feedback"
                 >
-                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
+                  <li className="group -mx-2 my-1 flex cursor-pointer gap-x-3 rounded-md p-1.5 text-sm font-semibold text-primary hover:bg-primary-foreground hover:text-primary-accent">
                     <MessageSquarePlus
-                      className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                      className="h-5 w-5 shrink-0 text-muted-foreground group-hover:text-primary-accent"
                       aria-hidden="true"
                     />
                     Feedback
                   </li>
                 </FeedbackButtonWrapper>
                 <div className="mb-2 flex flex-row place-content-between items-center">
-                  <div className="text-xs font-semibold leading-6 text-gray-400">
+                  <div className="text-xs font-semibold text-muted-foreground">
                     Project
                   </div>
                   <NewProjectButton size="xs" />
@@ -321,9 +345,9 @@ export default function Layout(props: PropsWithChildren) {
             </nav>
 
             <Menu as="div" className="relative">
-              <Menu.Button className="flex w-full items-center gap-x-4 overflow-hidden p-1.5 py-3 pl-6 pr-10 text-sm font-semibold leading-6 text-gray-900 hover:bg-gray-50">
+              <Menu.Button className="flex w-full items-center gap-x-2 overflow-hidden p-1.5 py-3 pl-6 pr-8 text-sm font-semibold text-primary hover:bg-primary-foreground">
                 <span className="sr-only">Open user menu</span>
-                <Avatar className="h-8 w-8">
+                <Avatar className="h-7 w-7">
                   <AvatarImage src={session.data?.user?.image ?? undefined} />
                   <AvatarFallback>
                     {session.data?.user?.name
@@ -335,12 +359,12 @@ export default function Layout(props: PropsWithChildren) {
                       : null}
                   </AvatarFallback>
                 </Avatar>
-                <span className="flex-shrink truncate text-sm font-semibold leading-6 text-gray-900">
+                <span className="flex-shrink truncate text-sm font-semibold text-primary">
                   {session.data?.user?.name}
                 </span>
                 <div className="flex-1" />
                 <ChevronDownIcon
-                  className="h-5 w-5 text-gray-400"
+                  className="h-5 w-5 text-muted-foreground"
                   aria-hidden="true"
                 />
               </Menu.Button>
@@ -353,8 +377,8 @@ export default function Layout(props: PropsWithChildren) {
                 leaveFrom="transform opacity-100 scale-100"
                 leaveTo="transform opacity-0 scale-95"
               >
-                <Menu.Items className="absolute -top-full right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                  <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+                <Menu.Items className="absolute -top-full bottom-1 right-0 z-10 overflow-hidden rounded-md bg-background py-2 shadow-lg ring-1 ring-border focus:outline-none">
+                  <span className="block border-b px-3 pb-2 text-sm leading-6 text-muted-foreground">
                     {session.data?.user?.email}
                   </span>
                   {userNavigation.map((item) => (
@@ -363,11 +387,12 @@ export default function Layout(props: PropsWithChildren) {
                         <a
                           onClick={() => void item.onClick()}
                           className={cn(
-                            active ? "bg-gray-50" : "",
-                            "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                            active ? "bg-primary-foreground" : "",
+                            "flex cursor-pointer items-center justify-between px-2 py-0.5 text-sm leading-6 text-primary",
                           )}
                         >
                           {item.name}
+                          {item.content}
                         </a>
                       )}
                     </Menu.Item>
@@ -378,14 +403,14 @@ export default function Layout(props: PropsWithChildren) {
           </div>
         </div>
 
-        <div className="sticky top-0 z-40 flex items-center gap-x-6 bg-white px-4 py-4 shadow-sm sm:px-6 lg:hidden">
+        <div className="sticky top-0 z-40 flex items-center gap-x-6 bg-background px-4 py-4 shadow-sm sm:px-6 lg:hidden">
           <button
             type="button"
-            className="-m-2.5 p-2.5 text-gray-700 lg:hidden"
+            className="-m-2.5 p-2.5 text-primary lg:hidden"
             onClick={() => setSidebarOpen(true)}
           >
             <span className="sr-only">Open sidebar</span>
-            <Bars3Icon className="h-6 w-6" aria-hidden="true" />
+            <Bars3Icon className="h-5 w-5" aria-hidden="true" />
           </button>
           <LangfuseLogo
             version
@@ -393,7 +418,7 @@ export default function Layout(props: PropsWithChildren) {
             showEnvLabel={session.data?.user?.email?.endsWith("@langfuse.com")}
           />
           <Menu as="div" className="relative">
-            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold leading-6 text-gray-900">
+            <Menu.Button className="flex items-center gap-x-4 text-sm font-semibold text-primary">
               <span className="sr-only">Open user menu</span>
               <Avatar className="h-8 w-8">
                 <AvatarImage src={session.data?.user?.image ?? undefined} />
@@ -417,8 +442,8 @@ export default function Layout(props: PropsWithChildren) {
               leaveFrom="transform opacity-100 scale-100"
               leaveTo="transform opacity-0 scale-95"
             >
-              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-white py-2 shadow-lg ring-1 ring-gray-900/5 focus:outline-none">
-                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-gray-500">
+              <Menu.Items className="absolute right-0 z-10 mt-2.5 rounded-md bg-background py-2 pb-1 shadow-lg ring-1 ring-border focus:outline-none">
+                <span className="mb-1 block border-b px-3 pb-2 text-sm leading-6 text-muted-foreground">
                   {session.data?.user?.email}
                 </span>
                 {userNavigation.map((item) => (
@@ -427,11 +452,12 @@ export default function Layout(props: PropsWithChildren) {
                       <a
                         onClick={() => void item.onClick()}
                         className={cn(
-                          active ? "bg-gray-50" : "",
-                          "block cursor-pointer px-3 py-1 text-sm leading-6 text-gray-900",
+                          active ? "bg-primary-foreground" : "",
+                          "flex cursor-pointer items-center justify-between px-2 py-1 text-sm leading-6 text-primary",
                         )}
                       >
                         {item.name}
+                        {item.content}
                       </a>
                     )}
                   </Menu.Item>
@@ -440,13 +466,13 @@ export default function Layout(props: PropsWithChildren) {
             </Transition>
           </Menu>
         </div>
-        <div className="lg:pl-60">
+        <div className="lg:pl-56">
           {env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           projectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
           (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING" ||
             env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU") &&
           !session.data?.user?.email?.endsWith("@langfuse.com") ? (
-            <div className="flex w-full items-center border-b border-yellow-500  bg-yellow-100 px-4 py-2 lg:sticky lg:top-0 lg:z-40">
+            <div className="flex w-full items-center border-b border-dark-yellow  bg-light-yellow px-4 py-2 lg:sticky lg:top-0 lg:z-40">
               <div className="flex flex-1 flex-wrap gap-1">
                 <div className="flex items-center gap-1">
                   <Info className="h-4 w-4" />
@@ -510,9 +536,9 @@ const MainNavigation: React.FC<{
                 href={item.href}
                 className={clsx(
                   item.current
-                    ? "bg-gray-50 text-indigo-600"
-                    : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold leading-6",
+                    ? "bg-primary-foreground text-primary-accent"
+                    : "text-primary hover:bg-primary-foreground hover:text-primary-accent",
+                  "group flex gap-x-3 rounded-md p-2 text-sm font-semibold",
                 )}
                 onClick={onNavitemClick}
                 target={item.newTab ? "_blank" : undefined}
@@ -521,9 +547,9 @@ const MainNavigation: React.FC<{
                   <item.icon
                     className={clsx(
                       item.current
-                        ? "text-indigo-600"
-                        : "text-gray-400 group-hover:text-indigo-600",
-                      "h-6 w-6 shrink-0",
+                        ? "text-primary-accent"
+                        : "text-muted-foreground group-hover:text-primary-accent",
+                      "h-5 w-5 shrink-0",
                     )}
                     aria-hidden="true"
                   />
@@ -532,10 +558,10 @@ const MainNavigation: React.FC<{
                 {item.label && (
                   <span
                     className={cn(
-                      "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                      "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                       item.current
-                        ? "border-indigo-600 text-indigo-600"
-                        : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
+                        ? "border-primary-accent text-primary-accent"
+                        : "border-border text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent",
                     )}
                   >
                     {item.label}
@@ -552,12 +578,12 @@ const MainNavigation: React.FC<{
                 {({ open }) => (
                   <>
                     <Disclosure.Button
-                      className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold leading-6 hover:bg-gray-50 hover:text-indigo-600"
+                      className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold hover:bg-primary-foreground hover:text-primary-accent"
                       onClick={() => setIsOpen(!isOpen)}
                     >
                       {item.icon && (
                         <item.icon
-                          className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"
+                          className="h-5 w-5 shrink-0 text-muted-foreground group-hover:text-primary-accent"
                           aria-hidden="true"
                         />
                       )}
@@ -565,10 +591,10 @@ const MainNavigation: React.FC<{
                       {item.label && (
                         <span
                           className={cn(
-                            "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
+                            "-my-0.5 self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                             item.current
-                              ? "border-indigo-600 text-indigo-600"
-                              : "border-gray-200 text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600",
+                              ? "border-primary-accent text-primary-accent"
+                              : "border-border text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent",
                           )}
                         >
                           {item.label}
@@ -576,13 +602,15 @@ const MainNavigation: React.FC<{
                       )}
                       <ChevronRightIcon
                         className={clsx(
-                          open ? "rotate-90 text-gray-500" : "text-gray-400",
+                          open
+                            ? "rotate-90 text-muted-foreground"
+                            : "text-muted-foreground",
                           "ml-auto h-5 w-5 shrink-0",
                         )}
                         aria-hidden="true"
                       />
                     </Disclosure.Button>
-                    <Disclosure.Panel as="ul" className="mt-1 px-2">
+                    <Disclosure.Panel as="ul" className="mt-1 space-y-1 px-2">
                       {item.children?.map((subItem) => (
                         <li key={subItem.name}>
                           {/* 44px */}
@@ -590,15 +618,15 @@ const MainNavigation: React.FC<{
                             href={subItem.href ?? "#"}
                             className={clsx(
                               subItem.current
-                                ? "bg-gray-50 text-indigo-600"
-                                : "text-gray-700 hover:bg-gray-50 hover:text-indigo-600",
-                              "flex w-full items-center gap-x-3 rounded-md py-2 pl-9 pr-2 text-sm leading-6",
+                                ? "bg-primary-foreground text-primary-accent"
+                                : "text-primary hover:bg-primary-foreground hover:text-primary-accent",
+                              "ml-0.5 flex w-full items-center gap-x-3 rounded-md p-1.5 pl-7 pr-2 text-sm",
                             )}
                             target={subItem.newTab ? "_blank" : undefined}
                           >
                             {subItem.name}
                             {subItem.label && (
-                              <span className="self-center whitespace-nowrap break-keep rounded-sm border border-gray-200 px-1 py-0.5 text-xs text-gray-400 group-hover:border-indigo-600 group-hover:text-indigo-600">
+                              <span className="self-center whitespace-nowrap break-keep rounded-sm border border-border px-1 py-0.5 text-xs text-muted-foreground group-hover:border-primary-accent group-hover:text-primary-accent">
                                 {subItem.label}
                               </span>
                             )}

web/src/components/layouts/routes.ts

@@ -75,13 +75,13 @@ export const ROUTES: Route[] = [
         name: "Configs",
         pathname: `/project/[projectId]/evals/configs`,
         cloudOnly: true,
-        rbacScope: "job:read",
+        rbacScope: "evalJob:read",
       },
       {
         name: "Log",
         pathname: `/project/[projectId]/evals/log`,
         cloudOnly: true,
-        rbacScope: "jobExecution:read",
+        rbacScope: "evalJobExecution:read",
       },
     ],
   },

web/src/components/layouts/spinner.tsx

@@ -5,7 +5,7 @@ export function Spinner(props: { message: string }) {
     <div className="flex min-h-full flex-1 flex-col justify-center py-12 sm:px-6 lg:px-8">
       <div className="sm:mx-auto sm:w-full sm:max-w-md">
         <LangfuseIcon className="mx-auto motion-safe:animate-spin" size={42} />
-        <h2 className="mt-5 text-center text-2xl font-bold leading-9 tracking-tight text-gray-900">
+        <h2 className="mt-5 text-center text-2xl font-bold leading-9 tracking-tight text-primary">
           {props.message} ...
         </h2>
       </div>

web/src/components/layouts/status-badge.tsx

@@ -11,22 +11,22 @@ export type Status =
   (typeof statusCategories)[keyof typeof statusCategories][number];
 
 export const StatusBadge = (props: { className?: string; type: Status }) => {
-  let badgeColor = "bg-gray-100 text-gray-800";
-  let dotColor = "bg-gray-500";
-  let dotPingColor = "bg-gray-600";
+  let badgeColor = "bg-muted-gray text-primary";
+  let dotColor = "bg-muted-foreground";
+  let dotPingColor = "bg-muted-foreground";
   let showDot = true;
 
   if (statusCategories.active.includes(props.type)) {
-    badgeColor = "bg-green-100 text-green-600";
-    dotColor = "animate-ping bg-green-500";
-    dotPingColor = "bg-green-600";
+    badgeColor = "bg-light-green text-dark-green";
+    dotColor = "animate-ping bg-dark-green";
+    dotPingColor = "bg-dark-green";
   } else if (statusCategories.error.includes(props.type)) {
-    badgeColor = "bg-red-100 text-red-600";
-    dotColor = "bg-red-500";
-    dotPingColor = "bg-red-600";
+    badgeColor = "bg-light-red text-dark-red";
+    dotColor = "animate-ping bg-dark-red";
+    dotPingColor = "bg-dark-red";
     showDot = false;
   } else if (statusCategories.completed.includes(props.type)) {
-    badgeColor = "bg-green-100 text-green-600";
+    badgeColor = "bg-light-green text-dark-green";
     showDot = false;
   }
 

web/src/components/level-colors.tsx

@@ -1,6 +1,6 @@
 export const LevelColors = {
   DEFAULT: { text: "", bg: "" },
-  DEBUG: { text: "text-gray-500", bg: "bg-gray-50" },
-  WARNING: { text: "text-yellow-800", bg: "bg-yellow-50" },
-  ERROR: { text: "text-red-800", bg: "bg-red-50" },
+  DEBUG: { text: "text-muted-foreground", bg: "bg-primary-foreground" },
+  WARNING: { text: "text-dark-yellow", bg: "bg-light-yellow" },
+  ERROR: { text: "text-dark-red", bg: "bg-light-red" },
 };

web/src/components/projectNavigation.tsx

@@ -32,9 +32,9 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
         router.push(`/project/${value}`);
       }}
     >
-      <SelectTrigger className="text-gray-700 ring-transparent focus:ring-0 focus:ring-offset-0">
+      <SelectTrigger className="h-8 text-primary ring-transparent focus:ring-0 focus:ring-offset-0">
         <SelectValue
-          className="text-sm font-semibold text-gray-700"
+          className="text-sm font-semibold text-primary"
           placeholder={currentProjectId}
         />
       </SelectTrigger>
@@ -46,8 +46,8 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
                 className={cn(
                   "truncate",
                   currentProjectId === project.id
-                    ? "border-gray-700 text-gray-700"
-                    : "text-gray-400",
+                    ? "border-primary text-primary"
+                    : "text-muted-foreground",
                 )}
               >
                 {project.name}
@@ -57,8 +57,8 @@ export const ProjectNavigation: React.FC<ProjectNavigationProps> = ({
                   className={cn(
                     "self-center whitespace-nowrap break-keep rounded-sm border px-1 py-0.5 text-xs",
                     currentProjectId === project.id
-                      ? "border-gray-700 text-gray-700"
-                      : "border-gray-200 text-gray-400 group-hover:border-gray-700 group-hover:text-gray-700",
+                      ? "border-primary text-primary"
+                      : "border-border text-muted-foreground group-hover:border-primary group-hover:text-primary",
                   )}
                 >
                   view-only

web/src/components/publish-object-switch.tsx

@@ -1,8 +1,8 @@
 import { Switch } from "@/src/components/ui/switch";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { Link, LockIcon } from "lucide-react";
-import { usePostHog } from "posthog-js/react";
 import { useState } from "react";
 
 export const PublishTraceSwitch = (props: {
@@ -10,7 +10,7 @@ export const PublishTraceSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -30,7 +30,7 @@ export const PublishTraceSwitch = (props: {
           traceId: props.traceId,
           public: val,
         });
-        posthog.capture("trace_detail:publish_trace_button_click");
+        capture("trace_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}
@@ -43,7 +43,7 @@ export const PublishSessionSwitch = (props: {
   projectId: string;
   isPublic: boolean;
 }) => {
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
   const hasAccess = useHasAccess({
     projectId: props.projectId,
     scope: "objects:publish",
@@ -63,7 +63,7 @@ export const PublishSessionSwitch = (props: {
           sessionId: props.sessionId,
           public: val,
         });
-        posthog.capture("session_detail:publish_session_button_click");
+        capture("session_detail:publish_button_click");
       }}
       isLoading={mut.isLoading}
       disabled={!hasAccess}
@@ -93,7 +93,7 @@ const Base = (props: {
           "Loading.."
         ) : props.isPublic ? (
           <div
-            className="flex cursor-pointer items-center gap-1 text-green-800"
+            className="text-dark-green flex cursor-pointer items-center gap-1"
             onClick={() => copyUrl()}
           >
             {isCopied ? "Link copied ..." : "Public"}

web/src/components/session/index.tsx

@@ -13,13 +13,32 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
 import { api } from "@/src/utils/api";
 import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
+import { useWindowVirtualizer } from "@tanstack/react-virtual";
+import { type RouterOutput } from "@/src/utils/types";
+import {
+  DataTableRowHeightSwitch,
+  type RowHeight,
+  useRowHeightLocalStorage,
+} from "@/src/components/table/data-table-row-height-switch";
+import { ScrollArea } from "@/src/components/ui/scroll-area";
+
+// do not use the usual table row heights here
+const rowHeightMapping: Record<RowHeight, number> = {
+  s: 200,
+  m: 350,
+  l: 700,
+};
 
 export const SessionPage: React.FC<{
   sessionId: string;
   projectId: string;
 }> = ({ sessionId, projectId }) => {
   const { setDetailPageList } = useDetailPageLists();
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
+    "single-session",
+    "m",
+  );
   const session = api.sessions.byId.useQuery(
     {
       sessionId,
@@ -77,6 +96,11 @@ export const SessionPage: React.FC<{
             }
             listKey="sessions"
           />,
+          <DataTableRowHeightSwitch
+            rowHeight={rowHeight}
+            setRowHeight={setRowHeight}
+            key="height"
+          />,
         ]}
       />
       <div className="flex flex-wrap gap-2">
@@ -97,36 +121,97 @@ export const SessionPage: React.FC<{
           </Badge>
         )}
       </div>
-      <div className="mt-5 flex flex-col gap-2 border-t pt-5">
-        {session.data?.traces.map((trace) => (
-          <Card
-            className="border-border-gray-150 group grid gap-3 p-2 shadow-none hover:border-gray-300 md:grid-cols-3"
-            key={trace.id}
-          >
-            <SessionIO traceId={trace.id} />
-            <div className="-mt-1 p-1 opacity-50 transition-opacity group-hover:opacity-100">
-              <Link
-                href={`/project/${projectId}/traces/${trace.id}`}
-                className="text-xs hover:underline"
-              >
-                Trace: {trace.name} ({trace.id})&nbsp;↗
-              </Link>
-              <div className="text-xs text-gray-500">
-                {trace.timestamp.toLocaleString()}
-              </div>
-              <div className="mb-1 mt-2 text-xs text-gray-500">Scores</div>
-              <div className="flex flex-wrap content-start items-start gap-1">
-                <GroupedScoreBadges scores={trace.scores} />
-              </div>
-              <ManualScoreButton
-                projectId={projectId}
-                traceId={trace.id}
-                scores={trace.scores}
-                variant="badge"
-              />
-            </div>
-          </Card>
-        ))}
+      {session.data && (
+        <TraceCardList
+          session={session.data}
+          projectId={projectId}
+          rowHeight={rowHeightMapping[rowHeight]}
+        />
+      )}
+    </div>
+  );
+};
+
+const TraceCardList = ({
+  session,
+  projectId,
+  rowHeight,
+}: {
+  session: RouterOutput["sessions"]["byId"];
+  projectId: string;
+  rowHeight: number;
+}) => {
+  const listVirtualizationRef = useRef<HTMLDivElement | null>(null);
+
+  const virtualizer = useWindowVirtualizer({
+    count: session.traces.length,
+    estimateSize: () => rowHeight,
+    overscan: 5,
+    scrollMargin: listVirtualizationRef.current?.offsetTop ?? 0,
+    gap: 10,
+  });
+  useEffect(() => {
+    // re-measure when rowHeight changes to update the virtualizer
+    virtualizer.measure();
+  }, [rowHeight, virtualizer]);
+  return (
+    <div className="mt-5 border-t pt-5">
+      <div
+        ref={listVirtualizationRef}
+        style={{
+          height: `${virtualizer.getTotalSize()}px`,
+          position: "relative",
+        }}
+      >
+        {virtualizer
+          .getVirtualItems()
+          .map((virtualItem) => ({
+            virtualItem,
+            trace: session.traces[virtualItem.index],
+          }))
+          .map(({ virtualItem, trace }) => (
+            <Card
+              className="group grid w-full gap-3 overflow-hidden border-border p-2 shadow-none hover:border-ring   md:grid-cols-3"
+              key={virtualItem.key}
+              data-index={virtualItem.index}
+              style={{
+                position: "absolute",
+                top: 0,
+                left: 0,
+                height: `${rowHeight}px`,
+                transform: `translateY(${virtualItem.start - virtualizer.options.scrollMargin}px)`,
+              }}
+            >
+              <ScrollArea className="col-span-2 pr-3">
+                <SessionIO traceId={trace.id} />
+              </ScrollArea>
+              <ScrollArea>
+                <div className="-mt-1 overflow-y-auto p-1 opacity-50 transition-opacity group-hover:opacity-100">
+                  <Link
+                    href={`/project/${projectId}/traces/${trace.id}`}
+                    className="text-xs hover:underline"
+                  >
+                    Trace: {trace.name} ({trace.id})&nbsp;↗
+                  </Link>
+                  <div className="text-xs text-muted-foreground">
+                    {trace.timestamp.toLocaleString()}
+                  </div>
+                  <div className="mb-1 mt-2 text-xs text-muted-foreground">
+                    Scores
+                  </div>
+                  <div className="flex flex-wrap content-start items-start gap-1">
+                    <GroupedScoreBadges scores={trace.scores} />
+                  </div>
+                  <ManualScoreButton
+                    projectId={projectId}
+                    traceId={trace.id}
+                    scores={trace.scores}
+                    variant="badge"
+                  />
+                </div>
+              </ScrollArea>
+            </Card>
+          ))}
       </div>
     </div>
   );
@@ -145,8 +230,9 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
       refetchOnMount: false, // prevents refetching loops
     },
   );
+
   return (
-    <div className="col-span-2 flex flex-col gap-2 p-0">
+    <div className="flex flex-col gap-2 overflow-x-hidden overflow-y-scroll p-0">
       {!trace.data ? (
         <JsonSkeleton
           className="h-full w-full overflow-hidden px-2 py-1"
@@ -160,7 +246,7 @@ const SessionIO = ({ traceId }: { traceId: string }) => {
           hideIfNull
         />
       ) : (
-        <div className="p-2 text-xs text-gray-500">
+        <div className="p-2 text-xs text-muted-foreground">
           This trace has no input or output.
         </div>
       )}

web/src/components/star-toggle.tsx

@@ -6,6 +6,7 @@ import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { cn } from "@/src/utils/tailwind";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function StarToggle({
   value,
@@ -31,7 +32,7 @@ export function StarToggle({
       <StarIcon
         className={cn(
           "h-4 w-4",
-          value ? "fill-current text-yellow-500" : "text-gray-500",
+          value ? "fill-current text-yellow-500" : "text-muted-foreground",
         )}
       />
     </Button>
@@ -53,7 +54,7 @@ export function StarTraceToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -109,13 +110,18 @@ export function StarTraceToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "traces",
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -133,7 +139,7 @@ export function StarTraceDetailsToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
-
+  const capture = usePostHogClientCapture();
   const [isLoading, setIsLoading] = useState(false);
 
   const mutBookmarkTrace = api.traces.bookmark.useMutation({
@@ -180,13 +186,17 @@ export function StarTraceDetailsToggle({
       size={size}
       disabled={!hasAccess}
       isLoading={isLoading}
-      onClick={(value) =>
-        mutBookmarkTrace.mutateAsync({
+      onClick={(value) => {
+        capture("trace_detail:bookmark_button_click", {
+          id: traceId,
+          value: value,
+        });
+        return mutBookmarkTrace.mutateAsync({
           projectId,
           traceId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }
@@ -204,6 +214,7 @@ export function StarSessionToggle({
 }) {
   const utils = api.useUtils();
   const hasAccess = useHasAccess({ projectId, scope: "objects:bookmark" });
+  const capture = usePostHogClientCapture();
   const mutBookmarkSession = api.sessions.bookmark.useMutation({
     onSuccess: () => {
       void utils.sessions.invalidate();
@@ -216,13 +227,18 @@ export function StarSessionToggle({
       size={size}
       isLoading={mutBookmarkSession.isLoading}
       disabled={!hasAccess}
-      onClick={(value) =>
-        mutBookmarkSession.mutateAsync({
+      onClick={(value) => {
+        capture("table:bookmark_button_click", {
+          table: "sessions",
+          id: sessionId,
+          value: value,
+        });
+        return mutBookmarkSession.mutateAsync({
           projectId,
           sessionId,
           bookmarked: value,
-        })
-      }
+        });
+      }}
     />
   );
 }

web/src/components/stats-cards.tsx

@@ -6,19 +6,19 @@ interface Statistic {
 export default function StatsCards({ stats }: { stats: Statistic[] }) {
   return (
     <div>
-      <h3 className="text-base font-semibold leading-6 text-gray-900">
+      <h3 className="text-base font-semibold leading-6 text-primary">
         Model configuration
       </h3>
       <dl className="mt-5 grid grid-cols-1 gap-5 sm:grid-cols-4">
         {stats.map((item) => (
           <div
             key={item.name}
-            className="overflow-hidden rounded-lg bg-white px-4 py-5 shadow sm:p-6"
+            className="overflow-hidden rounded-lg bg-background px-4 py-5 shadow sm:p-6"
           >
-            <dt className="truncate text-sm font-medium text-gray-500">
+            <dt className="truncate text-sm font-medium text-muted-foreground">
               {item.name}
             </dt>
-            <dd className="mt-1 text-3xl font-semibold tracking-tight text-gray-900">
+            <dd className="mt-1 text-3xl font-semibold tracking-tight text-primary">
               {item.stat}
             </dd>
           </div>

web/src/components/table/data-table-column-visibility-filter.tsx

@@ -14,6 +14,7 @@ import {
 import { type VisibilityState } from "@tanstack/react-table";
 import { ChevronDown, Columns } from "lucide-react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTableColumnVisibilityFilterProps<TData, TValue> {
   columns: LangfuseColumnDef<TData, TValue>[];
@@ -27,14 +28,24 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
   setColumnVisibility,
 }: DataTableColumnVisibilityFilterProps<TData, TValue>) {
   const [isOpen, setIsOpen] = useState(false);
-
+  const capture = usePostHogClientCapture();
   const toggleColumn = useCallback(
     (columnId: string) => {
-      setColumnVisibility((old) => ({
-        ...old,
-        [columnId]: !old[columnId],
-      }));
+      setColumnVisibility((old) => {
+        const newColumnVisibility = {
+          ...old,
+          [columnId]: !old[columnId],
+        };
+        const selectedColumns = Object.keys(newColumnVisibility).filter(
+          (key) => newColumnVisibility[key],
+        );
+        capture("table:column_visibility_changed", {
+          selectedColumns: selectedColumns,
+        });
+        return newColumnVisibility;
+      });
     },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
     [setColumnVisibility],
   );
 
@@ -72,7 +83,7 @@ export function DataTableColumnVisibilityFilter<TData, TValue>({
       >
         <Button variant="outline" title="Show/hide columns">
           <Columns className="mr-2 h-4 w-4" />
-          <span className="text-xs text-gray-500">{`(${count}/${total})`}</span>
+          <span className="text-xs text-muted-foreground">{`(${count}/${total})`}</span>
           <ChevronDown className="ml-2 h-4 w-4" />
         </Button>
       </DropdownMenuTrigger>

web/src/components/table/data-table-multi-select-actions/trace-table-multi-select-action.tsx

@@ -9,7 +9,6 @@ import {
 import { Button } from "@/src/components/ui/button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
-import { usePostHog } from "posthog-js/react";
 
 import {
   Dialog,
@@ -20,6 +19,7 @@ import {
   DialogTitle,
 } from "@/src/components/ui/dialog";
 import { useState } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function TraceTableMultiSelectAction({
   selectedTraceIds,
@@ -32,7 +32,7 @@ export function TraceTableMultiSelectAction({
 }) {
   const utils = api.useUtils();
   const [open, setOpen] = useState(false);
-  const posthog = usePostHog();
+  const capture = usePostHogClientCapture();
 
   const hasDeleteAccess = useHasAccess({ projectId, scope: "traces:delete" });
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
@@ -55,6 +55,10 @@ export function TraceTableMultiSelectAction({
           <DropdownMenuItem
             disabled={!hasDeleteAccess}
             onClick={() => {
+              capture("trace:delete_form_open", {
+                count: selectedTraceIds.length,
+                source: "table-multi-select",
+              });
               setOpen(true);
             }}
           >
@@ -94,7 +98,7 @@ export function TraceTableMultiSelectAction({
                   .then(() => {
                     setOpen(false);
                   });
-                posthog.capture("trace:delete", {
+                capture("trace:delete_form_submit", {
                   count: selectedTraceIds.length,
                   source: "table-multi-select",
                 });

web/src/components/table/data-table-pagination.tsx

@@ -14,6 +14,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/src/components/ui/select";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 interface DataTablePaginationProps<TData> {
   table: Table<TData>;
@@ -24,18 +25,27 @@ export function DataTablePagination<TData>({
   table,
   paginationOptions = [10, 20, 30, 40, 50],
 }: DataTablePaginationProps<TData>) {
+  const capture = usePostHogClientCapture();
   return (
-    <div className="mt-3 flex items-center justify-between overflow-x-auto px-2">
+    <div className="flex items-center justify-between">
       <div className="flex-1 text-sm text-muted-foreground">
         {/* {table.getFilteredSelectedRowModel().rows.length} of{" "}
         {table.getFilteredRowModel().rows.length} row(s) selected. */}
       </div>
-      <div className="flex items-center space-x-6 lg:space-x-8">
+      <div className="flex flex-wrap items-center space-x-6 lg:space-x-8">
         <div className="flex items-center space-x-2">
-          <p className="whitespace-nowrap text-sm font-medium">Rows per page</p>
+          <p className="whitespace-nowrap text-sm font-medium md:hidden">
+            Rows
+          </p>
+          <p className="hidden whitespace-nowrap text-sm font-medium md:block">
+            Rows per page
+          </p>
           <Select
             value={`${table.getState().pagination.pageSize}`}
             onValueChange={(value) => {
+              capture("table:pagination_page_size_select", {
+                pageSize: value,
+              });
               table.setPageSize(Number(value));
             }}
           >
@@ -51,7 +61,7 @@ export function DataTablePagination<TData>({
             </SelectContent>
           </Select>
         </div>
-        <div className="flex w-[100px] items-center justify-center whitespace-nowrap text-sm font-medium">
+        <div className="flex items-center justify-center whitespace-nowrap text-sm font-medium">
           Page {table.getState().pagination.pageIndex + 1} of{" "}
           {table.getPageCount()}
         </div>
@@ -59,7 +69,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(0)}
+            onClick={() => {
+              table.setPageIndex(0);
+              capture("table:pagination_button_click", {
+                type: "firstPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to first page</span>
@@ -68,7 +83,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.previousPage()}
+            onClick={() => {
+              table.previousPage();
+              capture("table:pagination_button_click", {
+                type: "previousPage",
+              });
+            }}
             disabled={!table.getCanPreviousPage()}
           >
             <span className="sr-only">Go to previous page</span>
@@ -77,7 +97,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="h-8 w-8 p-0"
-            onClick={() => table.nextPage()}
+            onClick={() => {
+              table.nextPage();
+              capture("table:pagination_button_click", {
+                type: "nextPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to next page</span>
@@ -86,7 +111,12 @@ export function DataTablePagination<TData>({
           <Button
             variant="outline"
             className="hidden h-8 w-8 p-0 lg:flex"
-            onClick={() => table.setPageIndex(table.getPageCount() - 1)}
+            onClick={() => {
+              table.setPageIndex(table.getPageCount() - 1);
+              capture("table:pagination_button_click", {
+                type: "lastPage",
+              });
+            }}
             disabled={!table.getCanNextPage()}
           >
             <span className="sr-only">Go to last page</span>

web/src/components/table/data-table-row-height-switch.tsx

@@ -1,5 +1,6 @@
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import useLocalStorage from "@/src/components/useLocalStorage";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import {
   MdDensityLarge,
   MdDensityMedium,
@@ -35,25 +36,33 @@ export const DataTableRowHeightSwitch = ({
 }: {
   rowHeight: RowHeight;
   setRowHeight: (e: RowHeight) => void;
-}) => (
-  <Tabs
-    //defaultValue={height}
-    value={rowHeight}
-    onValueChange={(e) => setRowHeight(e as any)}
-    key="height"
-  >
-    <TabsList className="gap-1 border bg-transparent px-2">
-      {heightOptions.map(({ id, label, icon }) => (
-        <TabsTrigger
-          key={id}
-          value={id}
-          className="px-2 shadow-none data-[state=active]:bg-slate-200 data-[state=active]:ring-border"
-        >
-          <span role="img" aria-label={`${label} size`}>
-            {icon}
-          </span>
-        </TabsTrigger>
-      ))}
-    </TabsList>
-  </Tabs>
-);
+}) => {
+  const capture = usePostHogClientCapture();
+  return (
+    <Tabs
+      //defaultValue={height}
+      value={rowHeight}
+      onValueChange={(e) => {
+        capture("table:row_height_switch_select", {
+          rowHeight: e,
+        });
+        setRowHeight(e as any);
+      }}
+      key="height"
+    >
+      <TabsList className="gap-1 border bg-transparent px-2">
+        {heightOptions.map(({ id, label, icon }) => (
+          <TabsTrigger
+            key={id}
+            value={id}
+            className="px-2 shadow-none data-[state=active]:bg-input data-[state=active]:ring-border"
+          >
+            <span role="img" aria-label={`${label} size`}>
+              {icon}
+            </span>
+          </TabsTrigger>
+        ))}
+      </TabsList>
+    </Tabs>
+  );
+};