chore: release v2.71.0

* chore(markdown-ui): disable image previews on public traces

* fix(markdown-ui): wrap markdown img in fragment to prevent dom nesting error

.env.dev.example

@@ -6,6 +6,12 @@
 DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+
 # Next Auth
 # You can generate a new secret on the command line with:
 # openssl rand -base64 32
@@ -44,4 +50,4 @@ REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 # openssl rand -hex 32 used only here
-ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 

.env.local.example

@@ -18,4 +18,7 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=myworkerpassword
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -10,36 +10,38 @@ DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
 # optional, set to true to disable automated database migrations on Docker start
-# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED= 
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED=
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
 NEXTAUTH_URL="http://localhost:3000"
 
-# You can generate a new secret on the command line with:
+# For each of these, you can generate a new secret on the command line with:
 # openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-NEXTAUTH_SECRET="secret"
-SALT="salt"
+NEXTAUTH_SECRET="secret" # https://next-auth.js.org/configuration/options#secret
+SALT="salt" # salt used to hash api keys
+
+# API level encryption for sensitive data
+# Must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32
+ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000"
 
 # Use CSP headers to enforce HTTPS, optional
-LANGFUSE_CSP_ENFORCE_HTTPS="true"
+# LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
 
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
 
-# Default project, optional
+# Default role for users who sign up, optional, can be org or org+project
+# LANGFUSE_DEFAULT_ORG_ID=
+# LANGFUSE_DEFAULT_ORG_ROLE=
 # LANGFUSE_DEFAULT_PROJECT_ID=
 # LANGFUSE_DEFAULT_PROJECT_ROLE=
 
 # Enable experimental features, optional
 # LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
-# Disable automated database migrations on Docker start, optional
-# LANFUSE_WEB_MIGRATION_DISABLED=true
-
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
@@ -95,8 +97,20 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
-# Disable logging to `events` table in db
-# ENABLE_EVENT_LOG=false
+
+
+### START Enterprise Edition Configuration
+
+# Allowlisted users that can create new organizations, by default all users can create organizations
+# LANGFUSE_ALLOWED_ORGANIZATION_CREATORS=user1@langfuse.com,user2@langfuse.com
+
+# UI Customization Options
+# LANGFUSE_UI_API_HOST=https://api.example.com
+# LANGFUSE_UI_DOCUMENTATION_HREF=https://docs.example.com
+# LANGFUSE_UI_SUPPORT_HREF=https://support.example.com
+# LANGFUSE_UI_FEEDBACK_HREF=https://feedback.example.com
+
+### END Enterprise Edition Configuration
 
 
 
@@ -131,6 +145,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # SENTRY_AUTH_TOKEN=
 # Used to determine the Sentry sample rate
 # LANGFUSE_TRACING_SAMPLE_RATE=
+# LANGFUSE_WORKER_BETTERSTACK_TOKEN=
 
 
 # Cloudflare Turnstile
@@ -138,14 +153,12 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # TURNSTILE_SECRET_KEY=
 
 # Demo project that users can use to try the platform
+# NEXT_PUBLIC_DEMO_ORG_ID=
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
-# Encryption
-# ENCRYPTION_KEY=
-
 # Admin API
 # ADMIN_API_KEY=
 
@@ -162,3 +175,26 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
 
 ### END Langfuse Cloud Config
+
+
+## START Langfuse V3 Ingestion 
+
+# Clickhouse
+# CLICKHOUSE_MIGRATION_URL=
+# CLICKHOUSE_URL=
+# CLICKHOUSE_USER=
+# CLICKHOUSE_PASSWORD=
+
+# Ingestion
+# LANGFUSE_INGESTION_BUFFER_TTL_SECONDS=
+# LANGFUSE_INGESTION_FLUSH_DELAY_MS= 
+# LANGFUSE_INGESTION_FLUSH_ATTEMPTS= 
+# LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE= 
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
+# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
+# LANGFUSE_LOG_LEVEL= 
+# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
+# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+
+## END Langfuse V3 Ingestion

.github/workflows/pipeline.yml

@@ -94,7 +94,7 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v '^REDIS_HOST=' .env.dev.example > web/.env
+          grep -v -e '^REDIS_HOST=' .env.dev.example > .env
 
       - name: Run + migrate
         run: |
@@ -147,10 +147,17 @@ jobs:
         run: |
           pnpm install
 
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+
       - name: Load default env
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
+          cp .env.dev.example worker/.env
 
       - name: Run + migrate
         run: |
@@ -160,6 +167,7 @@ jobs:
 
       - name: Seed DB
         run: |
+          pnpm run --filter=shared ch:up
           pnpm run db:migrate
           pnpm run db:seed
 
@@ -211,10 +219,62 @@ jobs:
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
 
+  e2e-server-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
+          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed:examples
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Run server
+        run: (pnpm run start&)
+
+      - name: Run e2e tests
+        run: pnpm --filter=web run test:e2e:server
+
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
+    needs:
+      [
+        lint,
+        tests-web,
+        tests-worker,
+        e2e-tests,
+        test-docker-build,
+        e2e-server-tests,
+      ]
     if: always()
     steps:
       - name: Successful deploy

.github/workflows/porter_stack_web_eu_temp.yml

@@ -1,29 +1,29 @@
 "on":
   push:
     branches:
-    - production
+      - production
 name: Deploy to web
 jobs:
   porter-deploy:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Setup porter
-      uses: porter-dev/setup-porter@v0.1.0
-    - name: Deploy stack
-      timeout-minutes: 30
-      run: exec porter apply
-      env:
-        PORTER_CLUSTER: "4060"
-        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
-        PORTER_HOST: https://dashboard.porter.run
-        PORTER_PR_NUMBER: ${{ github.event.number }}
-        PORTER_PROJECT: "12565"
-        PORTER_REPO_NAME: ${{ github.event.repository.name }}
-        PORTER_STACK_NAME: web
-        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4060"
+          PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -1,29 +1,29 @@
 "on":
   push:
     branches:
-    - production
+      - production
 name: Deploy to worker (EU temp)
 jobs:
   porter-deploy:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Setup porter
-      uses: porter-dev/setup-porter@v0.1.0
-    - name: Deploy stack
-      timeout-minutes: 30
-      run: exec porter apply
-      env:
-        PORTER_CLUSTER: "4060"
-        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
-        PORTER_HOST: https://dashboard.porter.run
-        PORTER_PR_NUMBER: ${{ github.event.number }}
-        PORTER_PROJECT: "12565"
-        PORTER_REPO_NAME: ${{ github.event.repository.name }}
-        PORTER_STACK_NAME: worker
-        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4060"
+          PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/snyk.yml

@@ -1,5 +1,12 @@
 name: Snyk Container
-on: push
+on:
+  push:
+    branches-ignore:
+      - "dependabot/**"
+  pull_request:
+    branches:
+      - "**"
+
 jobs:
   snyk:
     runs-on: ubuntu-latest
@@ -18,7 +25,7 @@ jobs:
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: web
@@ -33,7 +40,7 @@ jobs:
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: worker

README.md

@@ -1,4 +1,8 @@
-![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+![langfuse_logo_1024](https://github.com/user-attachments/assets/f17d8fc1-8206-4afa-a5ec-774826be88a8)
+
+<div align="center"><h1> Langfuse: Open Source LLM Engineering Platform</h4></div>
+<div align="center"><h2> Debug and improve your LLM app</h4></div>
+<div align="center"><h3> LLM Observability, Prompt Management, LLM Evaluations, Datasets, LLM Metrics and Prompt Playground</h4></div>
 
 <div align="center">
    <div>
@@ -37,7 +41,7 @@
 </div>
 </br>
 
-## Overview
+## Langfuse Overview
 
 _Unmute video for voice-over_
 
@@ -45,18 +49,18 @@ https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6
 
 ### Develop
 
-- **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
+- **LLM Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
 - **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
 - **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
-- **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
-- **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
+- **LLM Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
+- **LLM Evaluations:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
+  - Run ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) and LLM-as-a-Judge within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
-  - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
+  - Manually score LLM outputs in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 
 ### Test
 
@@ -72,6 +76,8 @@ Managed deployment by the Langfuse team, generous free-tier (hobby plan), no cre
 
 **[» Langfuse Cloud](https://cloud.langfuse.com)**
 
+## Self-Hosting Open Source LLM Observability with Langfuse
+
 ### Localhost (docker)
 
 ```bash
@@ -98,7 +104,7 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe
 
 You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
-### Ingesting Data · Instrumenting Your Application
+### Ingesting Data · Instrumenting Your Application · LLM Observability with Langfuse
 
 Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
@@ -106,7 +112,7 @@ You may want to use another integration to get started quickly or implement a us
 
 See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.
 
-### Integrations
+### LLM Observability Integrations
 
 | Integration                                              | Supports                   | Description                                                                                                                                      |
 | -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
@@ -132,9 +138,9 @@ Packages that integrate with Langfuse:
 
 ### Ideas and roadmap
 
-- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
-- [Feature Requests](https://langfuse.com/idea)
 - [Roadmap](https://langfuse.com/roadmap)
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
+- [Feature Requests](https://langfuse.com/idea
 
 ### Support and feedback
 

docker-compose.build.yml

@@ -16,6 +16,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

docker-compose.dev.yml

@@ -1,4 +1,22 @@
 services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
   redis:
     image: redis:7.2.4
     restart: always
@@ -7,7 +25,7 @@ services:
     ports:
       - 6379:6379
 
-  db:
+  postgres:
     image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     command: ["postgres", "-c", "log_statement=all"]
@@ -18,8 +36,12 @@ services:
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
     driver: local

docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

ee/src/ee-license-check/index.ts

@@ -1,4 +1,5 @@
 import { env } from "../env";
 
 export const isEeAvailable: boolean =
-  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ||
+  env.LANGFUSE_EE_LICENSE_KEY !== undefined;

ee/src/env.ts

@@ -3,6 +3,7 @@ import { env as sharedEnv } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
+  LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
 export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };

ee/src/index.ts

@@ -1 +0,0 @@
-export { isEeAvailable } from "./ee-license-check";

fern/apis/server/definition/prompts.yml

@@ -41,11 +41,17 @@ service:
           limit:
             type: optional<integer>
             docs: limit of items per page
+          fromUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated on or after a certain datetime (ISO 8601)
+          toUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated before a certain datetime (ISO 8601)
 
       response: PromptMetaListResponse
 
     create:
-      docs: Create a prompt
+      docs: Create a new version for the prompt with the given `name`
       method: POST
       path: /prompts
       request: CreatePromptRequest
@@ -63,6 +69,10 @@ types:
       versions: list<integer>
       labels: list<string>
       tags: list<string>
+      lastUpdatedAt: datetime
+      lastConfig:
+        type: unknown
+        docs: Config object of the most recent prompt version that matches the filters (if any are provided)
 
   CreatePromptRequest:
     union:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.67.0",
+  "version": "2.71.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -16,8 +16,9 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",

packages/shared/clickhouse/migrations/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces;

packages/shared/clickhouse/migrations/0001_traces.up.sql

@@ -0,0 +1,26 @@
+CREATE TABLE traces (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(String, String) CODEC(ZSTD(1)),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(1)),
+    `output` Nullable(String) CODEC(ZSTD(1)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/migrations/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations;

packages/shared/clickhouse/migrations/0002_observations.up.sql

@@ -0,0 +1,50 @@
+CREATE TABLE observations (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_input_usage_units` Nullable(Decimal64(12)),
+    `provided_output_usage_units` Nullable(Decimal64(12)),
+    `provided_total_usage_units` Nullable(Decimal64(12)),
+    `input_usage_units` Nullable(Decimal64(12)),
+    `output_usage_units` Nullable(Decimal64(12)),
+    `total_usage_units` Nullable(Decimal64(12)),
+    `unit` Nullable(String),
+    `provided_input_cost` Nullable(Decimal64(12)),
+    `provided_output_cost` Nullable(Decimal64(12)),
+    `provided_total_cost` Nullable(Decimal64(12)),
+    `input_cost` Nullable(Decimal64(12)),
+    `output_cost` Nullable(Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(start_time)
+ORDER BY (
+        project_id,
+        `type`,
+        trace_id,
+        toUnixTimestamp(start_time),
+        id
+    );

packages/shared/clickhouse/migrations/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores;

packages/shared/clickhouse/migrations/0003_scores.up.sql

@@ -0,0 +1,25 @@
+CREATE TABLE scores (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        trace_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/scripts/down.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the down command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down

packages/shared/clickhouse/scripts/drop.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the drop command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/up.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the up command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up

packages/shared/package.json

@@ -38,13 +38,19 @@
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
-    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load"
+    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
+    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
+    "ch:up": "bash clickhouse/scripts/up.sh",
+    "ch:down": "bash clickhouse/scripts/down.sh",
+    "ch:drop": "bash clickhouse/scripts/drop.sh",
+    "ch:reset": "pnpm run ch:down && pnpm run ch:up"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.550.0",
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@aws-sdk/client-s3": "^3.627.0",
     "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@clickhouse/client": "^1.4.0",
@@ -57,6 +63,8 @@
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.4.2",
+    "decimal.js": "^10.4.3",
+    "exponential-backoff": "^3.1.1",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.3",
     "langchain": "^0.2.6",
@@ -64,6 +72,7 @@
     "next-auth": "^4.24.7",
     "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
+    "uuid": "^9.0.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.22.5"
   },

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,14 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const ProjectRole = {
+export const Role = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
-    VIEWER: "VIEWER"
+    VIEWER: "VIEWER",
+    NONE: "NONE"
 } as const;
-export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
+export type Role = (typeof Role)[keyof typeof Role];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -90,8 +91,10 @@ export type AuditLog = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     user_id: string;
-    project_id: string;
-    user_project_role: ProjectRole;
+    org_id: string;
+    user_org_role: string;
+    project_id: string | null;
+    user_project_role: string | null;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -229,9 +232,11 @@ export type LlmApiKeys = {
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: ProjectRole;
-    project_id: string;
-    sender_id: string | null;
+    org_id: string;
+    org_role: Role;
+    project_id: string | null;
+    project_role: Role | null;
+    invited_by_user_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -321,6 +326,21 @@ export type ObservationView = {
     latency: number | null;
     time_to_first_token: number | null;
 };
+export type Organization = {
+    id: string;
+    name: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    cloud_config: unknown | null;
+};
+export type OrganizationMembership = {
+    id: string;
+    org_id: string;
+    user_id: string;
+    role: Role;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
     project_id: string;
     encrypted_posthog_api_key: string;
@@ -331,15 +351,16 @@ export type PosthogIntegration = {
 };
 export type Project = {
     id: string;
+    org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     name: string;
-    cloud_config: unknown | null;
 };
 export type ProjectMembership = {
+    org_membership_id: string;
     project_id: string;
     user_id: string;
-    role: ProjectRole;
+    role: Role;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -484,6 +505,8 @@ export type DB = {
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
+    organization_memberships: OrganizationMembership;
+    organizations: Organization;
     posthog_integrations: PosthogIntegration;
     project_memberships: ProjectMembership;
     projects: Project;

packages/shared/prisma/migrations/20240807111358_models_add_openai_gpt_4o_2024_08_06/migration.sql

@@ -0,0 +1,18 @@
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-2024-08-06
+  ('clzjr85f70000ymmzg7hqffra', NULL, 'gpt-4o-2024-08-06', '(?i)^(gpt-4o-2024-08-06)$', NULL, 0.0000025, 0.000010, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/migrations/20240807111359_add_organizations_main_migration/migration.sql

@@ -0,0 +1,164 @@
+-- Table ORGANIZATIONS
+-- Create empty table ORGANIZATIONS
+CREATE TABLE "organizations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "cloud_config" JSONB,
+
+    CONSTRAINT "organizations_pkey" PRIMARY KEY ("id")
+);
+-- Backfill table ORGANIZATIONS: create a new organization for each project, move cloudConfig, and set the org_id on the project to project_id prefixed with 'o'
+INSERT INTO "organizations" ("id", "name", "cloud_config", "created_at")
+SELECT
+  CONCAT('o', "id") as "id", -- This mapping is used in other migration steps as well, keep it consistent
+  "name",
+  "cloud_config",
+  "created_at"
+FROM "projects";
+-- Drop column cloud_config from projects as it's now on organization level
+ALTER TABLE "projects" DROP COLUMN "cloud_config";
+
+
+-- PROJECT.ORG_ID
+-- Add org_id to projects
+ALTER TABLE "projects" ADD COLUMN "org_id" TEXT;
+-- Backfill: org_id on projects, set it to project_id prefixed with 'o'
+UPDATE "projects"
+SET "org_id" = CONCAT('o', "id");
+-- Set not null after backfill
+ALTER TABLE "projects" ALTER COLUMN "org_id" SET NOT NULL;
+-- AddForeignKey
+ALTER TABLE "projects" ADD CONSTRAINT "projects_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "projects_org_id_idx" ON "projects"("org_id");
+
+
+-- ORGANIZATION MEMBERSHIPS
+-- Create UserRole ENUM
+CREATE TYPE "Role" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER', 'NONE');
+-- Create empty table
+CREATE TABLE "organization_memberships" (
+    "id" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "role" "Role" NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_memberships_pkey" PRIMARY KEY ("id")
+);
+-- Add indexes
+CREATE INDEX "organization_memberships_user_id_idx" ON "organization_memberships"("user_id");
+CREATE INDEX "organization_memberships_org_id_idx" ON "organization_memberships"("org_id");
+CREATE UNIQUE INDEX "organization_memberships_org_id_user_id_key" ON "organization_memberships"("org_id", "user_id");
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+
+
+-- Migrate project memberships to organization memberships
+INSERT INTO "organization_memberships" ("id", "org_id", "user_id", "role", "created_at", "updated_at")
+SELECT
+  md5(random()::text || clock_timestamp()::text || project_id::text || user_id::text)::uuid AS "id",
+  CONCAT('o', "project_id") as "org_id",
+  "user_id",
+  "role"::text::"Role" as "role",
+  "created_at",
+  "updated_at"
+FROM "project_memberships";
+
+
+
+-- Delete all project memberships after migration to organization memberships
+DELETE FROM "project_memberships";
+
+-- Add org_membership_id to project_memberships, ok to be not null as it's a new column on a now empty table
+ALTER TABLE "project_memberships" ADD COLUMN     "org_membership_id" TEXT NOT NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "project_memberships" DROP COLUMN "role",
+ADD COLUMN     "role" "Role" NOT NULL;
+-- AddForeignKey
+ALTER TABLE "project_memberships" ADD CONSTRAINT "project_memberships_org_membership_id_fkey" FOREIGN KEY ("org_membership_id") REFERENCES "organization_memberships"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "project_memberships_project_id_idx" ON "project_memberships"("project_id");
+-- CreateIndex
+CREATE INDEX "project_memberships_org_membership_id_idx" ON "project_memberships"("org_membership_id");
+
+
+
+-- AUDIT LOGS
+-- DropForeignKey, just index on these, should remain after project/org/user deletions
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_user_id_fkey";
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_project_id_fkey";
+-- Drop not null on project level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" DROP NOT NULL;
+-- Add org-level cols
+ALTER TABLE "audit_logs" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "audit_logs" ADD COLUMN "user_org_role" TEXT;
+-- Backfill org_id
+UPDATE "audit_logs"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "audit_logs"."project_id" = "projects"."id";
+-- Backfill user_org_role with value from user_project_role
+UPDATE "audit_logs"
+SET "user_org_role" = "user_project_role"::text::"Role";
+-- Drop and recreate user_project_role column as text column going forward, empty for historical data as it's all on org level now
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role" TEXT; -- nullable
+-- Add not null on org level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "org_id" SET NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "audit_logs_user_id_idx" ON "audit_logs"("user_id");
+CREATE INDEX "audit_logs_org_id_idx" ON "audit_logs"("org_id");
+
+
+-- MEMBERSHIP INVITATIONS
+-- Rename sender_id to invited_by_user_id
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_sender_id_fkey";
+ALTER TABLE "membership_invitations" RENAME COLUMN "sender_id" TO "invited_by_user_id";
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_invited_by_user_id_fkey" FOREIGN KEY ("invited_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+-- DropForeignKey
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_project_id_fkey";
+-- AlterTable
+ALTER TABLE "membership_invitations" RENAME COLUMN "role" TO "project_role";
+ALTER TABLE "membership_invitations" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "membership_invitations" ADD COLUMN "org_role" "Role";
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_role" DROP NOT NULL;
+-- Backfill org id
+UPDATE "membership_invitations"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "membership_invitations"."project_id" = "projects"."id";
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- Backfill org role with value from project role
+UPDATE "membership_invitations"
+SET "org_role" = "project_role"::text::"Role";
+-- Set project-level cols to null, as it's now org level for all existing invitations and role enum will change below
+UPDATE "membership_invitations"
+SET "project_role" = NULL, "project_id" = NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "membership_invitations" DROP COLUMN "project_role",
+ADD COLUMN     "project_role" "Role";
+
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- Add not null on org level cols after backfill
+ALTER TABLE "membership_invitations"
+ALTER COLUMN "org_id" SET NOT NULL,
+ALTER COLUMN "org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "membership_invitations_org_id_idx" ON "membership_invitations"("org_id");
+
+
+-- Drop ProjectRole enum as it is replaced by Role
+DROP TYPE "ProjectRole";

packages/shared/prisma/migrations/20240814223824_model_fix_text_embedding_3_large/migration.sql

@@ -0,0 +1,7 @@
+
+-- Fix model name
+-- https://github.com/langfuse/langfuse/issues/2688
+
+UPDATE models
+SET model_name = 'text-embedding-3-large'
+WHERE id = 'clruwn76700020al7gp8e4g4l'

packages/shared/prisma/migrations/20240814233029_dataset_items_drop_fkey_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_trace_id_fkey";

packages/shared/prisma/schema.prisma

@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id                 String                 @id @default(cuid())
-    name               String?
-    email              String?                @unique
-    emailVerified      DateTime?              @map("email_verified")
-    password           String?
-    image              String?
-    admin              Boolean                @default(false)
-    accounts           Account[]
-    sessions           Session[]
-    projectMemberships ProjectMembership[]
-    invitations        MembershipInvitation[]
-    createdAt          DateTime               @default(now()) @map("created_at")
-    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags       String[]               @default([]) @map("feature_flags")
-    AuditLog           AuditLog[]
+    id                      String                   @id @default(cuid())
+    name                    String?
+    email                   String?                  @unique
+    emailVerified           DateTime?                @map("email_verified")
+    password                String?
+    image                   String?
+    admin                   Boolean                  @default(false)
+    accounts                Account[]
+    sessions                Session[]
+    organizationMemberships OrganizationMembership[]
+    projectMemberships      ProjectMembership[]
+    invitations             MembershipInvitation[]
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    featureFlags            String[]                 @default([]) @map("feature_flags")
 
     @@map("users")
 }
@@ -91,13 +91,27 @@ model VerificationToken {
     @@map("verification_tokens")
 }
 
+model Organization {
+    id                      String                   @id @default(cuid())
+    name                    String
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+    organizationMemberships OrganizationMembership[]
+    projects                Project[]
+    MembershipInvitation    MembershipInvitation[]
+
+    @@map("organizations")
+}
+
 model Project {
     id                 String                 @id @default(cuid())
+    orgId              String                 @map("org_id")
     createdAt          DateTime               @default(now()) @map("created_at")
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
-    cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
     projectMembers     ProjectMembership[]
+    organization       Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -107,7 +121,6 @@ model Project {
     sessions           TraceSession[]
     Prompt             Prompt[]
     Model              Model[]
-    AuditLog           AuditLog[]
     EvalTemplate       EvalTemplate[]
     JobConfiguration   JobConfiguration[]
     JobExecution       JobExecution[]
@@ -117,6 +130,7 @@ model Project {
     scoreConfig        ScoreConfig[]
     BatchExport        BatchExport[]
 
+    @@index([orgId])
     @@map("projects")
 }
 
@@ -161,41 +175,68 @@ model LlmApiKeys {
     @@map("llm_api_keys")
 }
 
+model OrganizationMembership {
+    id                 String              @id @default(cuid())
+    orgId              String              @map("org_id")
+    organization       Organization        @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    userId             String              @map("user_id")
+    user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role               Role                @map("role")
+    createdAt          DateTime            @default(now()) @map("created_at")
+    updatedAt          DateTime            @default(now()) @updatedAt @map("updated_at")
+    ProjectMemberships ProjectMembership[]
+
+    @@unique([orgId, userId])
+    @@index([userId])
+    @@index([orgId])
+    @@map("organization_memberships")
+}
+
+// Set a project-specific role for a user in an organization
 model ProjectMembership {
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String      @map("user_id")
-    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      ProjectRole
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    orgMembershipId        String                 @map("org_membership_id")
+    organizationMembership OrganizationMembership @relation(fields: [orgMembershipId], references: [id], onDelete: Cascade)
+    projectId              String                 @map("project_id")
+    project                Project                @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId                 String                 @map("user_id")
+    user                   User                   @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role                   Role
+    createdAt              DateTime               @default(now()) @map("created_at")
+    updatedAt              DateTime               @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
+    @@index([projectId])
+    @@index([orgMembershipId])
     @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String      @id @unique @default(cuid())
-    email     String
-    role      ProjectRole
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?     @map("sender_id")
-    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    id              String       @id @unique @default(cuid())
+    email           String
+    orgId           String       @map("org_id")
+    organization    Organization @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    orgRole         Role         @map("org_role")
+    projectId       String?      @map("project_id")
+    project         Project?     @relation(fields: [projectId], references: [id], onDelete: SetNull)
+    projectRole     Role?        @map("project_role")
+    invitedByUserId String?      @map("invited_by_user_id")
+    invitedByUser   User?        @relation(fields: [invitedByUserId], references: [id], onDelete: SetNull)
+    createdAt       DateTime     @default(now()) @map("created_at")
+    updatedAt       DateTime     @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
+    @@index([orgId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum ProjectRole {
+enum Role {
     OWNER
     ADMIN
     MEMBER
     VIEWER
+    NONE
 }
 
 model TraceSession {
@@ -238,7 +279,6 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetItem  DatasetItem[]
     JobExecution JobExecution[]
 
     @@index([projectId])
@@ -323,9 +363,8 @@ model Observation {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
 
-    completionStartTime DateTime?     @map("completion_start_time")
-    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    derivedDatasetItems DatasetItem[]
+    completionStartTime DateTime? @map("completion_start_time")
+    project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     promptId String? @map("prompt_id") // no fk constraint, prompt can be deleted
 
@@ -518,9 +557,7 @@ model DatasetItem {
     expectedOutput      Json?             @map("expected_output")
     metadata            Json?
     sourceTraceId       String?           @map("source_trace_id")
-    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
     sourceObservationId String?           @map("source_observation_id")
-    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
     datasetId           String            @map("dataset_id")
     dataset             Dataset           @relation(fields: [datasetId, projectId], references: [id, projectId], onDelete: Cascade)
     createdAt           DateTime          @default(now()) @map("created_at")
@@ -653,22 +690,25 @@ model Model {
     @@map("models")
 }
 
+// No FK constraints to preserve audit logs
 model AuditLog {
-    id              String      @id @default(cuid())
-    createdAt       DateTime    @default(now()) @map("created_at")
-    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
-    userId          String      @map("user_id")
-    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String      @map("project_id")
-    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole ProjectRole @map("user_project_role")
-    resourceType    String      @map("resource_type")
-    resourceId      String      @map("resource_id")
+    id              String   @id @default(cuid())
+    createdAt       DateTime @default(now()) @map("created_at")
+    updatedAt       DateTime @default(now()) @updatedAt @map("updated_at")
+    userId          String   @map("user_id")
+    orgId           String   @map("org_id")
+    userOrgRole     String   @map("user_org_role")
+    projectId       String?  @map("project_id")
+    userProjectRole String?  @map("user_project_role")
+    resourceType    String   @map("resource_type")
+    resourceId      String   @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
 
     @@index([projectId])
+    @@index([userId])
+    @@index([orgId])
     @@index([createdAt])
     @@index([updatedAt])
     @@map("audit_logs")

packages/shared/prisma/seed.ts

@@ -34,39 +34,111 @@ async function main() {
     options,
   }).values.environment;
 
+  const seedOrgId = "seed-org-id";
+  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  const seedUserId1 = "user-1"; // Owner of org
+  const seedUserId2 = "user-2"; // Member of org, admin of project
+
   const user = await prisma.user.upsert({
-    where: { id: "user-1" },
+    where: { id: seedUserId1 },
     update: {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
     },
     create: {
-      id: "user-1",
+      id: seedUserId1,
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
       image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
+  const user2 = await prisma.user.upsert({
+    where: { id: seedUserId2 },
+    update: {
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId2,
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+  });
 
-  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  await prisma.organization.upsert({
+    where: { id: seedOrgId },
+    update: {
+      name: "Seed Org",
+    },
+    create: {
+      id: seedOrgId,
+      name: "Seed Org",
+    },
+  });
 
   const project1 = await prisma.project.upsert({
     where: { id: seedProjectId },
     update: {
       name: "llm-app",
+      orgId: seedOrgId,
     },
     create: {
       id: seedProjectId,
       name: "llm-app",
-      projectMembers: {
-        create: {
-          role: "OWNER",
-          userId: user.id,
-        },
+      orgId: seedOrgId,
+    },
+  });
+
+  const orgMembership = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user.id,
+        orgId: seedOrgId,
       },
     },
+    create: {
+      userId: user.id,
+      orgId: seedOrgId,
+      role: "OWNER",
+    },
+    update: {},
+  });
+
+  const orgMembership2 = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user2.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user2.id,
+      orgId: seedOrgId,
+      role: "MEMBER",
+    },
+    update: {},
+  });
+
+  const projectMembership = await prisma.projectMembership.upsert({
+    where: {
+      projectId_userId: {
+        projectId: project1.id,
+        userId: user2.id,
+      },
+    },
+    create: {
+      userId: user2.id,
+      projectId: project1.id,
+      role: "ADMIN",
+      orgMembershipId: orgMembership2.id,
+    },
+    update: {
+      orgMembershipId: orgMembership2.id,
+    },
   });
 
   await prisma.prompt.upsert({
@@ -114,18 +186,39 @@ async function main() {
 
   // Do not run the following for local docker compose setup
   if (environment === "examples" || environment === "load") {
-    const project2 = await prisma.project.upsert({
-      where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+    const seedOrgIdOrg2 = "demo-org-id";
+    const project2Id = "239ad00f-562f-411d-af14-831c75ddd875";
+    const org2 = await prisma.organization.upsert({
+      where: { id: seedOrgIdOrg2 },
+      update: {
+        name: "Langfuse Demo",
+      },
       create: {
-        id: "239ad00f-562f-411d-af14-831c75ddd875",
+        id: seedOrgIdOrg2,
+        name: "Langfuse Demo",
+      },
+    });
+    const project2 = await prisma.project.upsert({
+      where: { id: project2Id },
+      create: {
+        id: project2Id,
         name: "demo-app",
-        projectMembers: {
-          create: {
-            role: "OWNER",
-            userId: user.id,
-          },
+        orgId: org2.id,
+      },
+      update: { orgId: seedOrgIdOrg2 },
+    });
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: {
+          userId: user.id,
+          orgId: seedOrgIdOrg2,
         },
       },
+      create: {
+        userId: user.id,
+        orgId: seedOrgIdOrg2,
+        role: "VIEWER",
+      },
       update: {},
     });
 
@@ -773,6 +866,7 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -782,6 +876,7 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {

packages/shared/src/encryption/index.ts

@@ -1,7 +1,7 @@
 import crypto from "crypto";
 import { env } from "../env";
 
-const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const ENCRYPTION_KEY: string | undefined = env.ENCRYPTION_KEY; // Must be 256 bits (32 bytes, 64 hex characters)
 const IV_LENGTH: number = 16; // For AES, this is always 16
 
 // Alternatively: openssl rand -hex 32
@@ -13,17 +13,16 @@ export function keyGen() {
  * Encrypts the given plain text using AES-256-GCM algorithm.
  *
  * @param {string} plainText - The text to encrypt.
- * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
  * @returns {string} The encrypted data in hex format, including IV and authentication tag.
  */
-export function encrypt(
-  plainText: string,
-  keyHex: string = ENCRYPTION_KEY
-): string {
+export function encrypt(plainText: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
   const cipher = crypto.createCipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   let encrypted = cipher.update(plainText, "utf8", "hex");
@@ -34,7 +33,10 @@ export function encrypt(
   return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
 }
 
-export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+export function decrypt(text: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const [ivHex, encryptedHex, authTagHex] = text.split(":");
   if (!ivHex || !encryptedHex || !authTagHex) {
     throw new Error("Invalid or corrupted cipher format");
@@ -46,7 +48,7 @@ export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
 
   const decipher = crypto.createDecipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   decipher.setAuthTag(authTag);

packages/shared/src/env.ts

@@ -28,6 +28,15 @@ const EnvSchema = z.object({
   CLICKHOUSE_URL: z.string().url().optional(),
   CLICKHOUSE_USER: z.string().optional(),
   CLICKHOUSE_PASSWORD: z.string().optional(),
+  LANGFUSE_INGESTION_FLUSH_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(10000),
+  LANGFUSE_INGESTION_FLUSH_ATTEMPTS: z.coerce.number().positive().default(3),
+  LANGFUSE_INGESTION_BUFFER_TTL_SECONDS: z.coerce
+    .number()
+    .positive()
+    .default(60 * 10),
 });
 
 export const env = EnvSchema.parse(process.env);

packages/shared/src/features/evals/types.ts

@@ -81,6 +81,7 @@ export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
   [LLMAdapter.OpenAI]: [
     "gpt-4o",
     "gpt-4o-mini",
+    "gpt-4o-2024-08-06", // cheaper than gpt-4o
     "gpt-4-turbo-preview",
     "gpt-3.5-turbo",
   ] as const,

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -1,12 +1,13 @@
-import { isPresent } from "@/src/utils/typeChecks";
+import { z } from "zod";
+
+import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
+
+import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
   paginationZod,
-  type ScoreConfig as ScoreConfigDbType,
-} from "@langfuse/shared";
-import { z } from "zod";
-import * as Sentry from "@sentry/node";
+} from "../../utils/zod";
 
 /**
  * Types to use across codebase
@@ -16,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx,
+  ctx: z.RefinementCtx
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -93,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value,
+          category.value === expectedCategories[index].value
       );
     }),
 });
@@ -122,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      }),
+      })
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -149,13 +150,14 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
+  onParseError?: (error: z.ZodError) => void
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
     if (result.success) {
       acc.push(result.data);
     } else {
-      Sentry.captureException(result.error);
+      onParseError?.(result.error);
     }
     return acc;
   }, [] as ValidatedScoreConfig[]);
@@ -168,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType,
+  scoreConfig: ScoreConfigDbType
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -203,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      }),
+      })
     ),
   ])
   .superRefine((data, ctx) => {

packages/shared/src/features/scores/scoreTypes.ts

@@ -1,19 +1,19 @@
-import * as Sentry from "@sentry/node";
-import {
-  paginationZod,
-  paginationMetaResponseZod,
-  NonEmptyString,
-  type Score,
-  stringDateTime,
-} from "@langfuse/shared";
 import { z } from "zod";
-import { isPresent } from "@/src/utils/typeChecks";
-import { Category as ConfigCategory } from "./score-configs";
+
+import { Score } from "@prisma/client";
+
+import { isPresent, stringDateTime } from "../../utils/typeChecks";
+import {
+  NonEmptyString,
+  paginationMetaResponseZod,
+  paginationZod,
+} from "../../utils/zod";
+import { Category as ConfigCategory } from "./scoreConfigTypes";
 
 /**
  * Types to use across codebase
  */
-export type APIScore = z.infer<typeof APIScore>;
+export type APIScore = z.infer<typeof APIScoreSchema>;
 
 /**
  * Helpers
@@ -69,7 +69,7 @@ const BaseScoreBody = z.object({
  * Objects
  */
 
-export const APIScore = z.discriminatedUnion("dataType", [
+export const APIScoreSchema = z.discriminatedUnion("dataType", [
   ScoreBase.merge(NumericData),
   ScoreBase.merge(CategoricalData),
   ScoreBase.merge(BooleanData),
@@ -83,13 +83,13 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
     z.object({
       value: z.number(),
       dataType: z.literal("NUMERIC"),
-    }),
+    })
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
-    }),
+    })
   ),
   BaseScoreBody.merge(
     z.object({
@@ -97,7 +97,7 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
         message: "Value must be either 0 or 1",
       }),
       dataType: z.literal("BOOLEAN"),
-    }),
+    })
   ),
 ]);
 
@@ -159,14 +159,17 @@ export const ScorePropsAgainstConfig = z.union([
  * @param scores
  * @returns list of validated scores
  */
-export const filterAndValidateDbScoreList = (scores: Score[]): APIScore[] =>
+export const filterAndValidateDbScoreList = (
+  scores: Score[],
+  onParseError?: (error: z.ZodError) => void
+): APIScore[] =>
   scores.reduce((acc, ts) => {
-    const result = APIScore.safeParse(ts);
+    const result = APIScoreSchema.safeParse(ts);
     if (result.success) {
       acc.push(result.data);
     } else {
       console.error("Score parsing error: ", result.error);
-      Sentry.captureException(result.error);
+      onParseError?.(result.error);
     }
     return acc;
   }, [] as APIScore[]);
@@ -179,7 +182,7 @@ export const filterAndValidateDbScoreList = (scores: Score[]): APIScore[] =>
  * @throws error if score fails validation
  */
 export const validateDbScore = (score: Score): APIScore =>
-  APIScore.parse(score);
+  APIScoreSchema.parse(score);
 
 /**
  * Endpoints
@@ -195,14 +198,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       value: z.number(),
       dataType: z.literal("NUMERIC"),
       configId: z.string().nullish(),
-    }),
+    })
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
       configId: z.string().nullish(),
-    }),
+    })
   ),
   BaseScoreBody.merge(
     z.object({
@@ -212,14 +215,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       }),
       dataType: z.literal("BOOLEAN"),
       configId: z.string().nullish(),
-    }),
+    })
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.union([z.string(), z.number()]),
       dataType: z.undefined(),
       configId: z.string().nullish(),
-    }),
+    })
   ),
 ]);
 
@@ -248,12 +251,12 @@ export const GetScoresQuery = z.object({
 
 // LegacyGetScoreResponseDataV1 is only used for response of GET /scores list endpoint
 const LegacyGetScoreResponseDataV1 = z.intersection(
-  APIScore,
+  APIScoreSchema,
   z.object({
     trace: z.object({
       userId: z.string().nullish(),
     }),
-  }),
+  })
 );
 export const GetScoresResponse = z.object({
   data: z.array(LegacyGetScoreResponseDataV1),
@@ -262,6 +265,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
+  onParseError?: (error: z.ZodError) => void
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
     (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
@@ -270,11 +274,11 @@ export const legacyFilterAndValidateV1GetScoreList = (
         acc.push(result.data);
       } else {
         console.error("Score parsing error: ", result.error);
-        Sentry.captureException(result.error);
+        onParseError?.(result.error);
       }
       return acc;
     },
-    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[],
+    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[]
   );
 
 // GET /scores/{scoreId}
@@ -282,7 +286,7 @@ export const GetScoreQuery = z.object({
   scoreId: z.string(),
 });
 
-export const GetScoreResponse = APIScore;
+export const GetScoreResponse = APIScoreSchema;
 
 // DELETE /scores/{scoreId}
 export const DeleteScoreQuery = z.object({

packages/shared/src/index.ts

@@ -1,6 +1,5 @@
 export * from "./constants";
 export * from "./queries";
-export * from "./queues";
 export * from "./interfaces/filters";
 export * from "./interfaces/orderBy";
 export * from "./tableDefinitions";
@@ -8,12 +7,12 @@ export * from "./types";
 export * from "./filterToPrisma";
 export * from "./orderByToPrisma";
 export * from "./tracesTable";
-export * from "./server/auth";
+export * from "./server/auth/auth";
 export * from "./observationsTable";
-export * from "./features/ingestion/types";
 export * from "./utils/zod";
 export * from "./utils/json";
 export * from "./utils/objects";
+export * from "./utils/typeChecks";
 export { env } from "./env";
 
 // llm api
@@ -27,6 +26,10 @@ export * from "./features/batchExport/types";
 // annotation
 export * from "./features/annotation/types";
 
+// scores
+export * from "./features/scores/scoreConfigTypes";
+export * from "./features/scores/scoreTypes";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";

packages/shared/src/server/auth/types.ts

@@ -0,0 +1,16 @@
+export type AuthHeaderVerificationResult =
+  | AuthHeaderValidVerificationResult
+  | {
+      validKey: false;
+      error: string;
+    };
+
+export type AuthHeaderValidVerificationResult = {
+  validKey: true;
+  scope: ApiAccessScope;
+};
+
+export type ApiAccessScope = {
+  projectId: string;
+  accessLevel: "all" | "scores";
+};

packages/shared/src/server/definitions.ts

@@ -0,0 +1,167 @@
+import z from "zod";
+
+export const clickhouseStringDateSchema = z
+  .string()
+  // clickhouse stores UTC like '2024-05-23 18:33:41.602000'
+  // we need to convert it to '2024-05-23T18:33:41.602000Z'
+  .transform((str) => str.replace(" ", "T") + "Z")
+  .pipe(z.string().datetime());
+
+export const observationRecordBaseSchema = z.object({
+  id: z.string(),
+  trace_id: z.string().nullish(),
+  project_id: z.string(),
+  type: z.string(),
+  parent_observation_id: z.string().nullish(),
+  name: z.string().nullish(),
+  metadata: z.record(z.string()),
+  level: z.string().nullish(),
+  status_message: z.string().nullish(),
+  version: z.string().nullish(),
+  input: z.string().nullish(),
+  output: z.string().nullish(),
+  provided_model_name: z.string().nullish(),
+  internal_model_id: z.string().nullish(),
+  model_parameters: z.string().nullish(),
+  unit: z.string().nullish(),
+  input_usage_units: z.number().nullish(),
+  output_usage_units: z.number().nullish(),
+  total_usage_units: z.number().nullish(),
+  input_cost: z.number().nullish(),
+  output_cost: z.number().nullish(),
+  total_cost: z.number().nullish(),
+  provided_input_usage_units: z.number().nullish(),
+  provided_output_usage_units: z.number().nullish(),
+  provided_total_usage_units: z.number().nullish(),
+  provided_input_cost: z.number().nullish(),
+  provided_output_cost: z.number().nullish(),
+  provided_total_cost: z.number().nullish(),
+  prompt_id: z.string().nullish(),
+  prompt_name: z.string().nullish(),
+  prompt_version: z.number().nullish(),
+});
+export type ObservationRecordBaseType = z.infer<
+  typeof observationRecordBaseSchema
+>;
+
+export const observationRecordReadSchema = observationRecordBaseSchema.extend({
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+  start_time: clickhouseStringDateSchema,
+  end_time: clickhouseStringDateSchema.nullish(),
+  completion_start_time: clickhouseStringDateSchema.nullish(),
+});
+export type ObservationRecordReadType = z.infer<
+  typeof observationRecordReadSchema
+>;
+
+export const observationRecordInsertSchema = observationRecordBaseSchema.extend(
+  {
+    created_at: z.number(),
+    updated_at: z.number(),
+    start_time: z.number(),
+    end_time: z.number().nullish(),
+    completion_start_time: z.number().nullish(),
+  }
+);
+export type ObservationRecordInsertType = z.infer<
+  typeof observationRecordInsertSchema
+>;
+
+export const traceRecordBaseSchema = z.object({
+  id: z.string(),
+  name: z.string().nullish(),
+  user_id: z.string().nullish(),
+  metadata: z.record(z.string()),
+  release: z.string().nullish(),
+  version: z.string().nullish(),
+  project_id: z.string(),
+  public: z.boolean(),
+  bookmarked: z.boolean(),
+  tags: z.array(z.string()),
+  input: z.string().nullish(),
+  output: z.string().nullish(),
+  session_id: z.string().nullish(),
+});
+export type TraceRecordBaseType = z.infer<typeof traceRecordBaseSchema>;
+
+export const traceRecordReadSchema = traceRecordBaseSchema.extend({
+  timestamp: clickhouseStringDateSchema,
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+});
+export type TraceRecordReadType = z.infer<typeof traceRecordReadSchema>;
+
+export const traceRecordInsertSchema = traceRecordBaseSchema.extend({
+  timestamp: z.number(),
+  created_at: z.number(),
+  updated_at: z.number(),
+});
+export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;
+
+export const scoreRecordBaseSchema = z.object({
+  id: z.string(),
+  project_id: z.string(),
+  trace_id: z.string(),
+  observation_id: z.string().nullish(),
+  name: z.string().nullish(),
+  value: z.union([z.number(), z.string()]).nullish(),
+  source: z.string(),
+  comment: z.string().nullish(),
+  author_user_id: z.string().nullish(),
+  config_id: z.string().nullish(),
+  data_type: z.enum(["NUMERIC", "CATEGORICAL", "BOOLEAN"]).nullish(),
+  string_value: z.string().nullish(),
+});
+export type ScoreRecordBaseType = z.infer<typeof scoreRecordBaseSchema>;
+
+export const scoreRecordReadSchema = scoreRecordBaseSchema.extend({
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+  timestamp: clickhouseStringDateSchema,
+});
+export type ScoreRecordReadType = z.infer<typeof scoreRecordReadSchema>;
+
+export const scoreRecordInsertSchema = scoreRecordBaseSchema.extend({
+  created_at: z.number(),
+  updated_at: z.number(),
+  timestamp: z.number(),
+});
+export type ScoreRecordInsertType = z.infer<typeof scoreRecordInsertSchema>;
+
+export const convertTraceReadToInsert = (
+  record: TraceRecordReadType
+): TraceRecordInsertType => {
+  return {
+    ...record,
+    created_at: new Date(record.created_at).getTime(),
+    updated_at: new Date(record.created_at).getTime(),
+    timestamp: new Date(record.timestamp).getTime(),
+  };
+};
+
+export const convertObservationReadToInsert = (
+  record: ObservationRecordReadType
+): ObservationRecordInsertType => {
+  return {
+    ...record,
+    created_at: new Date(record.created_at).getTime(),
+    updated_at: new Date(record.created_at).getTime(),
+    start_time: new Date(record.start_time).getTime(),
+    end_time: record.end_time ? new Date(record.end_time).getTime() : undefined,
+    completion_start_time: record.completion_start_time
+      ? new Date(record.completion_start_time).getTime()
+      : undefined,
+  };
+};
+
+export const convertScoreReadToInsert = (
+  record: ScoreRecordReadType
+): ScoreRecordInsertType => {
+  return {
+    ...record,
+    created_at: new Date(record.created_at).getTime(),
+    updated_at: new Date(record.updated_at).getTime(),
+    timestamp: new Date(record.timestamp).getTime(),
+  };
+};

packages/shared/src/server/index.ts

@@ -1,13 +1,26 @@
 export * from "./services/S3StorageService";
-export * from "./services/email/projectInvitation/sendProjectInvitationEmail";
+export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
-export * from "./auth";
+export * from "./auth/auth";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
+export * from "./clickhouse";
+export * from "../server/definitions";
+export * from "../server/ingestion/IngestionUtils";
+export * from "../server/ingestion/types";
+export * from "../server/ingestion/model-match";
+export * from "../server/ingestion/types";
+export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/trace-upsert";
-export * from "./clickhouse";
+export * from "./redis/batch-export";
+export * from "./redis/ingestionFlushQueue";
+export * from "./redis/legacy-ingestion";
+export * from "./auth/types";
+export * from "./ingestion/legacy/index";
+export * from "./queues";
+export * from "./ingestion/legacy/EventProcessor";

packages/shared/src/server/ingestion/IngestionUtils.ts

@@ -0,0 +1,86 @@
+import { eventTypes, IngestionEventType } from "./types";
+
+const reservedCharsEscapeMap = [
+  { reserved: ":", escape: "|%|" },
+  { reserved: "_", escape: "|#|" },
+];
+
+export enum ClickhouseEntityType {
+  Trace = "trace",
+  Score = "score",
+  Observation = "observation",
+  SDK_LOG = "sdk-log",
+}
+
+export class IngestionUtils {
+  public static getBufferKey(flushKey: string): string {
+    const { projectId, eventType, entityId } =
+      IngestionUtils.parseFlushKey(flushKey);
+
+    const sanitizedEntityId = IngestionUtils.escapeReservedChars(entityId);
+
+    return (
+      "ingestionBuffer:" + `${projectId}_${eventType}_${sanitizedEntityId}`
+    );
+  }
+
+  public static getFlushKey(params: {
+    projectId: string;
+    eventType: ClickhouseEntityType;
+    entityId: string;
+    batchTimestamp: string;
+  }): string {
+    const { projectId, eventType, entityId, batchTimestamp } = params;
+    const sanitizedEntityId = IngestionUtils.escapeReservedChars(entityId);
+
+    return `${projectId}_${eventType}_${sanitizedEntityId}_${batchTimestamp}`;
+  }
+
+  public static parseFlushKey(projectEntityKey: string) {
+    const split = projectEntityKey.split("_");
+
+    if (split.length < 3) {
+      throw new Error(
+        `Invalid project entity key format ${projectEntityKey}, expected 3 or 4 parts`
+      );
+    }
+
+    const [projectId, eventType, escapedEntityId, batchTimestamp] = split;
+    const entityId = IngestionUtils.unescapeReservedChars(escapedEntityId);
+
+    return { projectId, eventType, entityId, batchTimestamp };
+  }
+
+  private static escapeReservedChars(string: string): string {
+    return reservedCharsEscapeMap.reduce(
+      (acc, { reserved, escape }) => acc.replaceAll(reserved, escape),
+      string
+    );
+  }
+
+  private static unescapeReservedChars(escapedString: string): string {
+    return reservedCharsEscapeMap.reduce(
+      (acc, { reserved, escape }) => acc.replaceAll(escape, reserved),
+      escapedString
+    );
+  }
+
+  public static getEventType(event: IngestionEventType): ClickhouseEntityType {
+    switch (event.type) {
+      case eventTypes.TRACE_CREATE:
+        return ClickhouseEntityType.Trace;
+      case eventTypes.OBSERVATION_CREATE:
+      case eventTypes.OBSERVATION_UPDATE:
+      case eventTypes.EVENT_CREATE:
+      case eventTypes.SPAN_CREATE:
+      case eventTypes.SPAN_UPDATE:
+      case eventTypes.GENERATION_CREATE:
+      case eventTypes.GENERATION_UPDATE:
+        return ClickhouseEntityType.Observation;
+      case eventTypes.SCORE_CREATE:
+        return ClickhouseEntityType.Score;
+      case eventTypes.SDK_LOG:
+        return ClickhouseEntityType.SDK_LOG;
+    }
+  }
+}

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -1,131 +1,53 @@
-import { tokenCount } from "@/src/features/ingest/lib/usage";
-import { type ApiAccessScope } from "@/src/features/public-api/server/types";
-import {
-  type legacyObservationCreateEvent,
-  eventTypes,
-  type scoreEvent,
-  type eventCreateEvent,
-  type spanCreateEvent,
-  type generationCreateEvent,
-  type spanUpdateEvent,
-  type generationUpdateEvent,
-  type legacyObservationUpdateEvent,
-  type sdkLogEvent,
-  type traceEvent,
-  LangfuseNotFoundError,
-  InvalidRequestError,
-} from "@langfuse/shared";
-import { ScoreDataType, prisma } from "@langfuse/shared/src/db";
-import { ResourceNotFoundError } from "@/src/utils/exceptions";
-import { mergeJson } from "@langfuse/shared";
-import {
-  type Trace,
-  type Observation,
-  type Score,
-  Prisma,
-  type Model,
-} from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
 import { type z } from "zod";
-import { jsonSchema } from "@langfuse/shared";
-import { ForbiddenError } from "@langfuse/shared";
-import { instrument } from "@/src/utils/instrumentation";
 import Decimal from "decimal.js";
+import { findModel } from "../model-match";
 import {
-  ScoreBodyWithoutConfig,
-  ScorePropsAgainstConfig,
-} from "@/src/features/public-api/types/scores";
-import {
-  validateDbScoreConfigSafe,
-  type ValidatedScoreConfig,
-} from "@/src/features/public-api/types/score-configs";
+  ObservationEvent,
+  eventTypes,
+  legacyObservationCreateEvent,
+  generationCreateEvent,
+  traceEvent,
+  scoreEvent,
+  sdkLogEvent,
+} from "../types";
+import { validateAndInflateScore } from "../validateAndInflateScore";
+import { ApiAccessScope } from "../../auth/types";
+import { Trace, Observation, Score, Prisma, Model } from "@prisma/client";
+import { ForbiddenError, LangfuseNotFoundError } from "../../../errors";
+import { mergeJson } from "../../../utils/json";
+import { jsonSchema } from "../../../utils/zod";
+import { prisma } from "../../../db";
 
 export interface EventProcessor {
+  auth(apiScope: ApiAccessScope): void;
+
   process(
-    apiScope: ApiAccessScope,
+    apiScope: ApiAccessScope
   ): Promise<Trace | Observation | Score> | undefined;
 }
 
-export async function findModel(p: {
-  event: {
-    projectId: string;
-    model?: string;
-    unit?: string;
-    startTime?: Date;
-  };
-  existingDbObservation?: Observation;
-}): Promise<Model | null> {
-  const { event, existingDbObservation } = p;
-  // either get the model from the existing observation
-  // or match pattern on the user provided model name
-  const modelCondition = event.model
-    ? Prisma.sql`AND ${event.model} ~ match_pattern`
-    : existingDbObservation?.internalModel
-      ? Prisma.sql`AND model_name = ${existingDbObservation.internalModel}`
-      : undefined;
-  if (!modelCondition) return null;
-
-  // unit based on the current event or the existing observation, both can be undefined
-  const mergedUnit = event.unit ?? existingDbObservation?.unit;
-
-  const unitCondition = mergedUnit
-    ? Prisma.sql`AND unit = ${mergedUnit}`
-    : Prisma.empty;
-
-  const sql = Prisma.sql`
-    SELECT
-      id,
-      created_at AS "createdAt",
-      updated_at AS "updatedAt",
-      project_id AS "projectId",
-      model_name AS "modelName",
-      match_pattern AS "matchPattern",
-      start_date AS "startDate",
-      input_price AS "inputPrice",
-      output_price AS "outputPrice",
-      total_price AS "totalPrice",
-      unit, 
-      tokenizer_id AS "tokenizerId",
-      tokenizer_config AS "tokenizerConfig"
-    FROM
-      models
-    WHERE (project_id = ${event.projectId}
-      OR project_id IS NULL)
-    ${modelCondition}
-    ${unitCondition}
-    AND (start_date IS NULL OR start_date <= ${
-      event.startTime ? new Date(event.startTime) : new Date()
-    }::timestamp with time zone at time zone 'UTC')
-    ORDER BY
-      project_id ASC,
-      start_date DESC NULLS LAST
-    LIMIT 1
-  `;
-
-  const foundModels = await prisma.$queryRaw<Array<Model>>(sql);
-
-  return foundModels[0] ?? null;
-}
-
-type ObservationEvent =
-  | z.infer<typeof legacyObservationCreateEvent>
-  | z.infer<typeof legacyObservationUpdateEvent>
-  | z.infer<typeof eventCreateEvent>
-  | z.infer<typeof spanCreateEvent>
-  | z.infer<typeof spanUpdateEvent>
-  | z.infer<typeof generationCreateEvent>
-  | z.infer<typeof generationUpdateEvent>;
-
 export class ObservationProcessor implements EventProcessor {
   event: ObservationEvent;
+  calculateTokenDelegate: (p: {
+    model: Model;
+    text: unknown;
+  }) => number | undefined;
 
-  constructor(event: ObservationEvent) {
+  constructor(
+    event: ObservationEvent,
+    calculateTokenDelegate: (p: {
+      model: Model;
+      text: unknown;
+    }) => number | undefined
+  ) {
     this.event = event;
+    this.calculateTokenDelegate = calculateTokenDelegate;
   }
 
   async convertToObservation(
     apiScope: ApiAccessScope,
-    existingObservation: Observation | null,
+    existingObservation: Observation | null
   ): Promise<{
     id: string;
     create: Prisma.ObservationUncheckedCreateInput;
@@ -154,7 +76,9 @@ export class ObservationProcessor implements EventProcessor {
       this.event.type === eventTypes.OBSERVATION_UPDATE &&
       !existingObservation
     ) {
-      throw new ResourceNotFoundError(this.event.id, "Observation not found");
+      throw new LangfuseNotFoundError(
+        `Observation with id ${this.event.id} not found`
+      );
     }
 
     // find matching model definition based on event and existing observation in db
@@ -197,8 +121,9 @@ export class ObservationProcessor implements EventProcessor {
       "usage" in this.event.body
         ? this.calculateTokenCounts(
             this.event.body,
+            this.calculateTokenDelegate,
             internalModel ?? undefined,
-            existingObservation ?? undefined,
+            existingObservation ?? undefined
           )
         : [undefined, undefined];
 
@@ -234,7 +159,7 @@ export class ObservationProcessor implements EventProcessor {
     const calculatedCosts = ObservationProcessor.calculateTokenCosts(
       internalModel,
       userProvidedTokenCosts,
-      tokenCounts,
+      tokenCounts
     );
 
     // merge metadata from existingObservation.metadata and metadata
@@ -242,7 +167,7 @@ export class ObservationProcessor implements EventProcessor {
       existingObservation?.metadata
         ? jsonSchema.parse(existingObservation.metadata)
         : undefined,
-      this.event.body.metadata ?? undefined,
+      this.event.body.metadata ?? undefined
     );
 
     const prompt =
@@ -388,34 +313,34 @@ export class ObservationProcessor implements EventProcessor {
     body:
       | z.infer<typeof legacyObservationCreateEvent>["body"]
       | z.infer<typeof generationCreateEvent>["body"],
+    calculateTokenDelegate: (p: {
+      model: Model;
+      text: unknown;
+    }) => number | undefined,
     model?: Model,
-    existingObservation?: Observation,
+    existingObservation?: Observation
   ) {
-    return instrument({ name: "calculate-tokens" }, () => {
-      const newPromptTokens =
-        body.usage?.input ??
-        ((body.input || existingObservation?.input) &&
-        model &&
-        model.tokenizerId
-          ? tokenCount({
-              model: model,
-              text: body.input ?? existingObservation?.input,
-            })
-          : undefined);
+    const newPromptTokens =
+      body.usage?.input ??
+      ((body.input || existingObservation?.input) && model && model.tokenizerId
+        ? calculateTokenDelegate({
+            model: model,
+            text: body.input ?? existingObservation?.input,
+          })
+        : undefined);
 
-      const newCompletionTokens =
-        body.usage?.output ??
-        ((body.output || existingObservation?.output) &&
-        model &&
-        model.tokenizerId
-          ? tokenCount({
-              model: model,
-              text: body.output ?? existingObservation?.output,
-            })
-          : undefined);
+    const newCompletionTokens =
+      body.usage?.output ??
+      ((body.output || existingObservation?.output) &&
+      model &&
+      model.tokenizerId
+        ? calculateTokenDelegate({
+            model: model,
+            text: body.output ?? existingObservation?.output,
+          })
+        : undefined);
 
-      return [newPromptTokens, newCompletionTokens];
-    });
+    return [newPromptTokens, newCompletionTokens];
   }
 
   static calculateTokenCosts(
@@ -425,7 +350,7 @@ export class ObservationProcessor implements EventProcessor {
       outputCost?: Decimal | null;
       totalCost?: Decimal | null;
     },
-    tokenCounts: { input?: number; output?: number; total?: number },
+    tokenCounts: { input?: number; output?: number; total?: number }
   ): {
     inputCost?: Decimal | null;
     outputCost?: Decimal | null;
@@ -442,7 +367,7 @@ export class ObservationProcessor implements EventProcessor {
         totalCost:
           userProvidedCosts.totalCost ??
           (userProvidedCosts.inputCost ?? new Decimal(0)).add(
-            userProvidedCosts.outputCost ?? new Decimal(0),
+            userProvidedCosts.outputCost ?? new Decimal(0)
           ),
       };
     }
@@ -473,9 +398,13 @@ export class ObservationProcessor implements EventProcessor {
     };
   }
 
-  async process(apiScope: ApiAccessScope): Promise<Observation> {
+  auth(apiScope: ApiAccessScope): void {
     if (apiScope.accessLevel !== "all")
       throw new ForbiddenError("Access denied for observation creation");
+  }
+
+  async process(apiScope: ApiAccessScope): Promise<Observation> {
+    this.auth(apiScope);
 
     const existingObservation = this.event.body.id
       ? await prisma.observation.findFirst({
@@ -488,7 +417,7 @@ export class ObservationProcessor implements EventProcessor {
       existingObservation.projectId !== apiScope.projectId
     ) {
       throw new ForbiddenError(
-        `Access denied for observation creation ${existingObservation.projectId} `,
+        `Access denied for observation creation ${existingObservation.projectId} `
       );
     }
 
@@ -512,15 +441,17 @@ export class TraceProcessor implements EventProcessor {
     this.event = event;
   }
 
+  auth(apiScope: ApiAccessScope): void {
+    if (apiScope.accessLevel !== "all")
+      throw new ForbiddenError("Access denied for trace creation");
+  }
+
   async process(
-    apiScope: ApiAccessScope,
+    apiScope: ApiAccessScope
   ): Promise<Trace | Observation | Score> {
     const { body } = this.event;
 
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError(
-        `Access denied for trace creation, ${apiScope.accessLevel}`,
-      );
+    this.auth(apiScope);
 
     const internalId = body.id ?? v4();
 
@@ -528,7 +459,7 @@ export class TraceProcessor implements EventProcessor {
       "Trying to create trace, project ",
       apiScope.projectId,
       ", id:",
-      internalId,
+      internalId
     );
 
     const existingTrace = await prisma.trace.findFirst({
@@ -539,7 +470,7 @@ export class TraceProcessor implements EventProcessor {
 
     if (existingTrace && existingTrace.projectId !== apiScope.projectId) {
       throw new ForbiddenError(
-        `Access denied for trace creation ${existingTrace.projectId}`,
+        `Access denied for trace creation ${existingTrace.projectId}`
       );
     }
 
@@ -547,7 +478,7 @@ export class TraceProcessor implements EventProcessor {
       existingTrace?.metadata
         ? jsonSchema.parse(existingTrace.metadata)
         : undefined,
-      body.metadata ?? undefined,
+      body.metadata ?? undefined
     );
 
     const mergedTags =
@@ -622,154 +553,19 @@ export class ScoreProcessor implements EventProcessor {
     this.event = event;
   }
 
-  static inferDataType(value: string | number): ScoreDataType {
-    return typeof value === "number"
-      ? ScoreDataType.NUMERIC
-      : ScoreDataType.CATEGORICAL;
-  }
-
-  static mapStringValueToNumericValue(
-    config: ValidatedScoreConfig,
-    label: string,
-  ): number | null {
-    return (
-      config.categories?.find((category) => category.label === label)?.value ??
-      null
-    );
-  }
-
-  static inflateScoreBody(
-    body: any,
-    id: string,
-    projectId: string,
-    config?: ValidatedScoreConfig,
-  ): Score {
-    const relevantDataType = config?.dataType ?? body.dataType;
-    const scoreProps = { ...body, id, projectId, source: "API" };
-
-    if (typeof body.value === "number") {
-      if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {
-        return {
-          ...scoreProps,
-          value: body.value,
-          stringValue: body.value === 1 ? "True" : "False",
-          dataType: ScoreDataType.BOOLEAN,
-        };
-      }
-
-      return {
-        ...scoreProps,
-        value: body.value,
-        dataType: ScoreDataType.NUMERIC,
-      };
-    }
-    return {
-      ...scoreProps,
-      value: config
-        ? ScoreProcessor.mapStringValueToNumericValue(config, body.value)
-        : null,
-      stringValue: body.value,
-      dataType: ScoreDataType.CATEGORICAL,
-    };
-  }
-
-  validateConfigAgainstBody(body: any, config: ValidatedScoreConfig): void {
-    const { maxValue, minValue, categories, dataType: configDataType } = config;
-    if (body.dataType && body.dataType !== configDataType) {
-      throw new InvalidRequestError(
-        `Data type mismatch based on config: expected ${configDataType}, got ${body.dataType}`,
+  auth(apiScope: ApiAccessScope) {
+    if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
+      throw new ForbiddenError(
+        `Access denied for score creation, ${apiScope.accessLevel}`
       );
-    }
-
-    if (config.isArchived) {
-      throw new InvalidRequestError(
-        "Config is archived and cannot be used to create new scores. Please restore the config first.",
-      );
-    }
-
-    if (config.name !== body.name) {
-      throw new InvalidRequestError(
-        `Name mismatch based on config: expected ${config.name}, got ${body.name}`,
-      );
-    }
-
-    const relevantDataType = configDataType ?? body.dataType;
-
-    const dataTypeValidation = ScoreBodyWithoutConfig.safeParse({
-      ...body,
-      dataType: relevantDataType,
-    });
-    if (!dataTypeValidation.success) {
-      throw new InvalidRequestError(
-        `Ingested score body not valid against provided config data type.`,
-      );
-    }
-
-    const rangeValidation = ScorePropsAgainstConfig.safeParse({
-      value: body.value,
-      dataType: relevantDataType,
-      ...(maxValue !== null && maxValue !== undefined && { maxValue }),
-      ...(minValue !== null && minValue !== undefined && { minValue }),
-      ...(categories && { categories }),
-    });
-    if (!rangeValidation.success) {
-      const errorDetails = rangeValidation.error.errors
-        .map((error) => `${error.path.join(".")} - ${error.message}`)
-        .join(", ");
-      throw new InvalidRequestError(
-        `Ingested score body not valid against provided config: ${errorDetails}`,
-      );
-    }
-  }
-
-  async validateAndInflate(
-    body: any,
-    id: string,
-    projectId: string,
-  ): Promise<Score> {
-    if (body.configId) {
-      const config = await prisma.scoreConfig.findFirst({
-        where: {
-          projectId,
-          id: body.configId,
-        },
-      });
-
-      if (!config || !validateDbScoreConfigSafe(config).success)
-        throw new LangfuseNotFoundError(
-          "The configId you provided does not match a valid config in this project",
-        );
-
-      this.validateConfigAgainstBody(body, config as ValidatedScoreConfig);
-      return ScoreProcessor.inflateScoreBody(
-        body,
-        id,
-        projectId,
-        config as ValidatedScoreConfig,
-      );
-    } else {
-      const validation = ScoreBodyWithoutConfig.safeParse({
-        ...body,
-        dataType: body.dataType ?? ScoreProcessor.inferDataType(body.value),
-      });
-      if (!validation.success) {
-        throw new InvalidRequestError(
-          `Ingested score value type not valid against provided data type. Provide numeric values for numeric and boolean scores, and string values for categorical scores.`,
-        );
-      }
-      return ScoreProcessor.inflateScoreBody(body, id, projectId);
-    }
   }
 
   async process(
-    apiScope: ApiAccessScope,
+    apiScope: ApiAccessScope
   ): Promise<Trace | Observation | Score> {
     const { body } = this.event;
 
-    if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
-      throw new ForbiddenError(
-        `Access denied for score creation, ${apiScope.accessLevel}`,
-      );
+    this.auth(apiScope);
 
     const id = body.id ?? v4();
 
@@ -783,15 +579,15 @@ export class ScoreProcessor implements EventProcessor {
     });
     if (existingScore && existingScore.projectId !== apiScope.projectId) {
       throw new ForbiddenError(
-        `Access denied for score creation ${existingScore.projectId}`,
+        `Access denied for score creation ${existingScore.projectId}`
       );
     }
 
-    const validatedScore = await this.validateAndInflate(
+    const validatedScore = await validateAndInflateScore({
       body,
-      id,
-      apiScope.projectId,
-    );
+      scoreId: id,
+      projectId: apiScope.projectId,
+    });
 
     return await prisma.score.upsert({
       where: {
@@ -817,6 +613,10 @@ export class SdkLogProcessor implements EventProcessor {
     this.event = event;
   }
 
+  auth(apiScope: ApiAccessScope) {
+    return;
+  }
+
   process() {
     try {
       console.log("SDK Log", this.event);

packages/shared/src/server/ingestion/legacy/enqueueIngestionEvents.ts

@@ -0,0 +1,71 @@
+import { type Redis } from "ioredis";
+import { QueueJobs } from "../../queues";
+import {
+  getIngestionFlushQueue,
+  IngestionFlushQueue,
+} from "../../redis/ingestionFlushQueue";
+import { IngestionUtils } from "../IngestionUtils";
+import { IngestionEventType } from "../types";
+import { redis } from "../../redis/redis";
+import { env } from "../../../env";
+
+export async function enqueueIngestionEvents(
+  projectId: string,
+  events: IngestionEventType[]
+) {
+  const ingestionFlushQueue = getIngestionFlushQueue();
+
+  if (!ingestionFlushQueue) {
+    throw Error("IngestionFlushQueue not initialized");
+  }
+  if (!redis) throw Error("Redis connection not available");
+
+  const queuedEventPromises: Promise<void>[] = [];
+  const batchTimestamp = Date.now().toString();
+
+  // Use for loop as TS does not narrow redis type in map function
+  for (const event of events) {
+    queuedEventPromises.push(
+      enqueueSingleIngestionEvent(
+        projectId,
+        event,
+        redis,
+        ingestionFlushQueue,
+        batchTimestamp
+      )
+    );
+  }
+
+  await Promise.all(queuedEventPromises);
+}
+
+async function enqueueSingleIngestionEvent(
+  projectId: string,
+  event: IngestionEventType,
+  redis: Redis,
+  ingestionFlushQueue: IngestionFlushQueue,
+  batchTimestamp: string
+): Promise<void> {
+  if (!("id" in event.body && event.body.id)) {
+    console.warn(
+      `Received ingestion event without id: ${JSON.stringify(event)}`
+    );
+
+    return;
+  }
+
+  const flushKey = IngestionUtils.getFlushKey({
+    entityId: event.body.id,
+    eventType: IngestionUtils.getEventType(event),
+    projectId,
+    batchTimestamp,
+  });
+  const bufferKey = IngestionUtils.getBufferKey(flushKey);
+  const serializedEventData = JSON.stringify({ ...event, projectId });
+
+  await redis.lpush(bufferKey, serializedEventData);
+  await redis.expire(bufferKey, env.LANGFUSE_INGESTION_BUFFER_TTL_SECONDS);
+  await ingestionFlushQueue.add(QueueJobs.FlushIngestionEntity, null, {
+    jobId: flushKey,
+  });
+}

packages/shared/src/server/ingestion/legacy/index.ts

@@ -0,0 +1,252 @@
+import { env } from "node:process";
+import z from "zod";
+import { ForbiddenError, UnauthorizedError } from "../../../errors";
+import { eventTypes, ingestionApiSchema, ingestionEvent } from "../types";
+import {
+  EventProcessor,
+  TraceProcessor,
+  ObservationProcessor,
+  ScoreProcessor,
+  SdkLogProcessor,
+} from "./EventProcessor";
+import { EventBodyType, EventName, TraceUpsertEventType } from "../../queues";
+import {
+  convertTraceUpsertEventsToRedisEvents,
+  getTraceUpsertQueue,
+} from "../../redis/trace-upsert";
+import { ApiAccessScope, AuthHeaderVerificationResult } from "../../auth/types";
+import { redis } from "../../redis/redis";
+import { backOff } from "exponential-backoff";
+import { Model } from "../../..";
+import { enqueueIngestionEvents } from "./enqueueIngestionEvents";
+
+export type BatchResult = {
+  result: unknown;
+  id: string;
+  type: string;
+};
+
+type TokenCountInput = {
+  model: Model;
+  text: unknown;
+};
+
+export const handleBatch = async (
+  events: z.infer<typeof ingestionApiSchema>["batch"],
+  authCheck: AuthHeaderVerificationResult,
+  calculateTokenDelegate: (p: TokenCountInput) => number | undefined
+) => {
+  console.log(`handling ingestion ${events.length} events`);
+
+  if (!authCheck.validKey) throw new UnauthorizedError(authCheck.error);
+
+  const results: BatchResult[] = []; // Array to store the results
+
+  const errors: {
+    error: unknown;
+    id: string;
+    type: string;
+  }[] = []; // Array to store the errors
+
+  for (const singleEvent of events) {
+    try {
+      const result = await retry(async () => {
+        return await handleSingleEvent(
+          singleEvent,
+          authCheck.scope,
+          calculateTokenDelegate
+        );
+      });
+      results.push({
+        result: result,
+        id: singleEvent.id,
+        type: singleEvent.type,
+      }); // Push each result into the array
+    } catch (error) {
+      // Handle or log the error if `handleSingleEvent` fails
+      console.error("Error handling event:", error);
+      // Decide how to handle the error: rethrow, continue, or push an error object to results
+      // For example, push an error object:
+      errors.push({
+        error: error,
+        id: singleEvent.id,
+        type: singleEvent.type,
+      });
+    }
+  }
+
+  if (env.CLICKHOUSE_URL) {
+    try {
+      await enqueueIngestionEvents(authCheck.scope.projectId, events);
+      console.log(`Added ${events.length} ingestion events to queue`);
+    } catch (err) {
+      console.error("Error adding ingestion events to queue", err);
+    }
+  }
+
+  return { results, errors };
+};
+
+async function retry<T>(request: () => Promise<T>): Promise<T> {
+  return await backOff(request, {
+    numOfAttempts: env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" ? 5 : 3,
+    retry: (e: Error, attemptNumber: number) => {
+      if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
+        console.log("not retrying auth error");
+        return false;
+      }
+      console.log(`retrying processing events ${attemptNumber}`);
+      return true;
+    },
+  });
+}
+
+const handleSingleEvent = async (
+  event: z.infer<typeof ingestionEvent>,
+  apiScope: ApiAccessScope,
+  calculateTokenDelegate: (p: {
+    model: Model;
+    text: unknown;
+  }) => number | undefined
+) => {
+  const { body } = event;
+  let restEvent = body;
+  if ("input" in body) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { input, ...rest } = body;
+    restEvent = rest;
+  }
+  if ("output" in restEvent) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { output, ...rest } = restEvent;
+    restEvent = rest;
+  }
+
+  console.log(
+    `handling single event ${event.id} of type ${event.type}:  ${JSON.stringify({ body: restEvent })}`
+  );
+
+  const cleanedEvent = ingestionEvent.parse(cleanEvent(event));
+
+  const { type } = cleanedEvent;
+
+  let processor: EventProcessor;
+  switch (type) {
+    case eventTypes.TRACE_CREATE:
+      processor = new TraceProcessor(cleanedEvent);
+      break;
+    case eventTypes.OBSERVATION_CREATE:
+    case eventTypes.OBSERVATION_UPDATE:
+    case eventTypes.EVENT_CREATE:
+    case eventTypes.SPAN_CREATE:
+    case eventTypes.SPAN_UPDATE:
+    case eventTypes.GENERATION_CREATE:
+    case eventTypes.GENERATION_UPDATE:
+      processor = new ObservationProcessor(
+        cleanedEvent,
+        calculateTokenDelegate
+      );
+      break;
+    case eventTypes.SCORE_CREATE: {
+      processor = new ScoreProcessor(cleanedEvent);
+      break;
+    }
+    case eventTypes.SDK_LOG:
+      processor = new SdkLogProcessor(cleanedEvent);
+  }
+
+  // Deny access to non-score events if the access level is not "all"
+  // This is an additional safeguard to auth checks in EventProcessor
+  if (apiScope.accessLevel !== "all" && type !== eventTypes.SCORE_CREATE) {
+    throw new ForbiddenError("Access denied. Event type not allowed.");
+  }
+
+  return await processor.process(apiScope);
+};
+
+// cleans NULL characters from the event
+export function cleanEvent(obj: unknown): unknown {
+  if (typeof obj === "string") {
+    return obj.replace(/\u0000/g, "");
+  } else if (typeof obj === "object" && obj !== null) {
+    if (Array.isArray(obj)) {
+      return obj.map(cleanEvent);
+    } else {
+      // Here we assert that obj is a Record<string, unknown>
+      const objAsRecord = obj as Record<string, unknown>;
+      const newObj: Record<string, unknown> = {};
+      for (const key in objAsRecord) {
+        newObj[key] = cleanEvent(objAsRecord[key]);
+      }
+      return newObj;
+    }
+  } else {
+    return obj;
+  }
+}
+
+export const isNotNullOrUndefined = <T>(
+  val?: T | null
+): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
+
+export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
+  val === undefined || val === null;
+
+export const sendToWorkerIfEnvironmentConfigured = async (
+  batchResults: BatchResult[],
+  projectId: string
+): Promise<void> => {
+  const traceEvents: TraceUpsertEventType[] = batchResults
+    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
+    .map((result) =>
+      result.result &&
+      typeof result.result === "object" &&
+      "id" in result.result
+        ? // ingestion API only gets traces for one projectId
+          { traceId: result.result.id as string, projectId }
+        : null
+    )
+    .filter(isNotNullOrUndefined);
+
+  try {
+    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
+      console.log(`Sending ${traceEvents.length} events to worker via Redis`);
+
+      const queue = getTraceUpsertQueue();
+      if (!queue) {
+        console.error("TraceUpsertQueue not initialized");
+        return;
+      }
+
+      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
+    } else if (
+      env.LANGFUSE_WORKER_HOST &&
+      env.LANGFUSE_WORKER_PASSWORD &&
+      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+    ) {
+      console.log(`Sending ${traceEvents.length} events to worker via HTTP`);
+      const body: EventBodyType = {
+        name: EventName.TraceUpsert,
+        payload: traceEvents,
+      };
+
+      if (traceEvents.length > 0) {
+        await fetch(`${env.LANGFUSE_WORKER_HOST}/api/events`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization:
+              "Basic " +
+              Buffer.from(
+                "admin" + ":" + env.LANGFUSE_WORKER_PASSWORD
+              ).toString("base64"),
+          },
+          body: JSON.stringify(body),
+          signal: AbortSignal.timeout(8 * 1000),
+        });
+      }
+    }
+  } catch (error) {
+    console.error("Error sending events to worker", error);
+  }
+};

packages/shared/src/server/ingestion/model-match.ts

@@ -0,0 +1,63 @@
+import { Model, Observation, Prisma } from "@prisma/client";
+import { prisma } from "../../db";
+
+export async function findModel(p: {
+  event: {
+    projectId: string;
+    model?: string;
+    unit?: string;
+    startTime?: Date;
+  };
+  existingDbObservation?: Observation;
+}): Promise<Model | null> {
+  const { event, existingDbObservation } = p;
+  // either get the model from the existing observation
+  // or match pattern on the user provided model name
+  const modelCondition = event.model
+    ? Prisma.sql`AND ${event.model} ~ match_pattern`
+    : existingDbObservation?.internalModel
+      ? Prisma.sql`AND model_name = ${existingDbObservation.internalModel}`
+      : undefined;
+  if (!modelCondition) return null;
+
+  // unit based on the current event or the existing observation, both can be undefined
+  const mergedUnit = event.unit ?? existingDbObservation?.unit;
+
+  const unitCondition = mergedUnit
+    ? Prisma.sql`AND unit = ${mergedUnit}`
+    : Prisma.empty;
+
+  const sql = Prisma.sql`
+    SELECT
+      id,
+      created_at AS "createdAt",
+      updated_at AS "updatedAt",
+      project_id AS "projectId",
+      model_name AS "modelName",
+      match_pattern AS "matchPattern",
+      start_date AS "startDate",
+      input_price AS "inputPrice",
+      output_price AS "outputPrice",
+      total_price AS "totalPrice",
+      unit, 
+      tokenizer_id AS "tokenizerId",
+      tokenizer_config AS "tokenizerConfig"
+    FROM
+      models
+    WHERE (project_id = ${event.projectId}
+      OR project_id IS NULL)
+    ${modelCondition}
+    ${unitCondition}
+    AND (start_date IS NULL OR start_date <= ${
+      event.startTime ? new Date(event.startTime) : new Date()
+    }::timestamp with time zone at time zone 'UTC')
+    ORDER BY
+      project_id ASC,
+      start_date DESC NULLS LAST
+    LIMIT 1
+  `;
+
+  const foundModels = await prisma.$queryRaw<Array<Model>>(sql);
+
+  return foundModels[0] ?? null;
+}

packages/shared/src/server/ingestion/types.ts

@@ -310,6 +310,16 @@ export const SdkLogEvent = z.object({
 
 // definitions for the ingestion API
 
+export const observationTypes = [
+  "observation-create",
+  "observation-update",
+  "generation-create",
+  "generation-update",
+  "span-create",
+  "span-update",
+  "event-create",
+];
+
 export const eventTypes = {
   TRACE_CREATE: "trace-create",
   SCORE_CREATE: "score-create",
@@ -319,7 +329,6 @@ export const eventTypes = {
   GENERATION_CREATE: "generation-create",
   GENERATION_UPDATE: "generation-update",
   SDK_LOG: "sdk-log",
-
   // LEGACY, only required for backwards compatibility
   OBSERVATION_CREATE: "observation-create",
   OBSERVATION_UPDATE: "observation-update",
@@ -334,6 +343,7 @@ export const traceEvent = base.extend({
   type: z.literal(eventTypes.TRACE_CREATE),
   body: TraceBody,
 });
+export type TraceEventType = z.infer<typeof traceEvent>;
 
 export const eventCreateEvent = base.extend({
   type: z.literal(eventTypes.EVENT_CREATE),
@@ -359,6 +369,7 @@ export const scoreEvent = base.extend({
   type: z.literal(eventTypes.SCORE_CREATE),
   body: ScoreBody,
 });
+export type ScoreEventType = z.infer<typeof scoreEvent>;
 export const sdkLogEvent = base.extend({
   type: z.literal(eventTypes.SDK_LOG),
   body: SdkLogEvent,
@@ -385,10 +396,35 @@ export const ingestionEvent = z.discriminatedUnion("type", [
   legacyObservationCreateEvent,
   legacyObservationUpdateEvent,
 ]);
+export type IngestionEventType = z.infer<typeof ingestionEvent>;
 
 export const ingestionBatchEvent = z.array(ingestionEvent);
+export type IngestionBatchEventType = z.infer<typeof ingestionBatchEvent>;
+
+export const ingestionEventWithProjectId = ingestionEvent.and(
+  z.object({ projectId: z.string() })
+);
+export type IngestionEventWithProjectIdType = z.infer<
+  typeof ingestionEventWithProjectId
+>;
 
 export const ingestionApiSchema = z.object({
   batch: ingestionBatchEvent,
   metadata: jsonSchema.nullish(),
 });
+
+export const ingestionApiSchemaWithProjectId = ingestionApiSchema.extend({
+  projectId: z.string(),
+});
+export type IngestionApiSchemaWithProjectId = z.infer<
+  typeof ingestionApiSchemaWithProjectId
+>;
+
+export type ObservationEvent =
+  | z.infer<typeof legacyObservationCreateEvent>
+  | z.infer<typeof legacyObservationUpdateEvent>
+  | z.infer<typeof eventCreateEvent>
+  | z.infer<typeof spanCreateEvent>
+  | z.infer<typeof spanUpdateEvent>
+  | z.infer<typeof generationCreateEvent>
+  | z.infer<typeof generationUpdateEvent>;

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -0,0 +1,160 @@
+import {
+  ScoreBodyWithoutConfig,
+  ScorePropsAgainstConfig,
+  validateDbScoreConfigSafe,
+  ValidatedScoreConfig,
+} from "../../../src";
+import { prisma, Score, ScoreDataType } from "../../db";
+
+import { InvalidRequestError, LangfuseNotFoundError } from "../../errors";
+
+type ValidateAndInflateScoreParams = {
+  projectId: string;
+  scoreId: string;
+  body: any;
+};
+
+export async function validateAndInflateScore(
+  params: ValidateAndInflateScoreParams
+): Promise<Score> {
+  const { body, projectId } = params;
+
+  if (body.configId) {
+    const config = await prisma.scoreConfig.findFirst({
+      where: {
+        projectId,
+        id: body.configId,
+      },
+    });
+
+    if (!config || !validateDbScoreConfigSafe(config).success)
+      throw new LangfuseNotFoundError(
+        "The configId you provided does not match a valid config in this project"
+      );
+
+    validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+
+    return inflateScoreBody({
+      ...params,
+      config: config as ValidatedScoreConfig,
+    });
+  }
+
+  const validation = ScoreBodyWithoutConfig.safeParse({
+    ...body,
+    dataType: body.dataType ?? inferDataType(body.value),
+  });
+
+  if (!validation.success) {
+    throw new InvalidRequestError(
+      `Ingested score value type not valid against provided data type. Provide numeric values for numeric and boolean scores, and string values for categorical scores.`
+    );
+  }
+
+  return inflateScoreBody(params);
+}
+
+function inferDataType(value: string | number): ScoreDataType {
+  return typeof value === "number"
+    ? ScoreDataType.NUMERIC
+    : ScoreDataType.CATEGORICAL;
+}
+
+function mapStringValueToNumericValue(
+  config: ValidatedScoreConfig,
+  label: string
+): number | null {
+  return (
+    config.categories?.find((category) => category.label === label)?.value ??
+    null
+  );
+}
+
+function inflateScoreBody(
+  params: ValidateAndInflateScoreParams & { config?: ValidatedScoreConfig }
+): Score {
+  const { body, projectId, scoreId, config } = params;
+
+  const relevantDataType = config?.dataType ?? body.dataType;
+  const scoreProps = { ...body, id: scoreId, projectId, source: "API" };
+
+  if (typeof body.value === "number") {
+    if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {
+      return {
+        ...scoreProps,
+        value: body.value,
+        stringValue: body.value === 1 ? "True" : "False",
+        dataType: ScoreDataType.BOOLEAN,
+      };
+    }
+
+    return {
+      ...scoreProps,
+      value: body.value,
+      dataType: ScoreDataType.NUMERIC,
+    };
+  }
+
+  return {
+    ...scoreProps,
+    value: config ? mapStringValueToNumericValue(config, body.value) : null,
+    stringValue: body.value,
+    dataType: ScoreDataType.CATEGORICAL,
+  };
+}
+
+function validateConfigAgainstBody(
+  body: any,
+  config: ValidatedScoreConfig
+): void {
+  const { maxValue, minValue, categories, dataType: configDataType } = config;
+
+  if (body.dataType && body.dataType !== configDataType) {
+    throw new InvalidRequestError(
+      `Data type mismatch based on config: expected ${configDataType}, got ${body.dataType}`
+    );
+  }
+
+  if (config.isArchived) {
+    throw new InvalidRequestError(
+      "Config is archived and cannot be used to create new scores. Please restore the config first."
+    );
+  }
+
+  if (config.name !== body.name) {
+    throw new InvalidRequestError(
+      `Name mismatch based on config: expected ${config.name}, got ${body.name}`
+    );
+  }
+
+  const relevantDataType = configDataType ?? body.dataType;
+
+  const dataTypeValidation = ScoreBodyWithoutConfig.safeParse({
+    ...body,
+    dataType: relevantDataType,
+  });
+
+  if (!dataTypeValidation.success) {
+    throw new InvalidRequestError(
+      `Ingested score body not valid against provided config data type.`
+    );
+  }
+
+  const rangeValidation = ScorePropsAgainstConfig.safeParse({
+    value: body.value,
+    dataType: relevantDataType,
+    ...(maxValue !== null && maxValue !== undefined && { maxValue }),
+    ...(minValue !== null && minValue !== undefined && { minValue }),
+    ...(categories && { categories }),
+  });
+
+  if (!rangeValidation.success) {
+    const errorDetails = rangeValidation.error.errors
+      .map((error) => `${error.path.join(".")} - ${error.message}`)
+      .join(", ");
+
+    throw new InvalidRequestError(
+      `Ingested score body not valid against provided config: ${errorDetails}`
+    );
+  }
+}

packages/shared/src/server/llm/types.ts

@@ -48,6 +48,7 @@ export const ZodModelConfig = z.object({
 // NOTE: Update docs page when changing this!
 export const openAIModels = [
   "gpt-4o",
+  "gpt-4o-2024-08-06",
   "gpt-4o-2024-05-13",
   "gpt-4o-mini",
   "gpt-4o-mini-2024-07-18",

packages/shared/src/server/queues.ts

@@ -1,11 +1,24 @@
 import { z } from "zod";
+import { ingestionBatchEvent } from ".";
 
 export enum EventName {
   TraceUpsert = "TraceUpsert",
   BatchExport = "BatchExport",
   EvaluationExecution = "EvaluationExecution",
+  LegacyIngestion = "LegacyIngestion",
 }
 
+export const LegacyIngestionEvent = z.object({
+  data: ingestionBatchEvent,
+  authCheck: z.object({
+    validKey: z.literal(true),
+    scope: z.object({
+      projectId: z.string(),
+      accessLevel: z.enum(["all", "scores"]),
+    }),
+  }),
+});
+
 export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
@@ -22,6 +35,7 @@ export const EvalExecutionEvent = z.object({
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
+export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 
 export const EventBodySchema = z.union([
   z.object({
@@ -44,6 +58,8 @@ export enum QueueName {
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
   BatchExport = "batch-export-queue",
   RepeatQueue = "repeat-queue",
+  IngestionFlushQueue = "ingestion-flush-queue",
+  LegacyIngestionQueue = "legacy-ingestion-queue",
 }
 
 export enum QueueJobs {
@@ -51,6 +67,8 @@ export enum QueueJobs {
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
   EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
+  FlushIngestionEntity = "flush-ingestion-entity",
+  LegacyIngestionJob = "legacy-ingestion-job",
 }
 
 export type TQueueJobTypes = {
@@ -72,4 +90,10 @@ export type TQueueJobTypes = {
     payload: BatchExportJobType;
     name: QueueJobs.BatchExportJob;
   };
+  [QueueName.LegacyIngestionQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: LegacyIngestionEventType;
+    name: QueueJobs.LegacyIngestionJob;
+  };
 };

packages/shared/src/server/redis/batch-export.ts

@@ -0,0 +1,23 @@
+import { Queue } from "bullmq";
+import { QueueName, TQueueJobTypes } from "../queues";
+import { redis } from "./redis";
+
+let batchExportQueue: Queue<TQueueJobTypes[QueueName.BatchExport]> | null =
+  null;
+
+export const getBatchExportQueue = () => {
+  if (batchExportQueue) return batchExportQueue;
+
+  batchExportQueue = redis
+    ? new Queue<TQueueJobTypes[QueueName.BatchExport]>(QueueName.BatchExport, {
+        connection: redis,
+        defaultJobOptions: {
+          removeOnComplete: true,
+          removeOnFail: 100,
+          attempts: 2,
+        },
+      })
+    : null;
+
+  return batchExportQueue;
+};

packages/shared/src/server/redis/ingestionFlushQueue.ts

@@ -0,0 +1,27 @@
+import { Queue } from "bullmq";
+
+import { env } from "../../env";
+import { redis } from "../redis/redis";
+import { QueueName } from "../queues";
+
+export type IngestionFlushQueue = Queue<null>;
+
+let ingestionFlushQueue: IngestionFlushQueue | null = null;
+
+export const getIngestionFlushQueue = () => {
+  if (ingestionFlushQueue) return ingestionFlushQueue;
+
+  ingestionFlushQueue = redis
+    ? new Queue<null>(QueueName.IngestionFlushQueue, {
+        connection: redis,
+        defaultJobOptions: {
+          removeOnComplete: true, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
+          removeOnFail: 1000,
+          delay: env.LANGFUSE_INGESTION_FLUSH_DELAY_MS,
+          attempts: env.LANGFUSE_INGESTION_FLUSH_ATTEMPTS,
+        },
+      })
+    : null;
+
+  return ingestionFlushQueue;
+};

packages/shared/src/server/redis/legacy-ingestion.ts

@@ -0,0 +1,27 @@
+import { Queue } from "bullmq";
+import { QueueName, TQueueJobTypes } from "../queues";
+import { redis } from "./redis";
+
+let legacyIngestionQueue: Queue<
+  TQueueJobTypes[QueueName.LegacyIngestionQueue]
+> | null = null;
+
+export const getLegacyIngestionQueue = () => {
+  if (legacyIngestionQueue) return legacyIngestionQueue;
+
+  legacyIngestionQueue = redis
+    ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
+        QueueName.LegacyIngestionQueue,
+        {
+          connection: redis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+          },
+        }
+      )
+    : null;
+
+  return legacyIngestionQueue;
+};

packages/shared/src/server/redis/trace-upsert.ts

@@ -4,7 +4,7 @@ import {
   QueueName,
   TQueueJobTypes,
   TraceUpsertEventType,
-} from "../../queues";
+} from "../queues";
 import { Queue } from "bullmq";
 import { redis } from "./redis";
 

packages/shared/src/server/services/PromptService.ts

@@ -18,12 +18,6 @@ export class PromptService {
       (cacheEnabled || env.LANGFUSE_CACHE_PROMPT_ENABLED === "true");
 
     this.ttlSeconds = env.LANGFUSE_CACHE_PROMPT_TTL_SECONDS;
-
-    if (this.cacheEnabled) {
-      this.logInfo("Prompt cache enabled with TTL seconds", this.ttlSeconds);
-    } else {
-      this.logInfo("Prompt cache disabled");
-    }
   }
 
   public async getPrompt(params: PromptParams): Promise<Prompt | null> {

packages/shared/src/server/services/email/organizationInvitation/MembershipInvitationEmailTemplate.tsx

@@ -15,25 +15,25 @@ import {
   Text,
 } from "@react-email/components";
 
-interface ProjectInvitationTemplateProps {
+interface MembershipInvitationTemplateProps {
   invitedByUsername: string;
   invitedByUserEmail: string;
-  projectName: string;
+  orgName: string;
   receiverEmail: string;
   inviteLink: string;
   emailFromAddress: string;
   langfuseCloudRegion?: string;
 }
 
-export const ProjectInvitationTemplate = ({
+export const MembershipInvitationTemplate = ({
   invitedByUsername,
   invitedByUserEmail,
-  projectName,
+  orgName,
   receiverEmail,
   inviteLink,
   emailFromAddress,
   langfuseCloudRegion,
-}: ProjectInvitationTemplateProps) => {
+}: MembershipInvitationTemplateProps) => {
   const previewText = `Join ${invitedByUsername} on Langfuse`;
 
   return (
@@ -53,7 +53,7 @@ export const ProjectInvitationTemplate = ({
               />
             </Section>
             <Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
-              Join <strong>{projectName}</strong> on <strong>Langfuse</strong>
+              Join <strong>{orgName}</strong> on <strong>Langfuse</strong>
             </Heading>
             <Text className="text-sm leading-6 text-black">Hello,</Text>
             <Text className="text-sm leading-6 text-black">
@@ -64,7 +64,8 @@ export const ProjectInvitationTemplate = ({
               >
                 {invitedByUserEmail}
               </Link>
-              ) has invited you to the <strong>{projectName}</strong> project on
+              ) has invited you to join the <strong>{orgName}</strong>{" "}
+              organization on
               {langfuseCloudRegion
                 ? ` Langfuse (${langfuseCloudRegion} data region)`
                 : " Langfuse"}
@@ -78,7 +79,7 @@ export const ProjectInvitationTemplate = ({
                 Accept Invitation
               </Button>
               <Text className="mt-2 text-xs leading-3 text-muted-foreground">
-                (you need to create an account)
+                (you need to create an account with this email address)
               </Text>
             </Section>
             <Text className="text-sm leading-6 text-black">
@@ -102,4 +103,4 @@ export const ProjectInvitationTemplate = ({
   );
 };
 
-export default ProjectInvitationTemplate;
+export default MembershipInvitationTemplate;

packages/shared/src/server/services/email/organizationInvitation/sendMembershipInvitationEmail.ts

@@ -2,7 +2,7 @@ import { createTransport } from "nodemailer";
 import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
 import { render } from "@react-email/render";
 
-import ProjectInvitationTemplate from "./ProjectInvitationEmailTemplate";
+import MembershipInvitationTemplate from "./MembershipInvitationEmailTemplate";
 
 const langfuseUrls = {
   US: "https://us.cloud.langfuse.com",
@@ -10,7 +10,7 @@ const langfuseUrls = {
   STAGING: "https://staging.langfuse.com",
 };
 
-type SendProjectInvitationParams = {
+type SendMembershipInvitationParams = {
   env: Partial<
     Record<
       | "EMAIL_FROM_ADDRESS"
@@ -23,19 +23,19 @@ type SendProjectInvitationParams = {
   to: string;
   inviterName: string;
   inviterEmail: string;
-  projectName: string;
+  orgName: string;
 };
 
-export const sendProjectInvitationEmail = async ({
+export const sendMembershipInvitationEmail = async ({
   env,
   to,
   inviterName,
   inviterEmail,
-  projectName,
-}: SendProjectInvitationParams) => {
+  orgName,
+}: SendMembershipInvitationParams) => {
   if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
     console.error(
-      "Missing environment variables for sending project invitation email."
+      "Missing environment variables for sending membership invitation email."
     );
     return;
   }
@@ -59,10 +59,10 @@ export const sendProjectInvitationEmail = async ({
     const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
 
     const htmlTemplate = render(
-      ProjectInvitationTemplate({
+      MembershipInvitationTemplate({
         invitedByUsername: inviterName,
         invitedByUserEmail: inviterEmail,
-        projectName: projectName,
+        orgName: orgName,
         receiverEmail: to,
         inviteLink: authUrl,
         emailFromAddress: env.EMAIL_FROM_ADDRESS,
@@ -72,11 +72,8 @@ export const sendProjectInvitationEmail = async ({
 
     await mailer.sendMail({
       to,
-      from: {
-        address: env.EMAIL_FROM_ADDRESS,
-        name: "Langfuse",
-      },
-      subject: `${inviterName} invited you to join "${projectName}"`,
+      from: `Langfuse <${env.EMAIL_FROM_ADDRESS}>`,
+      subject: `${inviterName} invited you to join "${orgName}" organization on Langfuse`,
       html: htmlTemplate,
     });
   } catch (error) {

packages/shared/src/utils/json.ts

@@ -1,6 +1,6 @@
-import { type z } from "zod";
+import { z } from "zod";
 import lodash from "lodash";
-import { jsonSchema } from "./zod";
+import { JsonNested, jsonSchema, jsonSchemaNullable } from "./zod";
 
 export const parseJson = (input: string) => {
   try {
@@ -57,3 +57,52 @@ export const mergeJson = (
   }
   return lodash.merge(json1, json2);
 };
+
+export const parseJsonPrioritised = (
+  json: string
+): z.infer<typeof jsonSchema> | string | undefined => {
+  try {
+    const parsedJson = JSON.parse(json);
+
+    if (Object.keys(parsedJson).length === 0) {
+      return parsedJson;
+    }
+
+    const parsedArray = z.array(jsonSchemaNullable).safeParse(parsedJson);
+    if (parsedArray.success) {
+      return parsedArray.data;
+    }
+
+    const parsedObject = z.record(jsonSchemaNullable).safeParse(parsedJson);
+    if (parsedObject.success) {
+      return parsedObject.data;
+    }
+
+    return jsonSchema.parse(parsedJson);
+  } catch (error) {
+    const parsed = jsonSchema.safeParse(json);
+
+    return parsed.success ? parsed.data : json;
+  }
+};
+
+export const convertRecordToJsonSchema = (
+  record: Record<string, string>
+): JsonNested | undefined => {
+  const jsonSchema: JsonNested = {};
+
+  // if record is empty, return undefined
+  if (Object.keys(record).length === 0) {
+    return undefined;
+  }
+
+  for (const key in record) {
+    try {
+      jsonSchema[key] = JSON.parse(record[key]);
+    } catch (e) {
+      jsonSchema[key] = record[key];
+    }
+  }
+
+  return jsonSchema;
+};

packages/shared/src/utils/typeChecks.ts

@@ -1,2 +1,6 @@
+import { z } from "zod";
+
 export const isPresent = <T>(value: T | null | undefined): value is T =>
   value !== null && value !== undefined && value !== "";
+
+export const stringDateTime = z.string().datetime({ offset: true }).nullish();

packages/shared/src/utils/zod.ts

@@ -17,11 +17,11 @@ const nestedLiteralSchema = z.union([
 type Root = z.infer<typeof rootLiteralSchema>;
 type Literal = z.infer<typeof nestedLiteralSchema>;
 
-type JsonNested = Literal | { [key: string]: JsonNested } | JsonNested[];
+export type JsonNested = Literal | { [key: string]: JsonNested } | JsonNested[];
 type Json = Root | { [key: string]: JsonNested } | JsonNested[];
 
 // Here, you define the schema recursively
-const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
+export const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
   z.union([
     nestedLiteralSchema,
     z.array(jsonSchemaNullable),

packages/shared/tsconfig.json

@@ -4,7 +4,7 @@
     "jsx": "react",
     "moduleResolution": "NodeNext",
     "module": "NodeNext",
-    "lib": ["ES2020"],
+    "lib": ["ES2021"],
     "outDir": "./dist",
     "types": ["node"],
     "target": "ES2020",

web/Dockerfile

@@ -34,13 +34,13 @@ ENV NEXT_MANUAL_SIG_HANDLE true
 
 # pass public variables in build step
 ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ARG NEXT_PUBLIC_DEMO_ORG_ID
 ARG NEXT_PUBLIC_DEMO_PROJECT_ID
 ARG NEXT_PUBLIC_SIGN_UP_DISABLED
 ARG NEXT_PUBLIC_TURNSTILE_SITE_KEY
 ARG NEXT_PUBLIC_POSTHOG_KEY
 ARG NEXT_PUBLIC_POSTHOG_HOST
 ARG NEXT_PUBLIC_CRISP_WEBSITE_ID
-ARG LANFUSE_WEB_MIGRATION_DISABLED
 
 # Copy source code of isolated subworkspace
 COPY --from=pruner /app/out/full/ .
@@ -74,7 +74,7 @@ RUN apk add --no-cache dumb-init
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
-RUN npm install -g --no-package-lock --no-save prisma
+RUN npm install -g --no-package-lock --no-save prisma@5.13.0
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.67.0",
+  "version": "2.71.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -14,9 +14,10 @@
     "prettier": "prettier --write ./src *.{ts,js}",
     "clean": "rm -rf node_modules",
     "start": "dotenv -e ../.env -- sh -c 'NEXT_MANUAL_SIG_HANDLE=true next start'",
-    "test": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose",
+    "test": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --testPathIgnorePatterns='__e2e__'",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
+    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --testPathPattern='__e2e__'",
     "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts",
     "generationCost:backfill": "dotenv -e ../.env -- tsx scripts/observations-backfill-calculated-cost.ts"
   },
@@ -60,10 +61,9 @@
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.13.0",
-    "@sentry/node": "^8.13.0",
-    "@sentry/profiling-node": "^8.13.0",
-    "@sentry/types": "^8.13.0",
+    "@sentry/nextjs": "8.13.0",
+    "@sentry/node": "8.13.0",
+    "@sentry/profiling-node": "8.13.0",
     "@t3-oss/env-nextjs": "^0.10.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -87,7 +87,6 @@
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
     "dompurify": "^3.1.5",
-    "exponential-backoff": "^3.1.1",
     "graphql": "^16.9.0",
     "ioredis": "^5.4.1",
     "ip-address": "^9.0.5",
@@ -100,7 +99,7 @@
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "next-themes": "^0.3.0",
-    "posthog-js": "^1.122.0",
+    "posthog-js": "^1.154.6",
     "posthog-node": "^3.6.3",
     "prexit": "^2.2.0",
     "prisma": "^5.13.0",
@@ -158,7 +157,8 @@
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.2",
-    "typescript": "^5.4.5"
+    "typescript": "^5.4.5",
+    "wait-for-expect": "^3.0.2"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"

web/public/generated/api/openapi.yml

@@ -1188,6 +1188,26 @@ paths:
           schema:
             type: integer
             nullable: true
+        - name: fromUpdatedAt
+          in: query
+          description: >-
+            Optional filter to only include prompt versions created/updated on
+            or after a certain datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toUpdatedAt
+          in: query
+          description: >-
+            Optional filter to only include prompt versions created/updated
+            before a certain datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1222,7 +1242,7 @@ paths:
               schema: {}
       security: *ref_0
     post:
-      description: Create a prompt
+      description: Create a new version for the prompt with the given `name`
       operationId: prompts_create
       tags:
         - Prompts
@@ -3687,11 +3707,20 @@ components:
           type: array
           items:
             type: string
+        lastUpdatedAt:
+          type: string
+          format: date-time
+        lastConfig:
+          description: >-
+            Config object of the most recent prompt version that matches the
+            filters (if any are provided)
       required:
         - name
         - versions
         - labels
         - tags
+        - lastUpdatedAt
+        - lastConfig
     CreatePromptRequest:
       title: CreatePromptRequest
       oneOf:

web/public/generated/postman/collection.json

@@ -877,7 +877,7 @@
           "request": {
             "description": "Get a list of prompt names with versions and labels",
             "url": {
-              "raw": "{{baseUrl}}/api/public/v2/prompts?name=&label=&tag=&page=&limit=",
+              "raw": "{{baseUrl}}/api/public/v2/prompts?name=&label=&tag=&page=&limit=&fromUpdatedAt=&toUpdatedAt=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -912,6 +912,16 @@
                   "key": "limit",
                   "value": "",
                   "description": "limit of items per page"
+                },
+                {
+                  "key": "fromUpdatedAt",
+                  "value": "",
+                  "description": "Optional filter to only include prompt versions created/updated on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toUpdatedAt",
+                  "value": "",
+                  "description": "Optional filter to only include prompt versions created/updated before a certain datetime (ISO 8601)"
                 }
               ],
               "variable": []
@@ -927,7 +937,7 @@
           "_type": "endpoint",
           "name": "Create",
           "request": {
-            "description": "Create a prompt",
+            "description": "Create a new version for the prompt with the given `name`",
             "url": {
               "raw": "{{baseUrl}}/api/public/v2/prompts",
               "host": [

web/public/images/posthog-logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>

web/scripts/model-match.ts

@@ -5,11 +5,11 @@
 
 import "dotenv/config";
 
-import { findModel } from "@/src/server/api/services/EventProcessor";
+import { findModel } from "@langfuse/shared/src/server";
 import { prisma } from "@langfuse/shared/src/db";
 import lodash from "lodash";
-import { tokenCount } from "@/src/features/ingest/lib/usage";
 import { type Prisma } from "@langfuse/shared/src/db";
+import { tokenCount } from "@/src/features/ingest/usage";
 
 async function main() {
   return await modelMatch();

web/src/__e2e__/api.servertest.ts

@@ -0,0 +1,203 @@
+import { v4 } from "uuid";
+import { JobExecutionStatus, prisma } from "@langfuse/shared/src/db";
+import { redis } from "@langfuse/shared/src/server";
+import waitForExpect from "wait-for-expect";
+import { ApiKeyZod } from "@/src/features/public-api/server/apiAuth";
+
+const generateAuth = (username: string, password: string) => {
+  const auth = Buffer.from(`${username}:${password}`).toString("base64");
+  return `Basic ${auth}`;
+};
+
+const workerAdminAuth = generateAuth("admin", "myworkerpassword");
+
+const userApiKeyAuth = generateAuth("pk-lf-1234567890", "sk-lf-1234567890");
+
+describe("Health endpoints", () => {
+  it("web container returns healthy", async () => {
+    // Arrange
+    const url = "http://localhost:3000/api/public/health";
+
+    // Act
+    const response = await fetch(url);
+    expect(response.status).toBe(200);
+  });
+
+  it("worker container returns healthy", async () => {
+    // Arrange
+    const url = "http://localhost:3030/api/health";
+
+    // Act
+    const response = await fetch(url, {
+      headers: {
+        Authorization: workerAdminAuth,
+      },
+    });
+    expect(response.status).toBe(200);
+  });
+});
+
+describe("Ingestion Pipeline", () => {
+  it("ingest a trace", async () => {
+    const traceId = v4();
+    const spanId = v4();
+
+    const event = {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            name: "test trace",
+            id: traceId,
+            userId: "user-1", // triggers the eval
+          },
+        },
+        {
+          id: v4(),
+          type: "span-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: spanId,
+            traceId: traceId,
+            name: "test span",
+          },
+        },
+      ],
+    };
+    // Arrange
+    const url = "http://localhost:3000/api/public/ingestion";
+
+    // Act
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: userApiKeyAuth,
+      },
+      body: JSON.stringify(event),
+    });
+
+    await waitForExpect(async () => {
+      // we need a second call to the public API with the API key, so that it is stored in redis
+      // first call (ingestion above) generates the new, fast API hash
+      // second call (below) stores the API key in redis
+      const traceUrl = `http://localhost:3000/api/public/traces/${traceId}`;
+
+      const traceResponse = await fetch(traceUrl, {
+        headers: {
+          Authorization: userApiKeyAuth,
+        },
+      });
+
+      expect(traceResponse.status).toBe(200);
+      expect(traceResponse.body).not.toBeNull();
+      expect((await traceResponse.json()).id).toBe(traceId);
+
+      const trace = await prisma.trace.findUnique({
+        where: {
+          id: traceId,
+        },
+      });
+      expect(trace).not.toBeNull();
+      expect(trace?.name).toBe("test trace");
+
+      const observation = await prisma.observation.findUnique({
+        where: {
+          id: spanId,
+          traceId: traceId,
+        },
+      });
+      expect(observation).not.toBeNull();
+      expect(observation?.name).toBe("test span");
+
+      expect(redis).not.toBeNull();
+
+      const redisKeys = await redis?.keys(`api-key:*`);
+      expect(redisKeys?.length).toBe(1);
+      const redisValue = await redis?.get(redisKeys![0]);
+
+      const llmApiKey = ApiKeyZod.parse(JSON.parse(redisValue!));
+      expect(llmApiKey.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    });
+
+    // check for eval
+    await waitForExpect(async () => {
+      const evalExecution = await prisma.jobExecution.findFirst({
+        where: {
+          jobInputTraceId: traceId,
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+      });
+
+      expect(evalExecution).not.toBeNull();
+
+      if (!evalExecution) {
+        return;
+      }
+
+      // failure due to missing openai key in the pipeline. Expected
+      expect(evalExecution.status).toBe(JobExecutionStatus.ERROR);
+    }, 20000);
+
+    expect(response.status).toBe(207);
+  }, 25000);
+});
+
+describe("Prompts endpoint", () => {
+  it("creates and returns a prompt", async () => {
+    const promptName = "prompt-name" + v4();
+    const chatMessages = [
+      { role: "system", content: "You are a bot" },
+      { role: "user", content: "What's up?" },
+    ];
+    const response = await fetch(
+      "http://localhost:3000/api/public/v2/prompts",
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: userApiKeyAuth,
+        },
+        body: JSON.stringify({
+          name: promptName,
+          prompt: chatMessages,
+          type: "chat",
+          labels: ["production"],
+        }),
+      },
+    );
+
+    expect(response.status).toBe(201);
+
+    const fetchedPrompt = await fetch(
+      `http://localhost:3000/api/public/v2/prompts/${encodeURIComponent(promptName)}`,
+      {
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: userApiKeyAuth,
+        },
+      },
+    );
+    expect(fetchedPrompt.status).toBe(200);
+    expect(fetchedPrompt.body).not.toBeNull();
+
+    if (fetchedPrompt.body === null) {
+      return;
+    }
+
+    const validatedPrompt = await fetchedPrompt.json();
+
+    expect(validatedPrompt.name).toBe(promptName);
+
+    const redisKey = `prompt:7a88fb47-b4e2-43b8-a06c-a5ce950dc53a:${promptName}:${validatedPrompt.labels[0]}`;
+    const redisValue = await redis?.get(redisKey);
+
+    expect(redisValue).not.toBeNull();
+    if (!redisValue) {
+      return;
+    }
+    expect(JSON.parse(redisValue)).toEqual(validatedPrompt);
+  });
+});

web/src/__e2e__/auth.spec.ts

@@ -15,16 +15,7 @@ test("should redirect to home if signed in", async ({ page }) => {
   // wait 2 seconds
   await page.waitForTimeout(2000);
 
-  if (process.env.CI)
-    await expect(page).toHaveURL(
-      // project id from seed.ts
-      "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    );
-  else
-    console.log(
-      "Test skipped as redirect depends on db state, URL after signing in:",
-      page.url(),
-    );
+  await expect(page).toHaveURL("/");
 });
 
 test("Successfully sign up & able to go to homepage", async ({ page }) => {
@@ -35,7 +26,7 @@ test("Successfully sign up & able to go to homepage", async ({ page }) => {
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
-  await expect(page).toHaveURL("/?getStarted=1");
+  await expect(page).toHaveURL("/");
 });
 
 test("Successfully sign up & able to go to homepage with uppercase email", async ({
@@ -48,7 +39,7 @@ test("Successfully sign up & able to go to homepage with uppercase email", async
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
-  await expect(page).toHaveURL("/?getStarted=1");
+  await expect(page).toHaveURL("/");
 });
 
 test("Signup input validation", async ({ page }) => {
@@ -64,7 +55,7 @@ test("Signup input validation", async ({ page }) => {
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // don't see get started page
-  await expect(page).not.toHaveURL("/?getStarted=1");
+  await expect(page).not.toHaveURL("/");
 });
 
 // random email address to be used in tests
@@ -83,7 +74,8 @@ test("Unauthenticated user should be redirected to target URL after login", asyn
   await page.waitForTimeout(2000);
 
   // project id and prompt from seed.ts
-  const promptUrl = "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
+  const promptUrl =
+    "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
 
   await page.getByRole("button", { name: /Demo User/ }).click();
 

web/src/__e2e__/create-project.spec.ts

@@ -1,48 +1,59 @@
 import { test, expect } from "@playwright/test";
 
-test("should see new projects dialog open after clicking new project btn", async ({
-  page,
-}) => {
-  await page.goto("auth/sign-up");
-  await page.fill('input[name="name"]', "demo user");
-  await page.fill('input[name="email"]', randomEmailAddress());
-  await page.fill('input[type="password"]', "password");
-  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
-  await page.waitForTimeout(2000);
-  await page.waitForTimeout(2000);
-  expect(await page.innerHTML("data-testid=create-new-project-title")).toBe(
-    "Create new project",
-  );
-  await page.click('[data-testid="create-project-btn"]');
-  await page.waitForTimeout(2000);
-  await expect(page.locator("data-testid=new-project-form")).toBeVisible();
-});
-
-test("Create a project with provided name", async ({ page }) => {
+test("Sign in, create an organization, create a project", async ({ page }) => {
   test.setTimeout(60000);
 
+  // Sign in
   await page.goto("/auth/sign-in");
   await page.fill('input[name="email"]', "demo@langfuse.com");
   await page.fill('input[type="password"]', "password");
   await page.click('button[data-testid="submit-email-password-sign-in-form"]');
   await page.waitForTimeout(2000);
-  await page.isVisible('[data-testid="new-project-form"]');
-  await page.click('[data-testid="create-project-btn"]');
-  await page.waitForTimeout(2000);
-  await expect(page.locator("data-testid=new-project-form")).toBeVisible();
-  await page.fill(
-    '[data-testid="new-project-name-input"]',
-    "my e2e demo project",
-  );
-  await page.click('button[type="submit"]');
-  await page.waitForTimeout(2000);
-  expect(page.url()).toContain("/project/");
-  await page.waitForTimeout(2000);
-  expect(await page.getByTestId("project-name").textContent()).toContain(
-    "my e2e demo project",
-  );
-});
+  await expect(page).toHaveURL("/");
 
-// random email address to be used in tests
-const randomEmailAddress = () =>
-  Math.random().toString(36).substring(2, 11) + "@example.com";
+  // Start create org flow
+  await page.isVisible('[data-testid="create-organization-btn"]');
+  await page.click('[data-testid="create-organization-btn"]');
+  await page.waitForTimeout(2000);
+  await expect(page).toHaveURL("/setup");
+
+  // Create an organization
+  await expect(page.locator("data-testid=new-org-form")).toBeVisible();
+  await page.fill('[data-testid="new-org-name-input"]', "e2e test org");
+  await page.click('button[type="submit"]');
+  await page.waitForTimeout(5000);
+  expect(page.url()).toContain("/organization/");
+  expect(page.url()).toContain("/setup?orgstep=invite-members");
+
+  // Parse the organization ID from the URL using a simpler method
+  const url = new URL(page.url());
+  const organizationId = url.pathname.split("/")[2];
+  console.log("organization", organizationId);
+
+  // Skip add new members step
+  await page.isVisible('[data-testid="btn-skip-add-members"]');
+  await page.click('[data-testid="btn-skip-add-members"]');
+  expect(page.url()).toContain(
+    "/organization/" + organizationId + "/setup?orgstep=create-project",
+  );
+
+  // Create project
+  await expect(page.locator("data-testid=new-project-form")).toBeVisible();
+  await page.fill('[data-testid="new-project-name-input"]', "e2e test project");
+  await page.click('button[type="submit"]');
+  await page.waitForTimeout(5000);
+  expect(page.url()).toContain("/project/");
+  expect(page.url()).toContain("/setup");
+
+  const projectUrl = new URL(page.url());
+  const projectId = projectUrl.pathname.split("/")[2];
+
+  // check that the project exists by navigating to its dashboard
+  await page.goto("/project/" + projectId);
+  await page.waitForTimeout(2000);
+  expect(page.url()).toContain("/project/" + projectId);
+  expect(page.url()).not.toContain("/setup");
+
+  const headings = await page.locator("h2").allTextContents();
+  expect(headings).toContain("Dashboard");
+});

web/src/__tests__/aggregateScores.clienttest.ts

@@ -0,0 +1,189 @@
+import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
+import { type APIScore } from "@langfuse/shared";
+
+describe("aggregateScores", () => {
+  it("should return an empty object for an empty array", () => {
+    const scores: APIScore[] = [];
+    expect(aggregateScores(scores)).toEqual({});
+  });
+
+  it("should correctly aggregate a single numeric score", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "test comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-API-NUMERIC": {
+        type: "NUMERIC",
+        values: [5],
+        average: 5,
+        comment: "test comment",
+      },
+    });
+  });
+
+  it("should correctly aggregate multiple numeric scores with the same key", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "test comment",
+      },
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 7,
+        comment: "another comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-API-NUMERIC": {
+        type: "NUMERIC",
+        values: [5, 7],
+        average: 6,
+        comment: undefined,
+      },
+    });
+  });
+
+  it("should correctly aggregate scores with different keys", () => {
+    const scores: APIScore[] = [
+      {
+        name: "test1",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "test comment",
+      },
+      {
+        name: "test1",
+        source: "ANNOTATION",
+        dataType: "NUMERIC",
+        value: 7,
+        comment: "another comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test1-API-NUMERIC": {
+        type: "NUMERIC",
+        values: [5],
+        average: 5,
+        comment: "test comment",
+      },
+      "test1-ANNOTATION-NUMERIC": {
+        type: "NUMERIC",
+        values: [7],
+        average: 7,
+        comment: "another comment",
+      },
+    });
+  });
+
+  it("should correctly aggregate a single Categorical score", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "ANNOTATION",
+        dataType: "CATEGORICAL",
+        stringValue: "good",
+        comment: "test comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-ANNOTATION-CATEGORICAL": {
+        type: "CATEGORICAL",
+        values: ["good"],
+        valueCounts: [{ value: "good", count: 1 }],
+        comment: "test comment",
+      },
+    });
+  });
+
+  it("should correctly aggregate multiple Categorical scores with the same key", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "API",
+        dataType: "BOOLEAN",
+        stringValue: "True",
+        comment: "test comment",
+      },
+      {
+        name: "test",
+        source: "API",
+        dataType: "BOOLEAN",
+        stringValue: "False",
+        comment: "another comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-API-BOOLEAN": {
+        type: "CATEGORICAL",
+        values: ["True", "False"],
+        valueCounts: [
+          { value: "True", count: 1 },
+          { value: "False", count: 1 },
+        ],
+        comment: undefined,
+      },
+    });
+  });
+
+  it("should correctly aggregate scores with mixed types and the same name", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "test comment",
+      },
+      {
+        name: "test",
+        source: "ANNOTATION",
+        dataType: "CATEGORICAL",
+        stringValue: "good",
+        comment: "another comment",
+      },
+      {
+        name: "test",
+        source: "ANNOTATION",
+        dataType: "CATEGORICAL",
+        stringValue: "bad",
+        comment: "last comment",
+      },
+      {
+        name: "test",
+        source: "ANNOTATION",
+        dataType: "CATEGORICAL",
+        stringValue: "good",
+        comment: "last comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-API-NUMERIC": {
+        type: "NUMERIC",
+        values: [5],
+        average: 5,
+        comment: "test comment",
+      },
+      "test-ANNOTATION-CATEGORICAL": {
+        type: "CATEGORICAL",
+        values: ["good", "bad", "good"],
+        valueCounts: [
+          { value: "good", count: 2 },
+          { value: "bad", count: 1 },
+        ],
+        comment: undefined,
+      },
+    });
+  });
+});

web/src/__tests__/datasets.servertest.ts

@@ -848,6 +848,12 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     const otherProject = await prisma.project.create({
       data: {
         name: "other-project",
+        organization: {
+          connectOrCreate: {
+            where: { id: "other-org" },
+            create: { name: "other-org", id: "other-org" },
+          },
+        },
       },
     });
 

web/src/__tests__/generation-exports.servertest.ts

@@ -71,12 +71,19 @@ describe("observations.export RPC", () => {
     expires: "1",
     user: {
       id: "clgb17vnp000008jjere5g15i",
-      name: "John Doe",
-      projects: [
+      name: "Demo User",
+      organizations: [
         {
-          id: projectId,
-          role: "ADMIN",
-          name: "test",
+          id: "seed-org-id",
+          role: "OWNER",
+          plan: "cloud:hobby",
+          cloudConfig: undefined,
+          projects: [
+            {
+              id: projectId,
+              role: "ADMIN",
+            },
+          ],
         },
       ],
       featureFlags: {

web/src/__tests__/ingestion.servertest.ts

@@ -4,8 +4,8 @@ import { v4 } from "uuid";
 
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { ModelUsageUnit } from "@langfuse/shared";
-import { cleanEvent } from "@/src/pages/api/public/ingestion";
 import { prisma } from "@langfuse/shared/src/db";
+import { cleanEvent } from "@langfuse/shared/src/server";
 
 describe("/api/public/ingestion API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -707,19 +707,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbScore?.traceId).toBe(traceId);
     expect(dbScore?.observationId).toBe(generationId);
     expect(dbScore?.value).toBe(100.5);
-
-    const logEvent = await prisma.events.findFirst({
-      where: {
-        data: {
-          path: ["body", "log"],
-          string_contains: "ERROR",
-        },
-      },
-    });
-
-    expect(logEvent).toBeDefined();
-    expect(logEvent).not.toBeFalsy();
-    expect(JSON.stringify(logEvent?.data)).toContain("KeyError: 'model_name'");
   });
 
   it("should upsert threats", async () => {
@@ -1140,11 +1127,16 @@ describe("/api/public/ingestion API Endpoint", () => {
   it("should not override a trace from a different project", async () => {
     const traceId = v4();
     const newProjectId = v4();
-
+    await prisma.organization.upsert({
+      where: { id: "other-org" },
+      create: { id: "other-org", name: "other-org" },
+      update: {},
+    });
     await prisma.project.create({
       data: {
         id: newProjectId,
         name: "another-project",
+        orgId: "other-org",
       },
     });
 

web/src/__tests__/llm-api-key.servertest.ts

@@ -18,19 +18,26 @@ describe("llmApiKey.all RPC", () => {
     user: {
       id: "user-1",
       name: "Demo User",
-      projects: [
+      organizations: [
         {
-          id: projectId,
-          role: "ADMIN",
-          name: "test",
+          id: "seed-org-id",
+          role: "OWNER",
+          plan: "cloud:hobby",
+          cloudConfig: undefined,
+          projects: [
+            {
+              id: projectId,
+              role: "ADMIN",
+            },
+          ],
         },
       ],
       featureFlags: {
         templateFlag: true,
-        evals: true,
       },
       admin: true,
     },
+    environment: {} as any,
   };
 
   const ctx = createInnerTRPCContext({ session });

web/src/__tests__/promptCache.servertest.ts

@@ -3,16 +3,6 @@ import { PromptService, type redis } from "@langfuse/shared/src/server"; // Adju
 
 type Redis = NonNullable<typeof redis>;
 
-// Mocks
-jest.mock("@prisma/client");
-jest.mock("ioredis");
-jest.mock("@langfuse/shared", () => ({
-  env: {
-    LANGFUSE_CACHE_PROMPT_ENABLED: "true",
-    LANGFUSE_CACHE_PROMPT_TTL_SECONDS: 300,
-  },
-}));
-
 describe("PromptService", () => {
   let promptService: PromptService;
   let mockPrisma: jest.Mocked<PrismaClient>;

web/src/__tests__/prompts.v2.servertest.ts

@@ -23,11 +23,13 @@ type CreatePromptInDBParams = {
   prompt: string;
   labels: string[];
   version: number;
-  config: Record<string, object | number | string>;
+  config: any;
   projectId: string;
   createdBy: string;
   type?: PromptType;
   tags?: string[];
+  createdAt?: Date;
+  updatedAt?: Date;
 };
 const createPromptInDB = async (params: CreatePromptInDBParams) => {
   return await prisma.prompt.create({
@@ -44,6 +46,8 @@ const createPromptInDB = async (params: CreatePromptInDBParams) => {
       createdBy: params.createdBy,
       type: params.type,
       tags: params.tags,
+      createdAt: params.createdAt,
+      updatedAt: params.updatedAt,
     },
   });
 };
@@ -75,7 +79,7 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       const projectId = uuidv4();
       const response = await makeAPICall(
         "GET",
-        `/api/public/v2/prompts`,
+        baseURI,
         undefined,
         `Bearer ${projectId}`,
       );
@@ -769,19 +773,26 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       });
 
       const otherProjectId = "239ad00f-562f-411d-af14-831c75ddd875";
+      await prisma.organization.upsert({
+        where: { id: "other-org" },
+        create: { id: "other-org", name: "other-org" },
+        update: {},
+      });
+      await prisma.organizationMembership.upsert({
+        where: {
+          orgId_userId: { orgId: "other-org", userId: "user-test" },
+        },
+        create: { userId: "user-test", orgId: "other-org", role: "OWNER" },
+        update: { role: "OWNER" },
+      });
       await prisma.project.upsert({
         where: { id: otherProjectId },
         create: {
           id: otherProjectId,
           name: "demo-app",
-          projectMembers: {
-            create: {
-              role: "OWNER",
-              userId: "user-test",
-            },
-          },
+          orgId: "other-org",
         },
-        update: {},
+        update: { name: "demo-app", orgId: "other-org" },
       });
 
       await createPromptInDB({
@@ -833,20 +844,23 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       // Validate prompt-1 meta
       expect(promptMeta1.name).toBe("prompt-1");
       expect(promptMeta1.versions).toEqual([1, 2, 4]);
-      expect(promptMeta1.labels).toEqual(["production"]);
+      expect(promptMeta1.labels).toEqual(["production", "version2"]);
       expect(promptMeta1.tags).toEqual([]);
+      expect(promptMeta1.lastUpdatedAt).toBeDefined();
 
       // Validate prompt-2 meta
       expect(promptMeta2.name).toBe("prompt-2");
       expect(promptMeta2.versions).toEqual([1, 2, 3]);
       expect(promptMeta2.labels).toEqual(["dev", "production", "staging"]);
       expect(promptMeta2.tags).toEqual([]);
+      expect(promptMeta2.lastUpdatedAt).toBeDefined();
 
       // Validate prompt-3 meta
       expect(promptMeta3.name).toBe("prompt-3");
       expect(promptMeta3.versions).toEqual([1]);
       expect(promptMeta3.labels).toEqual(["production"]);
       expect(promptMeta3.tags).toEqual(["tag-1"]);
+      expect(promptMeta3.lastUpdatedAt).toBeDefined();
 
       // Validate pagination
       expect(body.meta.page).toBe(1);
@@ -870,7 +884,7 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(body.data).toHaveLength(1);
       expect(body.data[0].name).toBe("prompt-1");
       expect(body.data[0].versions).toEqual([1, 2, 4]);
-      expect(body.data[0].labels).toEqual(["production"]);
+      expect(body.data[0].labels).toEqual(["production", "version2"]);
       expect(body.data[0].tags).toEqual([]);
 
       // Validate pagination
@@ -991,6 +1005,112 @@ describe("/api/public/v2/prompts API Endpoint", () => {
     expect(body.meta.totalPages).toBe(3);
     expect(body.meta.totalItems).toBe(3);
   });
+
+  it("should fetch lastConfig correctly for a prompt with multiple versions", async () => {
+    // no filters
+    const response = await makeAPICall("GET", `${baseURI}`);
+    expect(response.status).toBe(200);
+    const body = response.body as unknown as PromptsMetaResponse;
+
+    expect(body.data).toHaveLength(3);
+    expect(body.data.some((promptMeta) => promptMeta.name === "prompt-1")).toBe(
+      true,
+    );
+    expect(body.data.some((promptMeta) => promptMeta.name === "prompt-2")).toBe(
+      true,
+    );
+    expect(body.data.some((promptMeta) => promptMeta.name === "prompt-3")).toBe(
+      true,
+    );
+    const prompt1 = body.data.find(
+      (promptMeta) => promptMeta.name === "prompt-1",
+    );
+    expect(prompt1).toBeDefined();
+    expect(prompt1?.lastConfig).toEqual({ version: 4 });
+
+    const prompt2 = body.data.find(
+      (promptMeta) => promptMeta.name === "prompt-2",
+    );
+    expect(prompt2).toBeDefined();
+    expect(prompt2?.lastConfig).toEqual({});
+
+    // validate with label filter
+    const response2 = await makeAPICall("GET", `${baseURI}?label=version2`);
+    expect(response2.status).toBe(200);
+    const body2 = response2.body as unknown as PromptsMetaResponse;
+
+    expect(body2.data).toHaveLength(1);
+    expect(body2.data[0].name).toBe("prompt-1");
+    expect(body2.data[0].lastConfig).toEqual({ version: 2 });
+
+    // validate with version filter
+    const response3 = await makeAPICall("GET", `${baseURI}?version=1`);
+    expect(response3.status).toBe(200);
+    const body3 = response3.body as unknown as PromptsMetaResponse;
+
+    expect(body3.data).toHaveLength(3);
+    const prompt1v1 = body3.data.find(
+      (promptMeta) => promptMeta.name === "prompt-1",
+    );
+    expect(prompt1v1?.lastConfig).toEqual({ version: 1 });
+  });
+
+  it("should respect the fromUpdatedAt and toUpdatedAt filters on GET /prompts", async () => {
+    // to and from
+    const from = new Date("2024-01-02T00:00:00.000Z");
+    const to = new Date("2024-01-04T00:00:00.000Z");
+    const response = await makeAPICall(
+      "GET",
+      `${baseURI}?fromUpdatedAt=${from.toISOString()}&toUpdatedAt=${to.toISOString()}`,
+    );
+    expect(response.status).toBe(200);
+    const body = response.body as unknown as PromptsMetaResponse;
+
+    expect(body.data).toHaveLength(1);
+    expect(body.data[0].name).toBe("prompt-1");
+    expect(body.data[0].lastUpdatedAt).toBe("2024-01-02T00:00:00.000Z");
+    expect(body.data[0].versions.length).toBe(1);
+
+    expect(body.meta.totalItems).toBe(1);
+
+    // only from
+    const response2 = await makeAPICall(
+      "GET",
+      `${baseURI}?fromUpdatedAt=${from.toISOString()}`,
+    );
+    expect(response2.status).toBe(200);
+    const body2 = response2.body as unknown as PromptsMetaResponse;
+
+    expect(body2.data).toHaveLength(1);
+    expect(body2.data[0].name).toBe("prompt-1");
+    expect(body2.data[0].lastUpdatedAt).toBe("2024-01-04T00:00:00.000Z");
+    expect(body2.data[0].versions.length).toBe(2);
+
+    expect(body2.meta.totalItems).toBe(1);
+
+    // only to
+    const response3 = await makeAPICall(
+      "GET",
+      `${baseURI}?toUpdatedAt=${to.toISOString()}`,
+    );
+    expect(response3.status).toBe(200);
+    const body3 = response3.body as unknown as PromptsMetaResponse;
+
+    expect(body3.data).toHaveLength(3);
+    expect(body3.data[0].name).toBe("prompt-1");
+    expect(body3.data[0].lastUpdatedAt).toBe("2024-01-02T00:00:00.000Z");
+    expect(body3.data[0].versions.length).toBe(2);
+
+    expect(body3.data[1].name).toBe("prompt-2");
+    expect(body3.data[1].lastUpdatedAt).toBe("2000-03-01T00:00:00.000Z");
+    expect(body3.data[1].versions.length).toBe(3);
+
+    expect(body3.data[2].name).toBe("prompt-3");
+    expect(body3.data[2].lastUpdatedAt).toBe("2000-01-01T00:00:00.000Z");
+    expect(body3.data[2].versions.length).toBe(1);
+
+    expect(body3.meta.totalItems).toBe(3);
+  });
 });
 
 const isPrompt = (x: unknown): x is Prompt => {
@@ -1029,17 +1149,19 @@ const mockPrompts = [
     prompt: "prompt-1",
     createdBy: "user-test",
     projectId,
-    config: {},
+    config: { version: 1 },
     version: 1,
+    updatedAt: new Date("2024-01-01T00:00:00.000Z"),
   },
   {
     name: "prompt-1",
-    labels: ["production"],
+    labels: ["production", "version2"],
     prompt: "prompt-1",
     createdBy: "user-test",
     projectId,
-    config: {},
+    config: { version: 2 },
     version: 2,
+    updatedAt: new Date("2024-01-02T00:00:00.000Z"),
   },
   {
     name: "prompt-1",
@@ -1047,8 +1169,9 @@ const mockPrompts = [
     prompt: "prompt-1",
     createdBy: "user-test",
     projectId,
-    config: {},
+    config: { version: 4 },
     version: 4,
+    updatedAt: new Date("2024-01-04T00:00:00.000Z"),
   },
 
   // Prompt with different labels
@@ -1060,6 +1183,7 @@ const mockPrompts = [
     projectId,
     config: {},
     version: 1,
+    updatedAt: new Date("2000-01-01T00:00:00.000Z"),
   },
   {
     name: "prompt-2",
@@ -1069,6 +1193,7 @@ const mockPrompts = [
     projectId,
     config: {},
     version: 2,
+    updatedAt: new Date("2000-03-01T00:00:00.000Z"),
   },
   {
     name: "prompt-2",
@@ -1078,6 +1203,7 @@ const mockPrompts = [
     projectId,
     config: {},
     version: 3,
+    updatedAt: new Date("2000-02-01T00:00:00.000Z"),
   },
 
   // Prompt with different labels
@@ -1090,6 +1216,7 @@ const mockPrompts = [
     config: {},
     tags: ["tag-1"],
     version: 1,
+    updatedAt: new Date("2000-01-01T00:00:00.000Z"),
   },
 
   // Prompt in different project
@@ -1101,5 +1228,6 @@ const mockPrompts = [
     projectId: "239ad00f-562f-411d-af14-831c75ddd875",
     config: {},
     version: 1,
+    updatedAt: new Date("2000-01-01T00:00:00.000Z"),
   },
 ];

web/src/__tests__/query-builder.servertest.ts

@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
-          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id  WHERE  t."project_id" = `,
+          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = `,
           ";",
         ],
       } as const,
@@ -182,13 +182,19 @@ describe("Build valid SQL queries", () => {
 
   describe("should retrieve data", () => {
     it("should get a simple trace", async () => {
+      await prisma.organization.upsert({
+        where: { id: "other-org" },
+        create: { id: "other-org", name: "other-org" },
+        update: {},
+      });
       await prisma.project.upsert({
         where: { id: "different-project-id" },
         create: {
           id: "different-project-id",
           name: "test-project",
+          orgId: "other-org",
         },
-        update: {},
+        update: { name: "test-project", orgId: "other-org" },
       });
 
       await prisma.trace.createMany({

web/src/__tests__/score-configs.servertest.ts

@@ -14,7 +14,7 @@ import {
   GetScoreConfigResponse,
   PostScoreConfigResponse,
   GetScoreConfigsResponse,
-} from "@/src/features/public-api/types/score-configs";
+} from "@langfuse/shared";
 
 const configOne = [
   {

web/src/__tests__/scores.servertest.ts

@@ -11,7 +11,7 @@ import {
   DeleteScoreResponse,
   GetScoreResponse,
   GetScoresResponse,
-} from "@/src/features/public-api/types/scores";
+} from "@langfuse/shared";
 import { z } from "zod";
 import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 

web/src/__tests__/server/api/routers/traces.servertest.ts

@@ -14,12 +14,18 @@ describe("Traces TRPC Router", () => {
     user: {
       id: "clgb17vnp000008jjere5g15i",
       name: "John Doe",
-      projects: [
+      organizations: [
         {
-          id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-          role: "ADMIN",
-          name: "test",
-          cloudConfig: { defaultLookBackDays: null },
+          id: "seed-org-id",
+          role: "OWNER",
+          plan: "cloud:hobby",
+          cloudConfig: undefined,
+          projects: [
+            {
+              id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+              role: "ADMIN",
+            },
+          ],
         },
       ],
       featureFlags: {
@@ -27,6 +33,7 @@ describe("Traces TRPC Router", () => {
       },
       admin: true,
     },
+    environment: {} as any,
   };
 
   const ctx = createInnerTRPCContext({ session });

web/src/__tests__/tokenCost.servertest.ts

@@ -3,9 +3,9 @@ import Decimal from "decimal.js";
 import { v4 as uuidv4 } from "uuid";
 
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { ObservationProcessor } from "@/src/server/api/services/EventProcessor";
 import { ModelUsageUnit } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
+import { ObservationProcessor } from "@langfuse/shared/src/server";
 
 describe("Token Cost Calculation", () => {
   const modelName = "gpt-test-" + nanoid();

web/src/__tests__/traces.servertest.ts

@@ -1,18 +1,19 @@
 /** @jest-environment node */
 
+import { v4 as uuidv4 } from "uuid";
+
 import {
   makeZodVerifiedAPICall,
   pruneDatabase,
 } from "@/src/__tests__/test-utils";
 import { PostGenerationsV1Response } from "@/src/features/public-api/types/generations";
-import { PostScoresResponse } from "@/src/features/public-api/types/scores";
 import {
-  GetTraceV1Response,
   GetTracesV1Response,
+  GetTraceV1Response,
   PostTracesV1Response,
 } from "@/src/features/public-api/types/traces";
+import { PostScoresResponse } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
-import { v4 as uuidv4 } from "uuid";
 
 describe("/api/public/traces API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -33,7 +34,7 @@ describe("/api/public/traces API Endpoint", () => {
         release: "1.0.0",
         version: "2.0.0",
       },
-       );
+    );
 
     const trace = await makeZodVerifiedAPICall(
       GetTraceV1Response,
@@ -64,7 +65,7 @@ describe("/api/public/traces API Endpoint", () => {
         version: "2.0.0",
         public: true,
       },
-       );
+    );
 
     const dbTrace1 = await prisma.trace.findFirst({
       where: {
@@ -94,7 +95,7 @@ describe("/api/public/traces API Endpoint", () => {
         version: "5.0.0",
         public: false,
       },
-       );
+    );
 
     const dbTrace2 = await prisma.trace.findFirst({
       where: {
@@ -125,7 +126,7 @@ describe("/api/public/traces API Endpoint", () => {
         id: "trace-1",
         tags: ["tag-1", "tag-2", "tag-3"],
       },
-       );
+    );
 
     await makeZodVerifiedAPICall(
       PostTracesV1Response,
@@ -135,7 +136,7 @@ describe("/api/public/traces API Endpoint", () => {
         id: "trace-2",
         tags: ["tag-1"],
       },
-       );
+    );
 
     await makeZodVerifiedAPICall(
       PostTracesV1Response,
@@ -145,7 +146,7 @@ describe("/api/public/traces API Endpoint", () => {
         id: "trace-3",
         tags: ["tag-2", "tag-3"],
       },
-       );
+    );
 
     // multiple tags
     const traces = await makeZodVerifiedAPICall(
@@ -206,7 +207,7 @@ describe("/api/public/traces API Endpoint", () => {
         release: "1.0.0",
         version: "2.0.0",
       },
-       );
+    );
     console.log(traceId);
 
     // Simulate observations with costs and latencies
@@ -223,7 +224,7 @@ describe("/api/public/traces API Endpoint", () => {
         startTime: "2021-01-01T00:00:00.000Z",
         endTime: "2021-01-01T00:10:00.000Z",
       },
-       );
+    );
     await makeZodVerifiedAPICall(
       PostGenerationsV1Response,
       "POST",
@@ -234,7 +235,7 @@ describe("/api/public/traces API Endpoint", () => {
         startTime: "2021-01-01T00:10:00.000Z",
         endTime: "2021-01-01T00:20:00.000Z",
       },
-       );
+    );
 
     // Simulate scores on the trace
     const scoreId1 = uuidv4();
@@ -249,7 +250,7 @@ describe("/api/public/traces API Endpoint", () => {
         traceId: traceId,
         comment: "First score",
       },
-       );
+    );
     const scoreId2 = uuidv4();
     await makeZodVerifiedAPICall(
       PostScoresResponse,
@@ -262,7 +263,7 @@ describe("/api/public/traces API Endpoint", () => {
         traceId: traceId,
         comment: "Second score",
       },
-       );
+    );
     const scoreId3 = uuidv4();
     await makeZodVerifiedAPICall(
       PostScoresResponse,
@@ -275,7 +276,7 @@ describe("/api/public/traces API Endpoint", () => {
         traceId: traceId,
         comment: "Third score",
       },
-       );
+    );
 
     // GET traces
     // Retrieve the trace with totalCost and latency
@@ -334,7 +335,7 @@ describe("/api/public/traces API Endpoint", () => {
         release: "1.0.0",
         version: "1.0.0",
       },
-       );
+    );
 
     await makeZodVerifiedAPICall(
       PostTracesV1Response,
@@ -350,7 +351,7 @@ describe("/api/public/traces API Endpoint", () => {
         release: "1.0.0",
         version: "1.0.0",
       },
-       );
+    );
 
     // Filter by session ID
     const tracesBySessionId = await makeZodVerifiedAPICall(

web/src/__tests__/usage.servertest.ts

@@ -1,5 +1,5 @@
+import { tokenCount } from "@/src/features/ingest/usage";
 import { ModelUsageUnit } from "@langfuse/shared";
-import { tokenCount } from "@/src/features/ingest/lib/usage";
 
 describe("Token Count Functions", () => {
   const generateModel = (model: string, tokenizer: string) => {

web/src/components/BatchExportTableButton.tsx

@@ -15,8 +15,8 @@ import {
 import React from "react";
 import { api } from "@/src/utils/api";
 import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useHasOrgEntitlement } from "@/src/features/entitlements/hooks";
 
 export type BatchExportTableButtonProps = {
   projectId: string;
@@ -31,8 +31,8 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
 ) => {
   const [isExporting, setIsExporting] = React.useState(false);
   const createExport = api.batchExport.create.useMutation();
-  const isEeEnabled = useIsEeEnabled();
-  const hasAccess = useHasAccess({
+  const entitled = useHasOrgEntitlement("batch-export");
+  const hasAccess = useHasProjectAccess({
     projectId: props.projectId,
     scope: "batchExport:create",
   });
@@ -56,7 +56,7 @@ export const BatchExportTableButton: React.FC<BatchExportTableButtonProps> = (
     });
   };
 
-  if (!isEeEnabled || !hasAccess) return null;
+  if (!entitled || !hasAccess) return null;
 
   return (
     <DropdownMenu>

web/src/components/EnvLabel.tsx

@@ -0,0 +1,34 @@
+import { AlertTriangle, Check } from "lucide-react";
+import { env } from "@/src/env.mjs";
+import { cn } from "@/src/utils/tailwind";
+import { useSession } from "next-auth/react";
+
+export const EnvLabel = ({ className }: { className?: string }) => {
+  const session = useSession();
+  if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) return null;
+  if (!session.data?.user?.email?.endsWith("@langfuse.com")) return null;
+  return (
+    <div
+      className={cn(
+        "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
+        env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
+          ? "bg-light-blue text-dark-blue ring-dark-blue"
+          : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
+            ? "bg-light-green text-dark-green ring-dark-green"
+            : "bg-light-red text-dark-red ring-dark-red",
+        className,
+      )}
+    >
+      {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
+        <Check size={16} className="hidden sm:block" />
+      ) : (
+        <AlertTriangle size={16} className="hidden sm:block" />
+      )}
+      <span className="whitespace-nowrap">
+        {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)
+          ? `PROD-${env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}`
+          : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
+      </span>
+    </div>
+  );
+};

web/src/components/LangfuseLogo.tsx

@@ -1,124 +1,7 @@
-import {
-  AlertTriangle,
-  BadgeCheck,
-  Check,
-  Github,
-  HardDriveDownload,
-  Map,
-  Newspaper,
-} from "lucide-react";
-
-import { VERSION } from "@/src/constants";
-import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
-import {
-  DropdownMenu,
-  DropdownMenuTrigger,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuLabel,
-  DropdownMenuSeparator,
-} from "@/src/components/ui/dropdown-menu";
-import { ArrowUp } from "lucide-react";
-import { api } from "@/src/utils/api";
-import { Button } from "@/src/components/ui/button";
 import Link from "next/link";
-import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
-
-const VersionLabel = ({ className }: { className?: string }) => {
-  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
-    refetchOnMount: false,
-    refetchOnWindowFocus: false,
-    refetchOnReconnect: false,
-    enabled: !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION, // do not check for updates on Langfuse Cloud
-    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
-  });
-  const isEeVersion =
-    useIsEeEnabled() && !Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
-
-  const hasUpdate =
-    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
-    checkUpdate.data &&
-    checkUpdate.data.updateType;
-
-  const color =
-    checkUpdate.data?.updateType === "major"
-      ? "text-dark-red"
-      : checkUpdate.data?.updateType === "minor"
-        ? "text-dark-yellow"
-        : undefined;
-
-  return (
-    <DropdownMenu>
-      <DropdownMenuTrigger asChild>
-        <Button variant="ghost" size="xs" className={className}>
-          {VERSION}
-          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
-        </Button>
-      </DropdownMenuTrigger>
-      <DropdownMenuContent>
-        {hasUpdate ? (
-          <>
-            <DropdownMenuLabel>
-              New {checkUpdate.data?.updateType} version:{" "}
-              {checkUpdate.data?.latestRelease}
-            </DropdownMenuLabel>
-            <DropdownMenuSeparator />
-          </>
-        ) : !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ? (
-          <>
-            <DropdownMenuLabel>This is the latest release</DropdownMenuLabel>
-            <DropdownMenuSeparator />
-          </>
-        ) : null}
-        {isEeVersion && (
-          <>
-            <DropdownMenuLabel className="flex items-center font-normal">
-              <BadgeCheck size={16} className="mr-2" />
-              Enterprise Edition
-            </DropdownMenuLabel>
-            <DropdownMenuSeparator />
-          </>
-        )}
-        <DropdownMenuItem asChild>
-          <Link
-            href="https://github.com/langfuse/langfuse/releases"
-            target="_blank"
-          >
-            <Github size={16} className="mr-2" />
-            Releases
-          </Link>
-        </DropdownMenuItem>
-        <DropdownMenuItem asChild>
-          <Link href="https://langfuse.com/changelog" target="_blank">
-            <Newspaper size={16} className="mr-2" />
-            Changelog
-          </Link>
-        </DropdownMenuItem>
-        <DropdownMenuItem asChild>
-          <Link href="https://langfuse.com/roadmap" target="_blank">
-            <Map size={16} className="mr-2" />
-            Roadmap
-          </Link>
-        </DropdownMenuItem>
-        {hasUpdate && (
-          <>
-            <DropdownMenuSeparator />
-            <DropdownMenuItem asChild>
-              <Link
-                href="https://langfuse.com/docs/deployment/self-host#update"
-                target="_blank"
-              >
-                <HardDriveDownload size={16} className="mr-2" />
-                Update
-              </Link>
-            </DropdownMenuItem>
-          </>
-        )}
-      </DropdownMenuContent>
-    </DropdownMenu>
-  );
-};
+import { EnvLabel } from "./EnvLabel";
+import { VersionLabel } from "./VersionLabel";
 
 export const LangfuseIcon = ({
   size = 32,
@@ -152,40 +35,20 @@ export const LangfuseLogo = ({
     className={cn("flex flex-wrap gap-4 lg:flex-col lg:items-start", className)}
   >
     {/* Environment Labeling for Langfuse Maintainers */}
-    {showEnvLabel && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && (
-      <div
-        className={cn(
-          "flex items-center gap-2 self-stretch rounded-md px-1 py-1 text-xs ring-1 sm:px-3 sm:py-2 lg:-mx-2",
-          env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
-            ? "bg-light-blue text-dark-blue ring-dark-blue"
-            : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV"
-              ? "bg-light-green text-dark-green ring-dark-green"
-              : "bg-light-red text-dark-red ring-dark-red",
-        )}
-      >
-        {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "DEV" ? (
-          <Check size={16} className="hidden sm:block" />
-        ) : (
-          <AlertTriangle size={16} className="hidden sm:block" />
-        )}
-        <span className="whitespace-nowrap">
-          {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)
-            ? `PROD-${env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}`
-            : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-        </span>
-      </div>
-    )}
+    {showEnvLabel && <EnvLabel />}
     {/* Langfuse Logo */}
     <div className="flex items-center">
-      <LangfuseIcon size={size === "sm" ? 16 : 20} />
-      <span
-        className={cn(
-          "ml-2 font-mono font-semibold",
-          size === "sm" ? "text-sm" : "text-lg",
-        )}
-      >
-        Langfuse
-      </span>
+      <Link href="/" className="flex items-center">
+        <LangfuseIcon size={size === "sm" ? 16 : 20} />
+        <span
+          className={cn(
+            "ml-2 font-mono font-semibold",
+            size === "sm" ? "text-sm" : "text-xl",
+          )}
+        >
+          Langfuse
+        </span>
+      </Link>
       {version && <VersionLabel className="ml-2" />}
     </div>
   </div>

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -1,16 +1,16 @@
 import Link from "next/link";
 
 import { Label } from "@/src/components/ui/label";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { api } from "@/src/utils/api";
 import { type UIModelParams } from "@langfuse/shared";
 import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 
 export const LLMApiKeyComponent = (p: {
   projectId: string;
   modelParams: UIModelParams;
 }) => {
-  const hasAccess = useHasAccess({
+  const hasAccess = useHasProjectAccess({
     projectId: p.projectId,
     scope: "llmApiKeys:read",
   });

web/src/components/PagedSettingsContainer.tsx

@@ -0,0 +1,111 @@
+import { cn } from "@/src/utils/tailwind";
+import { ArrowUpRight } from "lucide-react";
+import Link from "next/link";
+import { type ReactNode } from "react";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { useRouter } from "next/router";
+
+type SettingsProps = {
+  pages: Array<
+    {
+      title: string;
+      slug: string;
+      show?: boolean | (() => boolean);
+    } & ({ content: ReactNode } | { href: string })
+  >;
+  activeSlug?: string;
+};
+
+export const PagedSettingsContainer = ({
+  pages,
+  activeSlug,
+}: SettingsProps) => {
+  const router = useRouter();
+  const availablePages = pages.filter((page) =>
+    "show" in page
+      ? typeof page.show === "function"
+        ? page.show()
+        : page.show
+      : true,
+  );
+
+  const currentPage =
+    availablePages.find((page) => page.slug === activeSlug) ??
+    availablePages[0]; // Fallback to first page if not found
+
+  const onChange = (newSlug: string) => {
+    const pathSegments = router.asPath.split("/");
+    if (pathSegments[pathSegments.length - 1] !== "settings")
+      pathSegments.pop();
+    if (newSlug !== "index") pathSegments.push(newSlug);
+    router.push(pathSegments.join("/"));
+  };
+
+  return (
+    <main className="flex flex-1 flex-col gap-4 py-4 md:gap-8">
+      <div className="grid w-full items-start gap-4 md:grid-cols-[180px_1fr] lg:grid-cols-[220px_1fr]">
+        <nav className="block md:hidden">
+          <Select
+            onValueChange={(slug) => {
+              const page = availablePages.find((p) => p.slug === slug);
+              if (page && "href" in page) router.push(page.href);
+              else onChange(slug);
+            }}
+            value={currentPage.slug}
+          >
+            <SelectTrigger>
+              <SelectValue placeholder="Select a page" />
+            </SelectTrigger>
+            <SelectContent>
+              {availablePages.map((page) => (
+                <SelectItem key={page.title} value={page.slug}>
+                  {page.title}
+                  {"href" in page && (
+                    <ArrowUpRight size={14} className="ml-1 inline" />
+                  )}
+                </SelectItem>
+              ))}
+            </SelectContent>
+          </Select>
+        </nav>
+        <nav
+          className="hidden gap-4 text-sm text-muted-foreground md:sticky md:top-5 md:grid"
+          x-chunk="dashboard-04-chunk-0"
+        >
+          {availablePages.map((page) =>
+            "href" in page ? (
+              <Link
+                key={page.title}
+                href={page.href}
+                className="flex flex-row items-center gap-2 font-semibold"
+              >
+                {page.title}
+                <ArrowUpRight size={14} className="inline" />
+              </Link>
+            ) : (
+              <span
+                key={page.title}
+                onClick={() => onChange(page.slug)}
+                className={cn(
+                  "cursor-pointer font-semibold",
+                  page.slug === currentPage.slug && "text-primary",
+                )}
+              >
+                {page.title}
+              </span>
+            ),
+          )}
+        </nav>
+        <div className="w-full overflow-hidden p-1">
+          {currentPage && "content" in currentPage ? currentPage.content : null}
+        </div>
+      </div>
+    </main>
+  );
+};

web/src/components/PosthogLogo.tsx

@@ -0,0 +1,35 @@
+// https://react-svgr.com/playground/?typescript=true
+
+import * as React from "react";
+import { type SVGProps } from "react";
+export const PostHogLogo = (props: SVGProps<SVGSVGElement>) => (
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140" {...props}>
+    <g fill="none">
+      <path
+        fill="#1D4AFF"
+        d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z"
+      />
+      <path
+        fill="#F9BD2B"
+        d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z"
+      />
+      <path
+        fill="currentColor"
+        className="bg-primary"
+        d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z"
+      />
+      <path
+        fill="#1D4AFF"
+        d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z"
+      />
+      <path
+        fill="#F54E00"
+        d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z"
+      />
+      <path
+        fill="currentColor"
+        d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z"
+      />
+    </g>
+  </svg>
+);

web/src/components/SettingsDangerZone.tsx

@@ -0,0 +1,30 @@
+import Header from "@/src/components/layouts/header";
+import React from "react";
+
+export const SettingsDangerZone: React.FC<{
+  items: {
+    title: string;
+    description: string;
+    button: React.ReactNode;
+  }[];
+}> = ({ items }) => {
+  return (
+    <div className="space-y-3">
+      <Header title="Danger Zone" level="h3" />
+      <div className="rounded border">
+        {items.map((item, index) => (
+          <div
+            key={index}
+            className="flex items-center justify-between gap-4 border-b p-4 last:border-b-0"
+          >
+            <div>
+              <h4 className="font-semibold">{item.title}</h4>
+              <p className="text-sm">{item.description}</p>
+            </div>
+            {item.button}
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+};

web/src/components/Support.tsx

@@ -27,7 +27,7 @@ const supportChannels = [
     description:
       "Create an issue on Github to report bugs or request new features.",
     href: "https://github.com/langfuse/langfuse/issues/new/choose",
-    buttonText: "Create issue",
+    buttonText: "Create Issue",
     primary: true,
   },
   {
@@ -35,7 +35,7 @@ const supportChannels = [
     title: "GitHub Support",
     description: "Create a support ticket via GitHub discussions.",
     href: "https://github.com/orgs/langfuse/discussions/categories/support",
-    buttonText: "Submit question",
+    buttonText: "Submit Question",
     primary: true,
   },
   {

web/src/components/VersionLabel.tsx

@@ -0,0 +1,116 @@
+import {
+  BadgeCheck,
+  Github,
+  HardDriveDownload,
+  Map,
+  Newspaper,
+} from "lucide-react";
+import { VERSION } from "@/src/constants";
+import Link from "next/link";
+import {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+} from "@/src/components/ui/dropdown-menu";
+import { ArrowUp } from "lucide-react";
+import { api } from "@/src/utils/api";
+import { Button } from "@/src/components/ui/button";
+import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
+import { env } from "@/src/env.mjs";
+
+export const VersionLabel = ({ className }: { className?: string }) => {
+  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
+    refetchOnMount: false,
+    refetchOnWindowFocus: false,
+    refetchOnReconnect: false,
+    enabled: !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION, // do not check for updates on Langfuse Cloud
+    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
+  });
+  const isEeVersion = useIsEeEnabled();
+
+  const hasUpdate =
+    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
+    checkUpdate.data &&
+    checkUpdate.data.updateType;
+
+  const color =
+    checkUpdate.data?.updateType === "major"
+      ? "text-dark-red"
+      : checkUpdate.data?.updateType === "minor"
+        ? "text-dark-yellow"
+        : undefined;
+
+  return (
+    <DropdownMenu>
+      <DropdownMenuTrigger asChild>
+        <Button variant="ghost" size="xs" className={className}>
+          {VERSION}
+          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
+        </Button>
+      </DropdownMenuTrigger>
+      <DropdownMenuContent onClick={(e) => e.stopPropagation()}>
+        {hasUpdate ? (
+          <>
+            <DropdownMenuLabel>
+              New {checkUpdate.data?.updateType} version:{" "}
+              {checkUpdate.data?.latestRelease}
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ? (
+          <>
+            <DropdownMenuLabel>This is the latest release</DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        ) : null}
+        {isEeVersion && (
+          <>
+            <DropdownMenuLabel className="flex items-center font-normal">
+              <BadgeCheck size={16} className="mr-2" />
+              Enterprise Edition
+            </DropdownMenuLabel>
+            <DropdownMenuSeparator />
+          </>
+        )}
+        <DropdownMenuItem asChild>
+          <Link
+            href="https://github.com/langfuse/langfuse/releases"
+            target="_blank"
+          >
+            <Github size={16} className="mr-2" />
+            Releases
+          </Link>
+        </DropdownMenuItem>
+        <DropdownMenuItem asChild>
+          <Link href="https://langfuse.com/changelog" target="_blank">
+            <Newspaper size={16} className="mr-2" />
+            Changelog
+          </Link>
+        </DropdownMenuItem>
+        <DropdownMenuItem asChild>
+          <Link href="https://langfuse.com/roadmap" target="_blank">
+            <Map size={16} className="mr-2" />
+            Roadmap
+          </Link>
+        </DropdownMenuItem>
+        {hasUpdate && (
+          <>
+            <DropdownMenuSeparator />
+            <DropdownMenuItem asChild>
+              <Link
+                href="https://langfuse.com/docs/deployment/self-host#update"
+                target="_blank"
+              >
+                <HardDriveDownload size={16} className="mr-2" />
+                Update
+              </Link>
+            </DropdownMenuItem>
+          </>
+        )}
+      </DropdownMenuContent>
+    </DropdownMenu>
+  );
+};

web/src/components/date-range-dropdowns.tsx

@@ -16,14 +16,14 @@ import {
   type DashboardDateRangeOptions,
   type TableDateRangeOptions,
   DASHBOARD_AGGREGATION_OPTIONS,
-  TABLE_AGGREGATION_OPTIONS,
   type DashboardDateRange,
+  TABLE_RANGE_DROPDOWN_OPTIONS,
 } from "@/src/utils/date-range-utils";
 import { Clock } from "lucide-react";
 
 type BaseDateRangeDropdownProps<T> = {
   selectedOption: T;
-  options: T[];
+  options: readonly T[];
   onSelectionChange: (value: T) => void;
 };
 
@@ -118,7 +118,7 @@ export const TableDateRangeDropdown: React.FC<TableDateRangeDropdownProps> = ({
   return (
     <BaseDateRangeDropdown
       selectedOption={selectedOption}
-      options={[...TABLE_AGGREGATION_OPTIONS, DEFAULT_AGGREGATION_SELECTION]}
+      options={TABLE_RANGE_DROPDOWN_OPTIONS}
       onSelectionChange={onDropDownSelection}
     />
   );

web/src/components/deleteButton.tsx

@@ -7,8 +7,8 @@ import {
 } from "@/src/components/ui/popover";
 import { Button } from "@/src/components/ui/button";
 import { TrashIcon } from "lucide-react";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { type Scope } from "@/src/features/rbac/constants/roleAccessRights";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { type ProjectScope } from "@/src/features/rbac/constants/projectAccessRights";
 import { api } from "@/src/utils/api";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
@@ -16,7 +16,7 @@ interface DeleteButtonProps {
   itemId: string;
   projectId: string;
   isTableAction?: boolean;
-  scope: Scope;
+  scope: ProjectScope;
   invalidateFunc: () => void;
   type: "trace" | "dataset";
   redirectUrl?: string;
@@ -35,7 +35,7 @@ export function DeleteButton({
   const router = useRouter();
   const capture = usePostHogClientCapture();
 
-  const hasAccess = useHasAccess({ projectId, scope: scope });
+  const hasAccess = useHasProjectAccess({ projectId, scope: scope });
   const traceMutation = api.traces.deleteMany.useMutation({
     onSuccess: () => {
       setIsDeleted(true);

web/src/components/grouped-score-badge.tsx

@@ -4,89 +4,61 @@ import {
   HoverCardContent,
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
-import {
-  isBooleanDataType,
-  isCategoricalDataType,
-} from "@/src/features/manual-scoring/lib/helpers";
-import { type ScoreSimplified } from "@/src/server/api/routers/generations/getAllQuery";
-import { cn } from "@/src/utils/tailwind";
 import { MessageCircleMore } from "lucide-react";
 
-export const GroupedScoreBadges = ({
+import { type APIScore } from "@langfuse/shared";
+import { type LastUserScore } from "@/src/features/scores/lib/types";
+
+export const GroupedScoreBadges = <T extends APIScore | LastUserScore>({
   scores,
-  variant = "badge",
 }: {
-  scores: ScoreSimplified[];
-  variant?: "badge" | "headings";
+  scores: T[];
 }) => {
-  const groupedScores = scores.reduce<Record<string, ScoreSimplified[]>>(
-    (acc, score) => {
-      if (!acc[score.name] || !Array.isArray(acc[score.name])) {
-        acc[score.name] = [score];
-      } else {
-        (acc[score.name] as ScoreSimplified[]).push(score);
-      }
-      return acc;
-    },
-    {},
-  );
+  const groupedScores = scores.reduce<Record<string, T[]>>((acc, score) => {
+    if (!acc[score.name] || !Array.isArray(acc[score.name])) {
+      acc[score.name] = [score];
+    } else {
+      acc[score.name].push(score);
+    }
+    return acc;
+  }, {});
 
-  const ScoresOfGroup = (props: {
-    scores: ScoreSimplified[];
-    className?: string;
-  }) => (
-    <div className={cn("text-xs", props.className)}>
-      {props.scores.map((s, i) => (
-        <span key={i} className="group/score ml-1 first:ml-0">
-          {isCategoricalDataType(s.dataType) || isBooleanDataType(s.dataType)
-            ? s.stringValue
-            : s.value?.toFixed(2)}
-          {s.comment && (
-            <HoverCard>
-              <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
-                <MessageCircleMore size={12} />
-              </HoverCardTrigger>
-              <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
-                <p>{s.comment}</p>
-              </HoverCardContent>
-            </HoverCard>
-          )}
-          <span className="group-last/score:hidden">,</span>
-        </span>
-      ))}
-    </div>
-  );
-
-  if (variant === "headings")
-    return (
-      <div className="flex items-center gap-3 text-nowrap">
-        {Object.entries(groupedScores)
-          .sort(([a], [b]) => (a < b ? -1 : 1))
-          .map(([name, scores]) => (
-            <div key={name}>
-              <div className="text-xs text-muted-foreground">{name}</div>
-              <ScoresOfGroup scores={scores} />
+  return (
+    <>
+      {Object.entries(groupedScores)
+        .sort(([a], [b]) => (a < b ? -1 : 1))
+        .map(([name, scores]) => (
+          <Badge
+            variant="outline"
+            key={name}
+            className="grid grid-cols-[1fr,auto] gap-1 font-normal"
+          >
+            <p className="truncate" title={name}>
+              {name}:
+            </p>
+            <div className="flex items-center gap-1 text-nowrap">
+              {scores.map((s, i) => (
+                <span
+                  key={i}
+                  className="group/score ml-1 rounded-sm first:ml-0"
+                >
+                  {s.stringValue ?? s.value?.toFixed(2) ?? ""}
+                  {s.comment && (
+                    <HoverCard>
+                      <HoverCardTrigger className="ml-1 inline-block cursor-pointer">
+                        <MessageCircleMore size={12} />
+                      </HoverCardTrigger>
+                      <HoverCardContent className="overflow-hidden whitespace-normal break-normal">
+                        <p>{s.comment}</p>
+                      </HoverCardContent>
+                    </HoverCard>
+                  )}
+                  <span className="group-last/score:hidden">,</span>
+                </span>
+              ))}
             </div>
-          ))}
-      </div>
-    );
-  else
-    return (
-      <>
-        {Object.entries(groupedScores)
-          .sort(([a], [b]) => (a < b ? -1 : 1))
-          .map(([name, scores]) => (
-            <Badge
-              variant="outline"
-              key={name}
-              className="grid grid-cols-[1fr,auto] gap-1 font-normal"
-            >
-              <p className="truncate" title={name}>
-                {name}:
-              </p>
-              <ScoresOfGroup scores={scores} />
-            </Badge>
-          ))}
-      </>
-    );
+          </Badge>
+        ))}
+    </>
+  );
 };

web/src/components/json-editor.tsx

@@ -7,8 +7,6 @@ import { useTheme } from "next-themes";
 import { cn } from "@/src/utils/tailwind";
 import { useState } from "react";
 
-// todo: add json linting
-
 export function JsonEditor({
   defaultValue,
   onChange,
@@ -24,7 +22,7 @@ export function JsonEditor({
 }) {
   const { resolvedTheme } = useTheme();
   const codeMirrorTheme = resolvedTheme === "dark" ? tokyoNight : githubLight;
-  
+
   // used to disable linter when field is empty
   const [linterEnabled, setLinterEnabled] = useState<boolean>(
     !!defaultValue && defaultValue !== "",