chore: release v2.72.0

* style(ui): align styling of comment button on sessions page

* style(dashboard): score analytics charts full width for mobile

* style(markdown): fix overflow behavior on sessions page for mobile

.devcontainer/devcontainer.json

@@ -1,14 +0,0 @@
-{
-	"name": "langfuse dev",
-    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
-    "features": {
-        "ghcr.io/devcontainers/features/docker-in-docker:2": {
-        "version": "latest",
-        "moby": true
-        }
-    },
-    "forwardPorts": [3000],
-
-    "onCreateCommand": "npm install -g pnpm@8.15.5",
-    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
-}

.dockerignore

@@ -3,6 +3,6 @@ Dockerfile
 node_modules
 npm-debug.log
 README.md
-.next
+**/.next
 .git
 **/node_modules

.env.dev.example

@@ -6,6 +6,12 @@
 DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+
 # Next Auth
 # You can generate a new secret on the command line with:
 # openssl rand -base64 32
@@ -44,4 +50,4 @@ REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 # openssl rand -hex 32 used only here
-ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 

.env.local.example

@@ -18,4 +18,7 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=myworkerpassword
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -9,25 +9,33 @@
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
+# optional, set to true to disable automated database migrations on Docker start
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED=
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
 NEXTAUTH_URL="http://localhost:3000"
 
-# You can generate a new secret on the command line with:
+# For each of these, you can generate a new secret on the command line with:
 # openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-NEXTAUTH_SECRET="secret"
-SALT="salt"
+NEXTAUTH_SECRET="secret" # https://next-auth.js.org/configuration/options#secret
+SALT="salt" # salt used to hash api keys
+
+# API level encryption for sensitive data
+# Must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32
+ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000"
 
 # Use CSP headers to enforce HTTPS, optional
-LANGFUSE_CSP_ENFORCE_HTTPS="true"
+# LANGFUSE_CSP_ENFORCE_HTTPS="true"
+
 
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
 
-# Default project, optional
+# Default role for users who sign up, optional, can be org or org+project
+# LANGFUSE_DEFAULT_ORG_ID=
+# LANGFUSE_DEFAULT_ORG_ROLE=
 # LANGFUSE_DEFAULT_PROJECT_ID=
 # LANGFUSE_DEFAULT_PROJECT_ROLE=
 
@@ -38,11 +46,13 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
+# AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
 
 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
 # AUTH_GOOGLE_CLIENT_SECRET=
 # AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GOOGLE_ALLOWED_DOMAINS=langfuse.com,google.com # optional allowlist of workspace domains that can sign in via Google
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
@@ -66,6 +76,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
@@ -80,13 +91,26 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # S3_SECRET_ACCESS_KEY=
 # S3_BUCKET_NAME=
 # S3_REGION=
+# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
 
 # Exports are streamed to S3 in pages to avoid memory issues
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
-# Disable logging to `events` table in db
-# ENABLE_EVENT_LOG=false
+
+
+### START Enterprise Edition Configuration
+
+# Allowlisted users that can create new organizations, by default all users can create organizations
+# LANGFUSE_ALLOWED_ORGANIZATION_CREATORS=user1@langfuse.com,user2@langfuse.com
+
+# UI Customization Options
+# LANGFUSE_UI_API_HOST=https://api.example.com
+# LANGFUSE_UI_DOCUMENTATION_HREF=https://docs.example.com
+# LANGFUSE_UI_SUPPORT_HREF=https://support.example.com
+# LANGFUSE_UI_FEEDBACK_HREF=https://feedback.example.com
+
+### END Enterprise Edition Configuration
 
 
 
@@ -95,9 +119,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Disable the expensive analytics queries and related features
 # LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
 
-# Customize the default datetime offset filter used in for the UI tables
-# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
-
 ### END Envs to be deprecated in Langfuse v3.0
 
 
@@ -122,24 +143,62 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_SENTRY_ORG=
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
+# Used to determine the Sentry sample rate
+# LANGFUSE_TRACING_SAMPLE_RATE=
+# LANGFUSE_WORKER_BETTERSTACK_TOKEN=
+
 
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Betterstack
-# LANGFUSE_TEAM_BETTERSTACK_TOKEN=
-
 # Demo project that users can use to try the platform
+# NEXT_PUBLIC_DEMO_ORG_ID=
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
-# Encryption
-# ENCRYPTION_KEY=
-
 # Admin API
 # ADMIN_API_KEY=
 
+# Redis
+# REDIS_HOST=
+# REDIS_PORT=
+# REDIS_AUTH=
+# REDIS_CONNECTION_STRING=
+
+# Cache configuration
+# LANGFUSE_CACHE_API_KEY_ENABLED=
+# LANGFUSE_CACHE_API_KEY_TTL_SECONDS=
+# LANGFUSE_CACHE_PROMPT_ENABLED=
+# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
+
+# Stripe
+# STRIPE_SECRET_KEY=
+# STRIPE_WEBHOOK_SIGNING_SECRET=
+
 ### END Langfuse Cloud Config
+
+
+## START Langfuse V3 Ingestion 
+
+# Clickhouse
+# CLICKHOUSE_MIGRATION_URL=
+# CLICKHOUSE_URL=
+# CLICKHOUSE_USER=
+# CLICKHOUSE_PASSWORD=
+
+# Ingestion
+# LANGFUSE_INGESTION_BUFFER_TTL_SECONDS=
+# LANGFUSE_INGESTION_FLUSH_DELAY_MS= 
+# LANGFUSE_INGESTION_FLUSH_ATTEMPTS= 
+# LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE= 
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
+# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
+# LANGFUSE_LOG_LEVEL= 
+# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
+# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+
+## END Langfuse V3 Ingestion

.github/workflows/pipeline.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -63,10 +63,13 @@ jobs:
           fi
 
   tests-web:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
+    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -76,7 +79,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -91,12 +94,15 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          cp .env.dev.example web/.env
+          grep -v -e '^REDIS_HOST=' .env.dev.example > .env
 
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
 
       - name: Seed DB
         run: |
@@ -113,10 +119,13 @@ jobs:
         run: pnpm --filter=web run test
 
   tests-worker:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -126,7 +135,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -138,18 +147,27 @@ jobs:
         run: |
           pnpm install
 
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+
       - name: Load default env
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
+          cp .env.dev.example worker/.env
 
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
 
       - name: Seed DB
         run: |
+          pnpm run --filter=shared ch:up
           pnpm run db:migrate
           pnpm run db:seed
 
@@ -165,7 +183,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -184,6 +202,7 @@ jobs:
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
 
       - name: Seed DB
@@ -200,10 +219,62 @@ jobs:
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
 
+  e2e-server-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
+          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed:examples
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Run server
+        run: (pnpm run start&)
+
+      - name: Run e2e tests
+        run: pnpm --filter=web run test:e2e:server
+
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
+    needs:
+      [
+        lint,
+        tests-web,
+        tests-worker,
+        e2e-tests,
+        test-docker-build,
+        e2e-server-tests,
+      ]
     if: always()
     steps:
       - name: Successful deploy
@@ -227,7 +298,7 @@ jobs:
     steps:
       - uses: pnpm/action-setup@v3
         with:
-          version: 8.15.5
+          version: 9.5.0
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -250,6 +321,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
@@ -273,6 +352,9 @@ jobs:
           push: true
           tags: ${{ steps.meta-web.outputs.tags }}
           labels: ${{ steps.meta-web.outputs.labels }}
+          platforms: |
+            linux/amd64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
@@ -297,3 +379,6 @@ jobs:
           push: true
           tags: ${{ steps.meta-worker.outputs.tags }}
           labels: ${{ steps.meta-worker.outputs.labels }}
+          platforms: |
+            linux/amd64
+            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}

.github/workflows/porter_stack_web_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to web
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4060"
+          PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_web_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to web (Staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_web_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to web (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/porter_stack_worker.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-    - main
-name: Deploy to worker
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Setup porter
-      uses: porter-dev/setup-porter@v0.1.0
-    - name: Deploy stack
-      timeout-minutes: 30
-      run: exec porter apply
-      env:
-        PORTER_CLUSTER: "3959"
-        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
-        PORTER_HOST: https://dashboard.getporter.dev
-        PORTER_PR_NUMBER: ${{ github.event.number }}
-        PORTER_PROJECT: "12565"
-        PORTER_REPO_NAME: ${{ github.event.repository.name }}
-        PORTER_STACK_NAME: worker
-        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (EU temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4060"
+          PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to worker (staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_worker_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/snyk.yml

@@ -1,5 +1,12 @@
 name: Snyk Container
-on: push
+on:
+  push:
+    branches-ignore:
+      - "dependabot/**"
+  pull_request:
+    branches:
+      - "**"
+
 jobs:
   snyk:
     runs-on: ubuntu-latest
@@ -18,7 +25,7 @@ jobs:
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: web
@@ -33,7 +40,7 @@ jobs:
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: worker

CONTRIBUTING.md

@@ -53,9 +53,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
-> [!NOTE]
-> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+**Langfuse v2**
 
 ```mermaid
 flowchart TB
@@ -93,15 +91,66 @@ flowchart TB
         I --- Prisma
     end
 
-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker_API[Public HTTP API]
+    Prisma --- DB
+    JS --- API
+    Python --- API
+```
+
+**Langfuse v3 (work in progress, not released yet)**
+
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
+
+```mermaid
+flowchart TB
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
     end
 
-    API --> Worker_API
-    Worker_API --- DB
-    Worker_API --- Redis
+    DB[Postgres Database]
+    Redis[Redis Cache/Queue]
+    Clickhouse[Clickhouse Database]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        ORM
+        H --> G
+        H --> I
+        G --> I
+        G --- ORM
+        API --- ORM
+        I --- ORM
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker
+    end
+
+    Worker --- DB
+    Worker --- Redis
+    Worker --- Clickhouse
+
+    ORM --- DB
+    ORM --- Redis
+    ORM --- Clickhouse
 
-    Prisma --- DB
     JS --- API
     Python --- API
 ```
@@ -116,7 +165,7 @@ flowchart LR
    Browser ---|Web UI & TRPC API| App
    Integrations/SDKs ---|Public HTTP API| App
    subgraph i1["Application Network"]
-      App["Langfuse Application (Docker or Serverless)"]
+      App["Langfuse Application"]
    end
    subgraph i2["Database Network"]
       DB["Postgres Database"]
@@ -149,6 +198,7 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/
 Requirements
 
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
+- Pnpm v.9.5.0
 - Docker to run the database locally
 
 **Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".

README.md

@@ -1,4 +1,8 @@
-![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+![langfuse_logo_1024](https://github.com/user-attachments/assets/f17d8fc1-8206-4afa-a5ec-774826be88a8)
+
+<div align="center"><h1> Langfuse: Open Source LLM Observability & Engineering Platform</h4></div>
+<div align="center"><h2> Debug and improve your LLM app</h4></div>
+<div align="center"><h3> LLM Observability, Prompt Management, LLM Evaluations, Datasets, LLM Metrics and Prompt Playground</h4></div>
 
 <div align="center">
    <div>
@@ -24,6 +28,8 @@
    </div>
    <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
+   <span>We're hiring. <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in Backend Engineering, Product Engineering, and Developer Relations.</span>
+   <br/>
    <br/>
    <div>
       <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
@@ -35,7 +41,7 @@
 </div>
 </br>
 
-## Overview
+## Langfuse Overview
 
 _Unmute video for voice-over_
 
@@ -43,18 +49,18 @@ https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6
 
 ### Develop
 
-- **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
+- **LLM Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
 - **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
 - **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
-- **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
-- **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
+- **LLM Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
+- **LLM Evaluations:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
+  - Run ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) and LLM-as-a-Judge within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
-  - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
+  - Manually score LLM outputs in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 
 ### Test
 
@@ -70,6 +76,8 @@ Managed deployment by the Langfuse team, generous free-tier (hobby plan), no cre
 
 **[» Langfuse Cloud](https://cloud.langfuse.com)**
 
+## Self-Hosting Open Source LLM Observability with Langfuse
+
 ### Localhost (docker)
 
 ```bash
@@ -96,7 +104,7 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe
 
 You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
-### Ingesting Data · Instrumenting Your Application
+### Ingesting Data · Instrumenting Your Application · LLM Observability with Langfuse
 
 Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
@@ -104,7 +112,7 @@ You may want to use another integration to get started quickly or implement a us
 
 See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.
 
-### Integrations
+### LLM Observability Integrations
 
 | Integration                                              | Supports                   | Description                                                                                                                                      |
 | -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
@@ -125,15 +133,14 @@ Packages that integrate with Langfuse:
 | [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
 | [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
 | [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
-| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
 
 ## Questions and feedback
 
 ### Ideas and roadmap
 
-- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
-- [Feature Requests](https://langfuse.com/idea)
 - [Roadmap](https://langfuse.com/roadmap)
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
+- [Feature Requests](https://langfuse.com/idea
 
 ### Support and feedback
 
@@ -157,7 +164,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

careers

@@ -3,7 +3,6 @@ Join us in scaling Langfuse in Berlin, Germany. We are an open source company, w
 
 _Open Roles_
 
-- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
 - Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
 - Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
 

docker-compose.build.yml

@@ -5,6 +5,8 @@ services:
     build:
       dockerfile: ./web/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis
@@ -14,6 +16,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
@@ -30,6 +33,8 @@ services:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis

docker-compose.dev.yml

@@ -1,6 +1,22 @@
-version: "3.5"
-
 services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
   redis:
     image: redis:7.2.4
     restart: always
@@ -9,8 +25,8 @@ services:
     ports:
       - 6379:6379
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     command: ["postgres", "-c", "log_statement=all"]
     environment:
@@ -20,8 +36,12 @@ services:
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
     driver: local

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.5"
-
 services:
   langfuse-server:
     image: langfuse/langfuse:2
@@ -12,6 +10,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

ee/package.json

@@ -15,23 +15,24 @@
     }
   },
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
+    "@opentelemetry/api": ">=1.0.0 <1.9.0",
     "next": "^14.2.3",
     "next-auth": "^4.24.7",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/node": "^20.11.29",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",

ee/src/ee-license-check/index.ts

@@ -1,4 +1,5 @@
 import { env } from "../env";
 
 export const isEeAvailable: boolean =
-  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ||
+  env.LANGFUSE_EE_LICENSE_KEY !== undefined;

ee/src/env.ts

@@ -3,7 +3,7 @@ import { env as sharedEnv } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
-  ADMIN_API_KEY: z.string().optional(),
+  LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
 export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };

ee/src/index.ts

@@ -1 +0,0 @@
-export { isEeAvailable } from "./ee-license-check";

fern/apis/client/definition/score.yml

@@ -9,7 +9,6 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: Score
 
 types:
   CreateScoreRequest:
@@ -17,15 +16,119 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -5,16 +5,36 @@ types:
       id:
         type: string
         docs: The unique identifier of a trace
-      timestamp: datetime
-      name: optional<string>
-      input: optional<unknown>
-      output: optional<unknown>
-      sessionId: optional<string>
-      release: optional<string>
-      version: optional<string>
-      userId: optional<string>
-      metadata: optional<unknown>
-      tags: optional<list<string>>
+      timestamp:
+        type: datetime
+        docs: The timestamp when the trace was created
+      name:
+        type: optional<string>
+        docs: The name of the trace
+      input:
+        type: optional<unknown>
+        docs: The input data of the trace. Can be any JSON.
+      output:
+        type: optional<unknown>
+        docs: The output data of the trace. Can be any JSON.
+      sessionId:
+        type: optional<string>
+        docs: The session identifier associated with the trace
+      release:
+        type: optional<string>
+        docs: The release version of the application when the trace was created
+      version:
+        type: optional<string>
+        docs: The version of the trace
+      userId:
+        type: optional<string>
+        docs: The user identifier associated with the trace
+      metadata:
+        type: optional<unknown>
+        docs: The metadata associated with the trace. Can be any JSON.
+      tags:
+        type: optional<list<string>>
+        docs: The tags associated with the trace. Can be an array of strings or null.
       public:
         type: optional<boolean>
         docs: Public traces are accessible via url without login
@@ -42,11 +62,18 @@ types:
       htmlPath:
         type: string
         docs: Path of trace in Langfuse UI
+      latency:
+        type: double
+        docs: Latency of trace in seconds
       totalCost:
         type: double
         docs: Cost of trace in USD
-      observations: list<ObservationsView>
-      scores: list<Score>
+      observations:
+        type: list<ObservationsView>
+        docs: List of observations
+      scores:
+        type: list<Score>
+        docs: List of scores
   Session:
     properties:
       id: string
@@ -58,36 +85,97 @@ types:
       traces: list<Trace>
   Observation:
     properties:
-      id: string
-      traceId: optional<string>
-      type: string
-      name: optional<string>
-      startTime: datetime
-      endTime: optional<datetime>
-      completionStartTime: optional<datetime>
-      model: optional<string>
-      modelParameters: optional<map<string, MapValue>>
-      input: optional<unknown>
-      version: optional<string>
-      metadata: optional<unknown>
-      output: optional<unknown>
-      usage: optional<Usage>
-      level: ObservationLevel
-      statusMessage: optional<string>
-      parentObservationId: optional<string>
-      promptId: optional<string>
+      id:
+        type: string
+        docs: The unique identifier of the observation
+      traceId:
+        type: optional<string>
+        docs: The trace ID associated with the observation
+      type:
+        type: string
+        docs: The type of the observation
+      name:
+        type: optional<string>
+        docs: The name of the observation
+      startTime:
+        type: datetime
+        docs: The start time of the observation
+      endTime:
+        type: optional<datetime>
+        docs: The end time of the observation.
+      completionStartTime:
+        type: optional<datetime>
+        docs: The completion start time of the observation
+      model:
+        type: optional<string>
+        docs: The model used for the observation
+      modelParameters:
+        type: optional<map<string, MapValue>>
+        docs: The parameters of the model used for the observation
+      input:
+        type: optional<unknown>
+        docs: The input data of the observation
+      version:
+        type: optional<string>
+        docs: The version of the observation
+      metadata:
+        type: optional<unknown>
+        docs: Additional metadata of the observation
+      output:
+        type: optional<unknown>
+        docs: The output data of the observation
+      usage:
+        type: optional<Usage>
+        docs: The usage data of the observation
+      level:
+        type: ObservationLevel
+        docs: The level of the observation
+      statusMessage:
+        type: optional<string>
+        docs: The status message of the observation
+      parentObservationId:
+        type: optional<string>
+        docs: The parent observation ID
+      promptId:
+        type: optional<string>
+        docs: The prompt ID associated with the observation
 
   ObservationsView:
     extends: Observation
     properties:
-      modelId: optional<string>
-      inputPrice: optional<double>
-      outputPrice: optional<double>
-      totalPrice: optional<double>
-      calculatedInputCost: optional<double>
-      calculatedOutputCost: optional<double>
-      calculatedTotalCost: optional<double>
-      latency: optional<double>
+      promptName:
+        type: optional<string>
+        docs: The name of the prompt associated with the observation
+      promptVersion:
+        type: optional<integer>
+        docs: The version of the prompt associated with the observation
+      modelId:
+        type: optional<string>
+        docs: The unique identifier of the model
+      inputPrice:
+        type: optional<double>
+        docs: The price of the input in USD
+      outputPrice:
+        type: optional<double>
+        docs: The price of the output in USD.
+      totalPrice:
+        type: optional<double>
+        docs: The total price in USD.
+      calculatedInputCost:
+        type: optional<double>
+        docs: The calculated cost of the input in USD
+      calculatedOutputCost:
+        type: optional<double>
+        docs: The calculated cost of the output in USD
+      calculatedTotalCost:
+        type: optional<double>
+        docs: The calculated total cost in USD
+      latency:
+        type: optional<double>
+        docs: The latency in seconds.
+      timeToFirstToken:
+        type: optional<double>
+        docs: The time to the first token in seconds
 
   Usage:
     docs: Standard interface for usage and cost
@@ -111,18 +199,91 @@ types:
       totalCost:
         docs: USD total cost, defaults to input+output
         type: optional<double>
-
-  Score:
+  ScoreConfig:
+    docs: Configuration for a score
+    properties:
+      id: string
+      name: string
+      createdAt: datetime
+      updatedAt: datetime
+      projectId: string
+      dataType: ScoreDataType
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
+      description: optional<string>
+  ConfigCategory:
+    properties:
+      value: double
+      label: string
+  BaseScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
       comment: optional<string>
-  DatasetCore:
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  NumericScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScore:
+    extends: BaseScore
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - double
+      - string
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
+  Dataset:
     properties:
       id: string
       name: string
@@ -131,21 +292,6 @@ types:
       projectId: string
       createdAt: datetime
       updatedAt: datetime
-  DatasetWithReferences: # GET datasets
-    extends: DatasetCore
-    properties:
-      items:
-        type: list<string>
-        docs: list of dataset item ids
-      runs:
-        type: list<string>
-        docs: list of dataset run names
-  Dataset: # GET datasets/[name]
-    docs: Dataset including all items
-    extends: DatasetCore
-    properties:
-      items: list<DatasetItem>
-      runs: list<string>
   DatasetItem:
     properties:
       id: string
@@ -171,15 +317,67 @@ types:
       updatedAt: datetime
   DatasetRun:
     properties:
-      id: string
-      name: string
-      description: optional<string>
-      metadata: optional<unknown>
-      datasetId: string
-      datasetName: string
-      createdAt: datetime
-      updatedAt: datetime
+      id:
+        type: string
+        docs: Unique identifier of the dataset run
+      name:
+        type: string
+        docs: Name of the dataset run
+      description:
+        type: optional<string>
+        docs: Description of the run
+      metadata:
+        type: optional<unknown>
+        docs: Metadata of the dataset run
+      datasetId:
+        type: string
+        docs: Id of the associated dataset
+      datasetName:
+        type: string
+        docs: Name of the associated dataset
+      createdAt:
+        type: datetime
+        docs: The date and time when the dataset run was created
+      updatedAt:
+        type: datetime
+        docs: The date and time when the dataset run was last updated
+  DatasetRunWithItems:
+    extends: DatasetRun
+    properties:
       datasetRunItems: list<DatasetRunItem>
+  Model:
+    docs: Model definition used for transforming usage into USD cost and/or tokenization.
+    properties:
+      id: string
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>
+      isLangfuseManaged:
+        type: boolean
 
   # Utilities
   ModelUsageUnit:
@@ -190,6 +388,7 @@ types:
       - MILLISECONDS
       - SECONDS
       - IMAGES
+      - REQUESTS
   ObservationLevel:
     enum:
       - DEBUG
@@ -212,6 +411,11 @@ types:
       - ANNOTATION
       - API
       - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - BOOLEAN
+      - CATEGORICAL
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
@@ -19,6 +20,23 @@ service:
         id:
           type: string
       response: commons.DatasetItem
+    list:
+      docs: Get dataset items
+      method: GET
+      path: /dataset-items
+      request:
+        name: GetDatasetItemsRequest
+        query-parameters:
+          datasetName: optional<string>
+          sourceTraceId: optional<string>
+          sourceObservationId: optional<string>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetItems
 
 types:
   CreateDatasetItemRequest:
@@ -31,7 +49,11 @@ types:
       sourceObservationId: optional<string>
       id:
         type: optional<string>
-        docs: Dataset items are upserted on their id
+        docs: Dataset items are upserted on their id. Id needs to be unique (project-level) and cannot be reused across datasets.
       status:
         type: optional<commons.DatasetStatus>
         docs: Defaults to ACTIVE for newly created items
+  PaginatedDatasetItems:
+    properties:
+      data: list<commons.DatasetItem>
+      meta: pagination.MetaResponse

fern/apis/server/definition/datasets.yml

@@ -9,42 +9,66 @@ service:
     list:
       method: GET
       docs: Get all datasets
-      path: /datasets
+      path: /v2/datasets
       request:
         name: GetDatasetsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
       response: PaginatedDatasets
     get:
       method: GET
-      docs: Get a dataset and its items
-      path: /datasets/{datasetName}
+      docs: Get a dataset
+      path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
       response: commons.Dataset
     create:
       method: POST
       docs: Create a dataset
-      path: /datasets
+      path: /v2/datasets
       request: CreateDatasetRequest
       response: commons.Dataset
-    getRuns:
+    getRun:
       method: GET
       docs: Get a dataset run and its items
       path: /datasets/{datasetName}/runs/{runName}
       path-parameters:
         datasetName: string
         runName: string
-      response: commons.DatasetRun
+      response: commons.DatasetRunWithItems
+    getRuns:
+      method: GET
+      docs: Get dataset runs
+      path: /datasets/{datasetName}/runs
+      path-parameters:
+        datasetName: string
+      request:
+        name: GetDatasetRunsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetRuns
 
 types:
   PaginatedDatasets:
     properties:
-      data: list<commons.DatasetWithReferences>
+      data: list<commons.Dataset>
       meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
       description: optional<string>
       metadata: optional<unknown>
+  PaginatedDatasetRuns:
+    properties:
+      data: list<commons.DatasetRun>
+      meta: pagination.MetaResponse

fern/apis/server/definition/ingestion.yml

@@ -7,7 +7,12 @@ service:
   base-path: /api/public
   endpoints:
     batch:
-      docs: Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
+      docs: |
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
+
+        Notes:
+        - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
+        - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
       path: /ingestion
       request:
@@ -180,9 +185,54 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/models.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      method: POST
+      docs: Create a model
+      path: /models
+      request: CreateModelRequest
+      response: commons.Model
+    list:
+      method: GET
+      docs: Get all models
+      path: /models
+      request:
+        name: GetModelsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedModels
+    get:
+      method: GET
+      docs: Get a model
+      path: /models/{id}
+      path-parameters:
+        id: string
+      response: commons.Model
+    delete:
+      method: DELETE
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      path: /models/{id}
+      path-parameters:
+        id: string
+
+types:
+  PaginatedModels:
+    properties:
+      data: list<commons.Model>
+      meta: pagination.MetaResponse
+  CreateModelRequest:
+    properties:
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: commons.ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total units. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>

fern/apis/server/definition/observations.yml

@@ -35,7 +35,13 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
+            docs: Retrieve only observations with a start_time or or after this datetime (ISO 8601).
+          toStartTime:
+            type: optional<datetime>
+            docs: Retrieve only observations with a start_time before this datetime (ISO 8601).
+          version:
+            type: optional<string>
+            docs: Optional filter to only include observations with a certain version.
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -35,13 +35,23 @@ service:
           name: optional<string>
           label: optional<string>
           tag: optional<string>
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+          fromUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated on or after a certain datetime (ISO 8601)
+          toUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated before a certain datetime (ISO 8601)
 
       response: PromptMetaListResponse
 
     create:
-      docs: Create a prompt
+      docs: Create a new version for the prompt with the given `name`
       method: POST
       path: /prompts
       request: CreatePromptRequest
@@ -59,6 +69,10 @@ types:
       versions: list<integer>
       labels: list<string>
       tags: list<string>
+      lastUpdatedAt: datetime
+      lastConfig:
+        type: unknown
+        docs: Config object of the most recent prompt version that matches the filters (if any are provided)
 
   CreatePromptRequest:
     union:

fern/apis/server/definition/score-configs.yml

@@ -0,0 +1,58 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
+    get:
+      docs: Get all score configs
+      method: GET
+      path: /score-configs
+      request:
+        name: GetScoreConfigsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
+      response: ScoreConfigs
+    get-by-id:
+      docs: Get a score config
+      method: GET
+      path: /score-configs/{configId}
+      path-parameters:
+        configId:
+          type: string
+          docs: The unique langfuse identifier of a score config
+      response: commons.ScoreConfig
+types:
+  ScoreConfigs:
+    properties:
+      data: list<commons.ScoreConfig>
+      meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -11,7 +11,7 @@ service:
       method: POST
       path: /scores
       request: CreateScoreRequest
-      response: commons.Score
+      response: CreateScoreResponse
     get:
       docs: Get a list of scores
       method: GET
@@ -25,11 +25,18 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this datetime (ISO 8601).
+            docs: Optional filter to only include scores created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include scores created before a certain datetime (ISO 8601)
           source:
             type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
@@ -42,6 +49,12 @@ service:
           scoreIds:
             type: optional<string>
             docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -66,9 +79,59 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  CreateScoreResponse:
+    properties:
+      id:
+        type: string
+        docs: The id of the created object in Langfuse
   Scores:
     properties:
       data: list<commons.Score>

fern/apis/server/definition/sessions.yml

@@ -6,6 +6,26 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
+    list:
+      docs: Get sessions
+      method: GET
+      path: /sessions
+      request:
+        name: GetSessionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include sessions created before a certain datetime (ISO 8601)
+      response: PaginatedSessions
     get:
       docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
@@ -15,3 +35,8 @@ service:
           type: string
           docs: The unique id of a session
       response: commons.SessionWithTraces
+types:
+  PaginatedSessions:
+    properties:
+      data: list<commons.Session>
+      meta: pagination.MetaResponse

fern/apis/server/definition/trace.yml

@@ -16,7 +16,7 @@ service:
           docs: The unique langfuse identifier of a trace
       response: commons.TraceWithFullDetails
     list:
-      docs: Get list of traces.
+      docs: Get list of traces
       method: GET
       path: /traces
       request:
@@ -33,7 +33,10 @@ service:
           sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this datetime (ISO 8601).
+            docs: Optional filter to only include traces with a trace.timestamp on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces with a trace.timestamp before a certain datetime (ISO 8601)
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"
@@ -41,6 +44,12 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Only traces that include all of these tags will be returned.
+          version:
+            type: optional<string>
+            docs: Optional filter to only include traces with a certain version.
+          release:
+            type: optional<string>
+            docs: Optional filter to only include traces with a certain release.
       response: Traces
 
 types:

fern/apis/server/definition/utils/pagination.yml

@@ -1,4 +1,13 @@
 types:
+  # to be used as copy/paste template for query strings
+  # PaginationRequest:
+  #   properties:
+  #     page:
+  #       type: optional<integer>
+  #       docs: page number, starts at 1
+  #     limit:
+  #       type: optional<integer>
+  #       docs: limit of items per page
   MetaResponse:
     properties:
       page:

package.json

@@ -1,12 +1,11 @@
 {
   "name": "langfuse",
-  "version": "2.53.0",
+  "version": "2.72.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">=20",
-    "pnpm": "<9"
+    "node": "20"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -17,8 +16,9 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
@@ -35,7 +35,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.2.5",
     "release-it": "^17.3.0",
-    "turbo": "^1.13.3"
+    "turbo": "^1.13.4"
   },
   "release-it": {
     "git": {
@@ -79,5 +79,6 @@
         "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
       }
     }
-  }
+  },
+  "packageManager": "pnpm@9.5.0"
 }

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.3",
+    "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces;

packages/shared/clickhouse/migrations/0001_traces.up.sql

@@ -0,0 +1,26 @@
+CREATE TABLE traces (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(String, String) CODEC(ZSTD(1)),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(1)),
+    `output` Nullable(String) CODEC(ZSTD(1)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/migrations/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations;

packages/shared/clickhouse/migrations/0002_observations.up.sql

@@ -0,0 +1,50 @@
+CREATE TABLE observations (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_input_usage_units` Nullable(Decimal64(12)),
+    `provided_output_usage_units` Nullable(Decimal64(12)),
+    `provided_total_usage_units` Nullable(Decimal64(12)),
+    `input_usage_units` Nullable(Decimal64(12)),
+    `output_usage_units` Nullable(Decimal64(12)),
+    `total_usage_units` Nullable(Decimal64(12)),
+    `unit` Nullable(String),
+    `provided_input_cost` Nullable(Decimal64(12)),
+    `provided_output_cost` Nullable(Decimal64(12)),
+    `provided_total_cost` Nullable(Decimal64(12)),
+    `input_cost` Nullable(Decimal64(12)),
+    `output_cost` Nullable(Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(start_time)
+ORDER BY (
+        project_id,
+        `type`,
+        trace_id,
+        toUnixTimestamp(start_time),
+        id
+    );

packages/shared/clickhouse/migrations/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores;

packages/shared/clickhouse/migrations/0003_scores.up.sql

@@ -0,0 +1,25 @@
+CREATE TABLE scores (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        trace_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/scripts/down.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the down command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down

packages/shared/clickhouse/scripts/drop.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the drop command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/up.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the up command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up

packages/shared/package.json

@@ -6,7 +6,7 @@
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "engines": {
-    "node": ">=20"
+    "node": "20"
   },
   "exports": {
     ".": {
@@ -17,9 +17,9 @@
       "import": "./dist/src/db.js",
       "require": "./dist/src/db.js"
     },
-    "./src/server/auth": {
-      "import": "./dist/src/server/auth.js",
-      "require": "./dist/src/server/auth.js"
+    "./src/server": {
+      "import": "./dist/src/server/index.js",
+      "require": "./dist/src/server/index.js"
     },
     "./encryption": {
       "import": "./dist/src/encryption/index.js",
@@ -38,31 +38,52 @@
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
-    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load"
+    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
+    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
+    "ch:up": "bash clickhouse/scripts/up.sh",
+    "ch:down": "bash clickhouse/scripts/down.sh",
+    "ch:drop": "bash clickhouse/scripts/drop.sh",
+    "ch:reset": "pnpm run ch:down && pnpm run ch:up"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@aws-sdk/client-s3": "^3.627.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@clickhouse/client": "^1.4.0",
     "@langchain/anthropic": "^0.1.21",
     "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.33",
-    "@prisma/client": "^5.13.0",
+    "@opentelemetry/api": ">=1.0.0 <1.9.0",
+    "@prisma/client": "^5.17.0",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
+    "bullmq": "^5.4.2",
+    "decimal.js": "^10.4.3",
+    "exponential-backoff": "^3.1.1",
+    "dd-trace": "^5.19.0",
+    "ioredis": "^5.4.1",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.36",
+    "langchain": "^0.2.6",
     "lodash": "^4.17.21",
     "next-auth": "^4.24.7",
+    "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
-    "zod": "^3.22.4",
+    "uuid": "^9.0.1",
+    "zod": "^3.23.8",
     "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
-    "@repo/eslint-config": "*",
-    "@repo/typescript-config": "*",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
+    "@types/nodemailer": "^6.4.15",
     "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
@@ -73,12 +94,16 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.1.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.13.0",
+    "prisma": "^5.17.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
     "vitest": "^1.5.3"
+  },
+  "peerDependencies": {
+    "@types/react": "^18.2.79",
+    "react": "^18.0.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,14 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const ProjectRole = {
+export const Role = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
-    VIEWER: "VIEWER"
+    VIEWER: "VIEWER",
+    NONE: "NONE"
 } as const;
-export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
+export type Role = (typeof Role)[keyof typeof Role];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -41,6 +42,13 @@ export const DatasetStatus = {
     ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const CommentObjectType = {
+    TRACE: "TRACE",
+    OBSERVATION: "OBSERVATION",
+    SESSION: "SESSION",
+    PROMPT: "PROMPT"
+} as const;
+export type CommentObjectType = (typeof CommentObjectType)[keyof typeof CommentObjectType];
 export const JobType = {
     EVAL: "EVAL"
 } as const;
@@ -90,14 +98,41 @@ export type AuditLog = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     user_id: string;
-    project_id: string;
-    user_project_role: ProjectRole;
+    org_id: string;
+    user_org_role: string;
+    project_id: string | null;
+    user_project_role: string | null;
     resource_type: string;
     resource_id: string;
     action: string;
     before: string | null;
     after: string | null;
 };
+export type BatchExport = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    user_id: string;
+    finished_at: Timestamp | null;
+    expires_at: Timestamp | null;
+    name: string;
+    status: string;
+    query: unknown;
+    format: string;
+    url: string | null;
+    log: string | null;
+};
+export type Comment = {
+    id: string;
+    project_id: string;
+    object_type: CommentObjectType;
+    object_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    content: string;
+    author_user_id: string | null;
+};
 export type CronJobs = {
     name: string;
     last_run: Timestamp | null;
@@ -106,15 +141,16 @@ export type CronJobs = {
 };
 export type Dataset = {
     id: string;
+    project_id: string;
     name: string;
     description: string | null;
     metadata: unknown | null;
-    project_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
 export type DatasetItem = {
     id: string;
+    project_id: string;
     status: Generated<DatasetStatus>;
     input: unknown | null;
     expected_output: unknown | null;
@@ -127,6 +163,7 @@ export type DatasetItem = {
 };
 export type DatasetRunItems = {
     id: string;
+    project_id: string;
     dataset_run_id: string;
     dataset_item_id: string;
     trace_id: string;
@@ -136,6 +173,7 @@ export type DatasetRunItems = {
 };
 export type DatasetRuns = {
     id: string;
+    project_id: string;
     name: string;
     description: string | null;
     metadata: unknown | null;
@@ -204,16 +242,18 @@ export type LlmApiKeys = {
     display_secret_key: string;
     secret_key: string;
     base_url: string | null;
-    custom_models: string[];
+    custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
     project_id: string;
 };
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: ProjectRole;
-    project_id: string;
-    sender_id: string | null;
+    org_id: string;
+    org_role: Role;
+    project_id: string | null;
+    project_role: Role | null;
+    invited_by_user_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -249,6 +289,7 @@ export type Observation = {
     updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
+    internal_model_id: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
     output: unknown | null;
@@ -259,6 +300,9 @@ export type Observation = {
     input_cost: string | null;
     output_cost: string | null;
     total_cost: string | null;
+    calculated_input_cost: string | null;
+    calculated_output_cost: string | null;
+    calculated_total_cost: string | null;
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
@@ -267,7 +311,7 @@ export type ObservationView = {
     trace_id: string | null;
     project_id: string;
     type: ObservationType;
-    start_time: Generated<Timestamp>;
+    start_time: Timestamp;
     end_time: Timestamp | null;
     name: string | null;
     metadata: unknown | null;
@@ -275,7 +319,8 @@ export type ObservationView = {
     level: Generated<ObservationLevel>;
     status_message: string | null;
     version: string | null;
-    created_at: Generated<Timestamp>;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     model: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
@@ -286,6 +331,8 @@ export type ObservationView = {
     unit: string | null;
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
+    prompt_name: string | null;
+    prompt_version: number | null;
     model_id: string | null;
     input_price: string | null;
     output_price: string | null;
@@ -296,6 +343,21 @@ export type ObservationView = {
     latency: number | null;
     time_to_first_token: number | null;
 };
+export type Organization = {
+    id: string;
+    name: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    cloud_config: unknown | null;
+};
+export type OrganizationMembership = {
+    id: string;
+    org_id: string;
+    user_id: string;
+    role: Role;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
     project_id: string;
     encrypted_posthog_api_key: string;
@@ -306,15 +368,16 @@ export type PosthogIntegration = {
 };
 export type Project = {
     id: string;
+    org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     name: string;
-    cloud_config: unknown | null;
 };
 export type ProjectMembership = {
+    org_membership_id: string;
     project_id: string;
     user_id: string;
-    role: ProjectRole;
+    role: Role;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -338,7 +401,7 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
     author_user_id: string | null;
     comment: string | null;
@@ -419,6 +482,8 @@ export type TraceView = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     duration: number | null;
 };
 export type User = {
@@ -442,6 +507,8 @@ export type DB = {
     Account: Account;
     api_keys: ApiKey;
     audit_logs: AuditLog;
+    batch_exports: BatchExport;
+    comments: Comment;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;
@@ -456,6 +523,8 @@ export type DB = {
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
+    organization_memberships: OrganizationMembership;
+    organizations: Organization;
     posthog_integrations: PosthogIntegration;
     project_memberships: ProjectMembership;
     projects: Project;

packages/shared/prisma/migrations/20240530212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -1,7 +0,0 @@
--- Google Vertex uses @ to separate model name and version
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -1,5 +0,0 @@
--- does not include pricing yet, will be added as soon as it is calculated at ingestion time
-
-INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
-
-INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240607090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,8 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+-- remove model if added by faulty migration, context: https://github.com/langfuse/langfuse/issues/2266
+DELETE FROM models WHERE id in ('clx30djsn0000w9mzebiv41we', 'clx30hkrx0000w9mz7lqi0ial');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240607212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240611113517_backfill_manual_score_configs/migration.sql

@@ -10,7 +10,7 @@ INSERT INTO score_configs (id,
 		max_value,
 		description)
 SELECT
-	gen_random_uuid () AS id,
+	md5(random()::text || clock_timestamp()::text || s.project_id::text)::uuid AS id,
 	s.project_id,
 	'manual-score',
 	'NUMERIC',
@@ -24,6 +24,7 @@ FROM ( SELECT DISTINCT
 		scores
 	WHERE
 		name = 'manual-score'
+		AND config_id IS NULL
 		AND source = 'ANNOTATION') s
 WHERE
 	NOT EXISTS (
@@ -47,4 +48,4 @@ FROM
 WHERE
 	scores.project_id = pc.project_id
 	AND scores.name = 'manual-score';
-COMMIT;
+COMMIT;

packages/shared/prisma/migrations/20240612101858_add_index_observations_project_id_prompt_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_prompt_id_idx" ON "observations"("project_id", "prompt_id");

packages/shared/prisma/migrations/20240617094803_observations_remove_prompt_fk_constraint/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "observations" DROP CONSTRAINT "observations_prompt_id_fkey";

packages/shared/prisma/migrations/20240618134129_add_batch_exports_table/migration.sql

@@ -0,0 +1,27 @@
+-- CreateTable
+CREATE TABLE "batch_exports" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "finished_at" TIMESTAMP(3),
+    "expires_at" TIMESTAMP(3),
+    "name" TEXT NOT NULL,
+    "status" TEXT NOT NULL,
+    "query" JSONB NOT NULL,
+    "format" TEXT NOT NULL,
+    "url" TEXT,
+    "log" TEXT,
+
+    CONSTRAINT "batch_exports_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "batch_exports_project_id_user_id_idx" ON "batch_exports"("project_id", "user_id");
+
+-- CreateIndex
+CREATE INDEX "batch_exports_status_idx" ON "batch_exports"("status");
+
+-- AddForeignKey
+ALTER TABLE "batch_exports" ADD CONSTRAINT "batch_exports_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240618164950_drop_observations_parent_observation_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_parent_observation_id_idx";

packages/shared/prisma/migrations/20240618164951_drop_observations_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_updated_at_idx";

packages/shared/prisma/migrations/20240618164952_drop_scores_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "scores_updated_at_idx";

packages/shared/prisma/migrations/20240618164953_drop_traces_external_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_external_id_idx";

packages/shared/prisma/migrations/20240618164954_drop_traces_release_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_release_idx";

packages/shared/prisma/migrations/20240618164955_drop_traces_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_updated_at_idx";

packages/shared/prisma/migrations/20240618164956_create_traces_project_id_timestamp_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "traces_project_id_timestamp_idx" ON "traces"("project_id", "timestamp");

packages/shared/prisma/migrations/20240624133412_models_add_anthropic_3_5_sonnet/migration.sql

@@ -0,0 +1,15 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id
+)
+VALUES
+  -- add 3.5 sonnet model
+  ('clxt0n0m60000pumz1j5b7zsf', NULL, 'claude-3-5-sonnet-20240620', '(?i)^(claude-3-5-sonnet(-|@)?20240620)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude')

packages/shared/prisma/migrations/20240625103957_observations_add_calculated_cost_columns/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "calculated_input_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_output_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_total_cost" DECIMAL(65,30),
+ADD COLUMN     "internal_model_id" TEXT;

packages/shared/prisma/migrations/20240625103958_fix_model_match_gpt4_vision/migration.sql

@@ -0,0 +1,17 @@
+/*
+Previously, the pattern did not allow for model specifications like "1106" or any other 4-digit block. 
+This led to missing models on the generation-update call where the exact model name that was used was provided (including the 4-digit block).
+The new pattern allows for:
+
+- "gpt-4-vision-preview" as set on generation-create
+- "gpt-4-1106-vision-preview" as set on generation-update
+
+*/
+
+UPDATE
+	"models"
+SET
+	"match_pattern" = '(?i)^(gpt-4(-\d{4})?-vision-preview)$'
+WHERE
+	"id" = 'clrkvx5gp000108juaogs54ea'
+	AND "model_name" = 'gpt-4-turbo-vision';

packages/shared/prisma/migrations/20240703214747_models_anthropic_aws_bedrock/migration.sql

@@ -0,0 +1,9 @@
+-- Add AWS Bedrock model names for Anthropic models
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku-20240307|anthropic\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet-20240229|anthropic\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus-20240229|anthropic\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-5-sonnet-20240620|anthropic\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$' WHERE "id" = 'clxt0n0m60000pumz1j5b7zsf';

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240704103901_scores_make_value_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "value" DROP NOT NULL;

packages/shared/prisma/migrations/20240705152639_traces_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,20 @@
+-- Drop and create to be able to change columns, otherwise new t.* cols cannot be added
+
+DROP VIEW IF EXISTS traces_view;
+CREATE VIEW traces_view AS
+WITH observations_metrics AS (
+    SELECT
+        trace_id,
+        project_id,
+        EXTRACT(EPOCH FROM COALESCE(MAX(o.end_time), MAX(o.start_time))) - EXTRACT(EPOCH FROM MIN(o.start_time))::double precision AS duration
+    FROM
+        observations o
+    GROUP BY
+        project_id, trace_id
+)
+SELECT
+    t.*,
+    o.duration
+FROM
+    traces t
+    LEFT JOIN observations_metrics o ON t.id = o.trace_id and t.project_id = o.project_id

packages/shared/prisma/migrations/20240705154048_observation_view_add_created_at_updated_at/migration.sql

@@ -0,0 +1,89 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240704103900_observations_view_read_from_calculated and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240710114043_score_configs_drop_empty_categories_array_for_numeric_scores/migration.sql

@@ -0,0 +1,6 @@
+-- Migration script to update score_configs entries
+-- Set categories to NULL where data_type is 'NUMERIC' and categories is an empty array
+
+UPDATE score_configs
+SET categories = NULL
+WHERE data_type = 'NUMERIC' AND categories IS NOT NULL;

packages/shared/prisma/migrations/20240710114044_add_pricing_gpt4o_mini/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-mini
+  ('clyrjp56f0000t0mzapoocd7u', NULL, 'gpt-4o-mini', '(?i)^(gpt-4o-mini)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-mini-2024-07-18
+  ('clyrjpbe20000t0mzcbwc42rg', NULL, 'gpt-4o-mini-2024-07-18', '(?i)^(gpt-4o-mini-2024-07-18)$', NULL, 0.00000015, 0.0000006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/migrations/20240718004923_datasets_tables_add_projectid_composite_key/migration.sql

@@ -0,0 +1,68 @@
+-- Add project_id to datasets, dataset_items, dataset_runs, dataset_run_items
+ALTER TABLE "dataset_items" 
+ADD COLUMN "project_id" TEXT;
+
+ALTER TABLE "dataset_run_items" 
+ADD COLUMN "project_id" TEXT;
+
+ALTER TABLE "dataset_runs" 
+ADD COLUMN "project_id" TEXT;
+
+-- Backfill project_id for dataset_items
+UPDATE dataset_items
+SET project_id = datasets.project_id
+FROM datasets
+WHERE dataset_items.dataset_id = datasets.id;
+
+-- Backfill project_id for dataset_runs
+UPDATE dataset_runs
+SET project_id = datasets.project_id
+FROM datasets
+WHERE dataset_runs.dataset_id = datasets.id;
+
+-- Backfill project_id for dataset_run_items
+UPDATE dataset_run_items
+SET project_id = dataset_runs.project_id
+FROM dataset_runs
+WHERE dataset_run_items.dataset_run_id = dataset_runs.id;
+
+-- Drop the old foreign keys
+ALTER TABLE "dataset_run_items" 
+DROP CONSTRAINT "dataset_run_items_dataset_item_id_fkey",
+DROP CONSTRAINT "dataset_run_items_dataset_run_id_fkey";
+
+ALTER TABLE "dataset_items" 
+DROP CONSTRAINT "dataset_items_dataset_id_fkey";
+
+ALTER TABLE "dataset_runs" 
+DROP CONSTRAINT "dataset_runs_dataset_id_fkey";
+
+-- Now alter the columns to NOT NULL and update primary keys
+ALTER TABLE "datasets" 
+DROP CONSTRAINT "datasets_pkey",
+ADD CONSTRAINT "datasets_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_items" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_items_pkey",
+ADD CONSTRAINT "dataset_items_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_runs" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_runs_pkey",
+ADD CONSTRAINT "dataset_runs_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_run_items" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_run_items_pkey",
+ADD CONSTRAINT "dataset_run_items_pkey" PRIMARY KEY ("id", "project_id");
+
+
+-- Add new foreign keys
+ALTER TABLE "dataset_items" ADD CONSTRAINT "dataset_items_dataset_id_project_id_fkey" FOREIGN KEY ("dataset_id", "project_id") REFERENCES "datasets"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_runs" ADD CONSTRAINT "dataset_runs_dataset_id_project_id_fkey" FOREIGN KEY ("dataset_id", "project_id") REFERENCES "datasets"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_dataset_run_id_project_id_fkey" FOREIGN KEY ("dataset_run_id", "project_id") REFERENCES "dataset_runs"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_dataset_item_id_project_id_fkey" FOREIGN KEY ("dataset_item_id", "project_id") REFERENCES "dataset_items"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240718011733_dataset_runs_add_unique_dataset_id_project_id_name copy/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "dataset_runs_dataset_id_project_id_name_key" ON "dataset_runs"("dataset_id", "project_id", "name");

packages/shared/prisma/migrations/20240718011734_dataset_runs_drop_unique_dataset_id_name/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY "dataset_runs_dataset_id_name_key";

packages/shared/prisma/migrations/20240718011735_observation_view_add_prompt_name_and_version/migration.sql

@@ -0,0 +1,100 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240705154048_observation_view_add_created_at_updated_at and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    p.name as prompt_name,         -- added in this change
+    p.version as prompt_version,   -- added in this change
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+LEFT JOIN LATERAL (
+    SELECT
+        prompts.*
+    FROM
+        prompts
+    WHERE prompts.id = o.prompt_id
+    AND prompts.project_id = o.project_id
+    LIMIT 1
+) p ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240807111358_models_add_openai_gpt_4o_2024_08_06/migration.sql

@@ -0,0 +1,18 @@
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-2024-08-06
+  ('clzjr85f70000ymmzg7hqffra', NULL, 'gpt-4o-2024-08-06', '(?i)^(gpt-4o-2024-08-06)$', NULL, 0.0000025, 0.000010, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/migrations/20240807111359_add_organizations_main_migration/migration.sql

@@ -0,0 +1,164 @@
+-- Table ORGANIZATIONS
+-- Create empty table ORGANIZATIONS
+CREATE TABLE "organizations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "cloud_config" JSONB,
+
+    CONSTRAINT "organizations_pkey" PRIMARY KEY ("id")
+);
+-- Backfill table ORGANIZATIONS: create a new organization for each project, move cloudConfig, and set the org_id on the project to project_id prefixed with 'o'
+INSERT INTO "organizations" ("id", "name", "cloud_config", "created_at")
+SELECT
+  CONCAT('o', "id") as "id", -- This mapping is used in other migration steps as well, keep it consistent
+  "name",
+  "cloud_config",
+  "created_at"
+FROM "projects";
+-- Drop column cloud_config from projects as it's now on organization level
+ALTER TABLE "projects" DROP COLUMN "cloud_config";
+
+
+-- PROJECT.ORG_ID
+-- Add org_id to projects
+ALTER TABLE "projects" ADD COLUMN "org_id" TEXT;
+-- Backfill: org_id on projects, set it to project_id prefixed with 'o'
+UPDATE "projects"
+SET "org_id" = CONCAT('o', "id");
+-- Set not null after backfill
+ALTER TABLE "projects" ALTER COLUMN "org_id" SET NOT NULL;
+-- AddForeignKey
+ALTER TABLE "projects" ADD CONSTRAINT "projects_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "projects_org_id_idx" ON "projects"("org_id");
+
+
+-- ORGANIZATION MEMBERSHIPS
+-- Create UserRole ENUM
+CREATE TYPE "Role" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER', 'NONE');
+-- Create empty table
+CREATE TABLE "organization_memberships" (
+    "id" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "role" "Role" NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_memberships_pkey" PRIMARY KEY ("id")
+);
+-- Add indexes
+CREATE INDEX "organization_memberships_user_id_idx" ON "organization_memberships"("user_id");
+CREATE INDEX "organization_memberships_org_id_idx" ON "organization_memberships"("org_id");
+CREATE UNIQUE INDEX "organization_memberships_org_id_user_id_key" ON "organization_memberships"("org_id", "user_id");
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+
+
+-- Migrate project memberships to organization memberships
+INSERT INTO "organization_memberships" ("id", "org_id", "user_id", "role", "created_at", "updated_at")
+SELECT
+  md5(random()::text || clock_timestamp()::text || project_id::text || user_id::text)::uuid AS "id",
+  CONCAT('o', "project_id") as "org_id",
+  "user_id",
+  "role"::text::"Role" as "role",
+  "created_at",
+  "updated_at"
+FROM "project_memberships";
+
+
+
+-- Delete all project memberships after migration to organization memberships
+DELETE FROM "project_memberships";
+
+-- Add org_membership_id to project_memberships, ok to be not null as it's a new column on a now empty table
+ALTER TABLE "project_memberships" ADD COLUMN     "org_membership_id" TEXT NOT NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "project_memberships" DROP COLUMN "role",
+ADD COLUMN     "role" "Role" NOT NULL;
+-- AddForeignKey
+ALTER TABLE "project_memberships" ADD CONSTRAINT "project_memberships_org_membership_id_fkey" FOREIGN KEY ("org_membership_id") REFERENCES "organization_memberships"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "project_memberships_project_id_idx" ON "project_memberships"("project_id");
+-- CreateIndex
+CREATE INDEX "project_memberships_org_membership_id_idx" ON "project_memberships"("org_membership_id");
+
+
+
+-- AUDIT LOGS
+-- DropForeignKey, just index on these, should remain after project/org/user deletions
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_user_id_fkey";
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_project_id_fkey";
+-- Drop not null on project level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" DROP NOT NULL;
+-- Add org-level cols
+ALTER TABLE "audit_logs" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "audit_logs" ADD COLUMN "user_org_role" TEXT;
+-- Backfill org_id
+UPDATE "audit_logs"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "audit_logs"."project_id" = "projects"."id";
+-- Backfill user_org_role with value from user_project_role
+UPDATE "audit_logs"
+SET "user_org_role" = "user_project_role"::text::"Role";
+-- Drop and recreate user_project_role column as text column going forward, empty for historical data as it's all on org level now
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role" TEXT; -- nullable
+-- Add not null on org level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "org_id" SET NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "audit_logs_user_id_idx" ON "audit_logs"("user_id");
+CREATE INDEX "audit_logs_org_id_idx" ON "audit_logs"("org_id");
+
+
+-- MEMBERSHIP INVITATIONS
+-- Rename sender_id to invited_by_user_id
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_sender_id_fkey";
+ALTER TABLE "membership_invitations" RENAME COLUMN "sender_id" TO "invited_by_user_id";
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_invited_by_user_id_fkey" FOREIGN KEY ("invited_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+-- DropForeignKey
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_project_id_fkey";
+-- AlterTable
+ALTER TABLE "membership_invitations" RENAME COLUMN "role" TO "project_role";
+ALTER TABLE "membership_invitations" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "membership_invitations" ADD COLUMN "org_role" "Role";
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_role" DROP NOT NULL;
+-- Backfill org id
+UPDATE "membership_invitations"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "membership_invitations"."project_id" = "projects"."id";
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- Backfill org role with value from project role
+UPDATE "membership_invitations"
+SET "org_role" = "project_role"::text::"Role";
+-- Set project-level cols to null, as it's now org level for all existing invitations and role enum will change below
+UPDATE "membership_invitations"
+SET "project_role" = NULL, "project_id" = NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "membership_invitations" DROP COLUMN "project_role",
+ADD COLUMN     "project_role" "Role";
+
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- Add not null on org level cols after backfill
+ALTER TABLE "membership_invitations"
+ALTER COLUMN "org_id" SET NOT NULL,
+ALTER COLUMN "org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "membership_invitations_org_id_idx" ON "membership_invitations"("org_id");
+
+
+-- Drop ProjectRole enum as it is replaced by Role
+DROP TYPE "ProjectRole";

packages/shared/prisma/migrations/20240814223824_model_fix_text_embedding_3_large/migration.sql

@@ -0,0 +1,7 @@
+
+-- Fix model name
+-- https://github.com/langfuse/langfuse/issues/2688
+
+UPDATE models
+SET model_name = 'text-embedding-3-large'
+WHERE id = 'clruwn76700020al7gp8e4g4l'

packages/shared/prisma/migrations/20240814233029_dataset_items_drop_fkey_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_trace_id_fkey";

packages/shared/prisma/migrations/20240815171916_add_comments/migration.sql

@@ -0,0 +1,22 @@
+-- CreateEnum
+CREATE TYPE "CommentObjectType" AS ENUM ('TRACE', 'OBSERVATION', 'SESSION', 'PROMPT');
+
+-- CreateTable
+CREATE TABLE "comments" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object_type" "CommentObjectType" NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "content" TEXT NOT NULL,
+    "author_user_id" TEXT,
+
+    CONSTRAINT "comments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "comments_project_id_object_type_object_id_idx" ON "comments"("project_id", "object_type", "object_id");
+
+-- AddForeignKey
+ALTER TABLE "comments" ADD CONSTRAINT "comments_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views", "relationJoins"]
+    previewFeatures = ["tracing", "views", "relationJoins", "metrics"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id                 String                 @id @default(cuid())
-    name               String?
-    email              String?                @unique
-    emailVerified      DateTime?              @map("email_verified")
-    password           String?
-    image              String?
-    admin              Boolean                @default(false)
-    accounts           Account[]
-    sessions           Session[]
-    projectMemberships ProjectMembership[]
-    invitations        MembershipInvitation[]
-    createdAt          DateTime               @default(now()) @map("created_at")
-    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags       String[]               @default([]) @map("feature_flags")
-    AuditLog           AuditLog[]
+    id                      String                   @id @default(cuid())
+    name                    String?
+    email                   String?                  @unique
+    emailVerified           DateTime?                @map("email_verified")
+    password                String?
+    image                   String?
+    admin                   Boolean                  @default(false)
+    accounts                Account[]
+    sessions                Session[]
+    organizationMemberships OrganizationMembership[]
+    projectMemberships      ProjectMembership[]
+    invitations             MembershipInvitation[]
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    featureFlags            String[]                 @default([]) @map("feature_flags")
 
     @@map("users")
 }
@@ -91,13 +91,27 @@ model VerificationToken {
     @@map("verification_tokens")
 }
 
+model Organization {
+    id                      String                   @id @default(cuid())
+    name                    String
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+    organizationMemberships OrganizationMembership[]
+    projects                Project[]
+    MembershipInvitation    MembershipInvitation[]
+
+    @@map("organizations")
+}
+
 model Project {
     id                 String                 @id @default(cuid())
+    orgId              String                 @map("org_id")
     createdAt          DateTime               @default(now()) @map("created_at")
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
-    cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
     projectMembers     ProjectMembership[]
+    organization       Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -107,7 +121,6 @@ model Project {
     sessions           TraceSession[]
     Prompt             Prompt[]
     Model              Model[]
-    AuditLog           AuditLog[]
     EvalTemplate       EvalTemplate[]
     JobConfiguration   JobConfiguration[]
     JobExecution       JobExecution[]
@@ -115,7 +128,10 @@ model Project {
     PosthogIntegration PosthogIntegration[]
     Score              Score[]
     scoreConfig        ScoreConfig[]
+    BatchExport        BatchExport[]
+    comment            Comment[]
 
+    @@index([orgId])
     @@map("projects")
 }
 
@@ -149,7 +165,7 @@ model LlmApiKeys {
     displaySecretKey  String   @map("display_secret_key")
     secretKey         String   @map("secret_key")
     baseURL           String?  @map("base_url")
-    customModels      String[] @map("custom_models")
+    customModels      String[] @default([]) @map("custom_models")
     withDefaultModels Boolean  @default(true) @map("with_default_models")
 
     projectId String  @map("project_id")
@@ -160,41 +176,68 @@ model LlmApiKeys {
     @@map("llm_api_keys")
 }
 
+model OrganizationMembership {
+    id                 String              @id @default(cuid())
+    orgId              String              @map("org_id")
+    organization       Organization        @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    userId             String              @map("user_id")
+    user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role               Role                @map("role")
+    createdAt          DateTime            @default(now()) @map("created_at")
+    updatedAt          DateTime            @default(now()) @updatedAt @map("updated_at")
+    ProjectMemberships ProjectMembership[]
+
+    @@unique([orgId, userId])
+    @@index([userId])
+    @@index([orgId])
+    @@map("organization_memberships")
+}
+
+// Set a project-specific role for a user in an organization
 model ProjectMembership {
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String      @map("user_id")
-    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      ProjectRole
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    orgMembershipId        String                 @map("org_membership_id")
+    organizationMembership OrganizationMembership @relation(fields: [orgMembershipId], references: [id], onDelete: Cascade)
+    projectId              String                 @map("project_id")
+    project                Project                @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId                 String                 @map("user_id")
+    user                   User                   @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role                   Role
+    createdAt              DateTime               @default(now()) @map("created_at")
+    updatedAt              DateTime               @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
+    @@index([projectId])
+    @@index([orgMembershipId])
     @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String      @id @unique @default(cuid())
-    email     String
-    role      ProjectRole
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?     @map("sender_id")
-    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    id              String       @id @unique @default(cuid())
+    email           String
+    orgId           String       @map("org_id")
+    organization    Organization @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    orgRole         Role         @map("org_role")
+    projectId       String?      @map("project_id")
+    project         Project?     @relation(fields: [projectId], references: [id], onDelete: SetNull)
+    projectRole     Role?        @map("project_role")
+    invitedByUserId String?      @map("invited_by_user_id")
+    invitedByUser   User?        @relation(fields: [invitedByUserId], references: [id], onDelete: SetNull)
+    createdAt       DateTime     @default(now()) @map("created_at")
+    updatedAt       DateTime     @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
+    @@index([orgId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum ProjectRole {
+enum Role {
     OWNER
     ADMIN
     MEMBER
     VIEWER
+    NONE
 }
 
 model TraceSession {
@@ -237,18 +280,15 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetItem  DatasetItem[]
     JobExecution JobExecution[]
 
     @@index([projectId])
+    @@index([projectId, timestamp])
     @@index([sessionId])
     @@index([name])
     @@index([userId])
     @@index([id, userId])
-    @@index([externalId])
     @@index(timestamp)
-    @@index(release)
-    @@index(updatedAt)
     @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
@@ -273,6 +313,8 @@ view TraceView {
     input      Json?
     output     Json?
     sessionId  String?  @map("session_id")
+    createdAt  DateTime @map("created_at")
+    updatedAt  DateTime @map("updated_at")
 
     // calculated fields
     duration Float? @map("duration") // can be null if no observations in trace
@@ -300,24 +342,32 @@ model Observation {
     updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
-    model               String?
-    internalModel       String?       @map("internal_model")
-    modelParameters     Json?
-    input               Json?
-    output              Json?
-    promptTokens        Int           @default(0) @map("prompt_tokens")
-    completionTokens    Int           @default(0) @map("completion_tokens")
-    totalTokens         Int           @default(0) @map("total_tokens")
-    unit                String?
-    inputCost           Decimal?      @map("input_cost")
-    outputCost          Decimal?      @map("output_cost")
-    totalCost           Decimal?      @map("total_cost")
-    completionStartTime DateTime?     @map("completion_start_time")
-    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    derivedDatasetItems DatasetItem[]
+    model           String? // user-provided model attribute
+    internalModel   String? @map("internal_model") // matched model.name that is matched at ingestion time, to be deprecated
+    internalModelId String? @map("internal_model_id") // matched model.id that is matched at ingestion time
 
-    promptId String? @map("prompt_id")
-    prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
+    modelParameters  Json?
+    input            Json?
+    output           Json?
+    promptTokens     Int     @default(0) @map("prompt_tokens")
+    completionTokens Int     @default(0) @map("completion_tokens")
+    totalTokens      Int     @default(0) @map("total_tokens")
+    unit             String?
+
+    // User provided cost at ingestion
+    inputCost  Decimal? @map("input_cost")
+    outputCost Decimal? @map("output_cost")
+    totalCost  Decimal? @map("total_cost")
+
+    // Calculated cost
+    calculatedInputCost  Decimal? @map("calculated_input_cost")
+    calculatedOutputCost Decimal? @map("calculated_output_cost")
+    calculatedTotalCost  Decimal? @map("calculated_total_cost")
+
+    completionStartTime DateTime? @map("completion_start_time")
+    project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    promptId String? @map("prompt_id") // no fk constraint, prompt can be deleted
 
     @@unique([id, projectId])
     @@index([projectId, internalModel, startTime, unit])
@@ -328,11 +378,10 @@ model Observation {
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
-    @@index(updatedAt)
     @@index(projectId)
-    @@index(parentObservationId)
     @@index(model)
     @@index(internalModel)
+    @@index([projectId, promptId])
     @@index(promptId)
     @@index([projectId, startTime, type])
     @@map("observations")
@@ -345,7 +394,7 @@ view ObservationView {
     traceId             String?          @map("trace_id")
     projectId           String           @map("project_id")
     type                ObservationType
-    startTime           DateTime         @default(now()) @map("start_time")
+    startTime           DateTime         @map("start_time")
     endTime             DateTime?        @map("end_time")
     name                String?
     metadata            Json?
@@ -353,7 +402,8 @@ view ObservationView {
     level               ObservationLevel @default(DEFAULT)
     statusMessage       String?          @map("status_message")
     version             String?
-    createdAt           DateTime         @default(now()) @map("created_at")
+    createdAt           DateTime         @map("created_at")
+    updatedAt           DateTime         @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -366,7 +416,10 @@ view ObservationView {
     unit                String?
     completionStartTime DateTime? @map("completion_start_time")
 
-    promptId String? @map("prompt_id")
+    // prompts
+    promptId      String? @map("prompt_id")
+    promptName    String? @map("prompt_name")
+    promptVersion Int?    @map("prompt_version")
 
     // model fields
     modelId     String?  @map("model_id")
@@ -403,14 +456,14 @@ model Score {
     projectId     String         @map("project_id")
     project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
-    value         Float
+    value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
     source        ScoreSource
     authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
-    stringValue   String?        @map("string_value")
+    stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
@@ -428,7 +481,6 @@ model Score {
     @@index([observationId], type: Hash)
     @@index([source])
     @@index([createdAt])
-    @@index([updatedAt])
     @@map("scores")
 }
 
@@ -479,17 +531,18 @@ model CronJobs {
 }
 
 model Dataset {
-    id           String        @id @default(cuid())
+    id           String        @default(cuid())
+    projectId    String        @map("project_id")
     name         String
     description  String?
     metadata     Json?
-    projectId    String        @map("project_id")
     project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     createdAt    DateTime      @default(now()) @map("created_at")
     updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
     datasetItems DatasetItem[]
     datasetRuns  DatasetRuns[]
 
+    @@id([id, projectId])
     @@unique([projectId, name])
     @@index([projectId], type: Hash)
     @@index([createdAt])
@@ -498,21 +551,21 @@ model Dataset {
 }
 
 model DatasetItem {
-    id                  String            @id @default(cuid())
+    id                  String            @default(cuid())
+    projectId           String            @map("project_id")
     status              DatasetStatus     @default(ACTIVE)
     input               Json?
     expectedOutput      Json?             @map("expected_output")
     metadata            Json?
     sourceTraceId       String?           @map("source_trace_id")
-    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
     sourceObservationId String?           @map("source_observation_id")
-    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
     datasetId           String            @map("dataset_id")
-    dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    dataset             Dataset           @relation(fields: [datasetId, projectId], references: [id, projectId], onDelete: Cascade)
     createdAt           DateTime          @default(now()) @map("created_at")
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@id([id, projectId])
     @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
@@ -527,17 +580,19 @@ enum DatasetStatus {
 }
 
 model DatasetRuns {
-    id              String            @id @default(cuid())
+    id              String            @default(cuid())
+    projectId       String            @map("project_id")
     name            String
     description     String?
     metadata        Json?
     datasetId       String            @map("dataset_id")
-    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    dataset         Dataset           @relation(fields: [datasetId, projectId], references: [id, projectId], onDelete: Cascade)
     createdAt       DateTime          @default(now()) @map("created_at")
     updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems DatasetRunItems[]
 
-    @@unique([datasetId, name])
+    @@id([id, projectId])
+    @@unique([datasetId, projectId, name])
     @@index([datasetId], type: Hash)
     @@index([createdAt])
     @@index([updatedAt])
@@ -545,16 +600,18 @@ model DatasetRuns {
 }
 
 model DatasetRunItems {
-    id            String      @id @default(cuid())
+    id            String      @default(cuid())
+    projectId     String      @map("project_id")
     datasetRunId  String      @map("dataset_run_id")
-    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId, projectId], references: [id, projectId], onDelete: Cascade)
     datasetItemId String      @map("dataset_item_id")
-    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    datasetItem   DatasetItem @relation(fields: [datasetItemId, projectId], references: [id, projectId], onDelete: Cascade)
     traceId       String      @map("trace_id")
     observationId String?     @map("observation_id")
     createdAt     DateTime    @default(now()) @map("created_at")
     updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
+    @@id([id, projectId])
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)
     @@index([observationId], type: Hash)
@@ -579,6 +636,28 @@ model Events {
     @@map("events")
 }
 
+model Comment {
+    id           String            @id @default(cuid())
+    projectId    String            @map("project_id")
+    project      Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    objectType   CommentObjectType @map("object_type")
+    objectId     String            @map("object_id")
+    createdAt    DateTime          @default(now()) @map("created_at")
+    updatedAt    DateTime          @default(now()) @updatedAt @map("updated_at")
+    content      String
+    authorUserId String?           @map("author_user_id") // no fk constraint, user can be deleted
+
+    @@index([projectId, objectType, objectId])
+    @@map("comments")
+}
+
+enum CommentObjectType {
+    TRACE
+    OBSERVATION
+    SESSION
+    PROMPT
+}
+
 model Prompt {
     id        String   @id @default(cuid())
     createdAt DateTime @default(now()) @map("created_at")
@@ -589,15 +668,14 @@ model Prompt {
 
     createdBy String @map("created_by")
 
-    prompt      Json
-    name        String
-    version     Int
-    type        String        @default("text")
-    isActive    Boolean?      @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-    config      Json          @default("{}")
-    tags        String[]      @default([])
-    labels      String[]      @default([])
-    Observation Observation[]
+    prompt   Json
+    name     String
+    version  Int
+    type     String   @default("text")
+    isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+    config   Json     @default("{}")
+    tags     String[] @default([])
+    labels   String[] @default([])
 
     @@unique([projectId, name, version])
     @@index([projectId, name, version])
@@ -624,7 +702,7 @@ model Model {
     inputPrice      Decimal?  @map("input_price")
     outputPrice     Decimal?  @map("output_price")
     totalPrice      Decimal?  @map("total_price")
-    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, REQUESTS, or IMAGES
     tokenizerId     String?   @map("tokenizer_id")
     tokenizerConfig Json?     @map("tokenizer_config")
 
@@ -635,22 +713,25 @@ model Model {
     @@map("models")
 }
 
+// No FK constraints to preserve audit logs
 model AuditLog {
-    id              String      @id @default(cuid())
-    createdAt       DateTime    @default(now()) @map("created_at")
-    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
-    userId          String      @map("user_id")
-    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String      @map("project_id")
-    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole ProjectRole @map("user_project_role")
-    resourceType    String      @map("resource_type")
-    resourceId      String      @map("resource_id")
+    id              String   @id @default(cuid())
+    createdAt       DateTime @default(now()) @map("created_at")
+    updatedAt       DateTime @default(now()) @updatedAt @map("updated_at")
+    userId          String   @map("user_id")
+    orgId           String   @map("org_id")
+    userOrgRole     String   @map("user_org_role")
+    projectId       String?  @map("project_id")
+    userProjectRole String?  @map("user_project_role")
+    resourceType    String   @map("resource_type")
+    resourceId      String   @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
 
     @@index([projectId])
+    @@index([userId])
+    @@index([orgId])
     @@index([createdAt])
     @@index([updatedAt])
     @@map("audit_logs")
@@ -782,3 +863,27 @@ model PosthogIntegration {
     @@index([projectId])
     @@map("posthog_integrations")
 }
+
+model BatchExport {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String  @map("user_id")
+
+    finishedAt DateTime? @map("finished_at")
+    expiresAt  DateTime? @map("expires_at")
+
+    name   String
+    status String
+    query  Json
+    format String
+    url    String?
+    log    String?
+
+    @@index([projectId, userId])
+    @@index([status])
+    @@map("batch_exports")
+}

packages/shared/prisma/seed.ts

@@ -12,11 +12,17 @@ import { parseArgs } from "node:util";
 import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
-import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server";
 import { encrypt } from "../src/encryption";
+import { redis } from "../src/server/redis/redis";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
+type ConfigCategory = {
+  label: string;
+  value: number;
+};
+
 const options = {
   environment: { type: "string" },
 } as const;
@@ -28,39 +34,111 @@ async function main() {
     options,
   }).values.environment;
 
+  const seedOrgId = "seed-org-id";
+  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  const seedUserId1 = "user-1"; // Owner of org
+  const seedUserId2 = "user-2"; // Member of org, admin of project
+
   const user = await prisma.user.upsert({
-    where: { id: "user-1" },
+    where: { id: seedUserId1 },
     update: {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
     },
     create: {
-      id: "user-1",
+      id: seedUserId1,
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
       image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
+  const user2 = await prisma.user.upsert({
+    where: { id: seedUserId2 },
+    update: {
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId2,
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+  });
 
-  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  await prisma.organization.upsert({
+    where: { id: seedOrgId },
+    update: {
+      name: "Seed Org",
+    },
+    create: {
+      id: seedOrgId,
+      name: "Seed Org",
+    },
+  });
 
   const project1 = await prisma.project.upsert({
     where: { id: seedProjectId },
     update: {
       name: "llm-app",
+      orgId: seedOrgId,
     },
     create: {
       id: seedProjectId,
       name: "llm-app",
-      projectMembers: {
-        create: {
-          role: "OWNER",
-          userId: user.id,
-        },
+      orgId: seedOrgId,
+    },
+  });
+
+  const orgMembership = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user.id,
+        orgId: seedOrgId,
       },
     },
+    create: {
+      userId: user.id,
+      orgId: seedOrgId,
+      role: "OWNER",
+    },
+    update: {},
+  });
+
+  const orgMembership2 = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user2.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user2.id,
+      orgId: seedOrgId,
+      role: "MEMBER",
+    },
+    update: {},
+  });
+
+  const projectMembership = await prisma.projectMembership.upsert({
+    where: {
+      projectId_userId: {
+        projectId: project1.id,
+        userId: user2.id,
+      },
+    },
+    create: {
+      userId: user2.id,
+      projectId: project1.id,
+      role: "ADMIN",
+      orgMembershipId: orgMembership2.id,
+    },
+    update: {
+      orgMembershipId: orgMembership2.id,
+    },
   });
 
   await prisma.prompt.upsert({
@@ -108,18 +186,39 @@ async function main() {
 
   // Do not run the following for local docker compose setup
   if (environment === "examples" || environment === "load") {
-    const project2 = await prisma.project.upsert({
-      where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+    const seedOrgIdOrg2 = "demo-org-id";
+    const project2Id = "239ad00f-562f-411d-af14-831c75ddd875";
+    const org2 = await prisma.organization.upsert({
+      where: { id: seedOrgIdOrg2 },
+      update: {
+        name: "Langfuse Demo",
+      },
       create: {
-        id: "239ad00f-562f-411d-af14-831c75ddd875",
+        id: seedOrgIdOrg2,
+        name: "Langfuse Demo",
+      },
+    });
+    const project2 = await prisma.project.upsert({
+      where: { id: project2Id },
+      create: {
+        id: project2Id,
         name: "demo-app",
-        projectMembers: {
-          create: {
-            role: "OWNER",
-            userId: user.id,
-          },
+        orgId: org2.id,
+      },
+      update: { orgId: seedOrgIdOrg2 },
+    });
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: {
+          userId: user.id,
+          orgId: seedOrgIdOrg2,
         },
       },
+      create: {
+        userId: user.id,
+        orgId: seedOrgIdOrg2,
+        role: "VIEWER",
+      },
       update: {},
     });
 
@@ -158,21 +257,29 @@ async function main() {
 
     const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;
 
-    const { traces, observations, scores, sessions, events } = createObjects(
-      traceVolume,
-      envTags,
-      colorTags,
-      project1,
-      project2,
-      promptIds,
-      configIdsAndNames
-    );
+    const { traces, observations, scores, sessions, events, comments } =
+      createObjects(
+        traceVolume,
+        envTags,
+        colorTags,
+        project1,
+        project2,
+        promptIds,
+        configIdsAndNames
+      );
 
     console.log(
       `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
     );
 
-    await uploadObjects(traces, observations, scores, sessions, events);
+    await uploadObjects(
+      traces,
+      observations,
+      scores,
+      sessions,
+      events,
+      comments
+    );
 
     // If openai key is in environment, add it to the projects LLM API keys
     const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
@@ -281,6 +388,7 @@ async function main() {
             : undefined;
         const datasetItem = await prisma.datasetItem.create({
           data: {
+            projectId: project2.id,
             datasetId: dataset.id,
             sourceTraceId: sourceObservation?.traceId,
             sourceObservationId:
@@ -307,6 +415,7 @@ async function main() {
       for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.create({
           data: {
+            projectId: project2.id,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
@@ -331,6 +440,7 @@ async function main() {
 
           await prisma.datasetRunItems.create({
             data: {
+              projectId: project2.id,
               datasetItemId,
               traceId: observation.traceId as string,
               observationId: Math.random() > 0.5 ? observation.id : undefined,
@@ -346,10 +456,14 @@ async function main() {
 main()
   .then(async () => {
     await prisma.$disconnect();
+    redis?.disconnect();
+    console.log("Disconnected from postgres and redis");
   })
   .catch(async (e) => {
     console.error(e);
     await prisma.$disconnect();
+    redis?.disconnect();
+    console.log("Disconnected from postgres and redis");
     process.exit(1);
   });
 
@@ -358,7 +472,8 @@ async function uploadObjects(
   observations: Prisma.ObservationCreateManyInput[],
   scores: Prisma.ScoreCreateManyInput[],
   sessions: Prisma.TraceSessionCreateManyInput[],
-  events: Prisma.ObservationCreateManyInput[]
+  events: Prisma.ObservationCreateManyInput[],
+  comments: Prisma.CommentCreateManyInput[]
 ) {
   let promises: Prisma.PrismaPromise<unknown>[] = [];
 
@@ -377,11 +492,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Sessions ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -395,11 +509,10 @@ async function uploadObjects(
     );
   });
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Traces ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -413,11 +526,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Observations ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -431,11 +543,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Events ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -448,11 +559,26 @@ async function uploadObjects(
     );
   });
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Scores ${(i / promises.length) * 100}% complete`
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`
+      );
+    await promises[i];
+  }
+
+  promises = [];
+  chunk(comments, chunkSize).forEach((chunk) => {
+    promises.push(
+      prisma.comment.createMany({
+        data: chunk,
+      })
     );
+  });
+  for (let i = 0; i < promises.length; i++) {
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 }
@@ -464,7 +590,15 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
-  configIdsAndNames: Map<string, { name: string; id: string }[]>
+  configParams: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -472,6 +606,7 @@ function createObjects(
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
   const configs: Prisma.ScoreConfigCreateManyInput[] = [];
+  const comments: Prisma.CommentCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -502,6 +637,7 @@ function createObjects(
     const trace = {
       id: `trace-${v4()}`,
       timestamp: traceTs,
+      createdAt: traceTs,
       projectId: projectId,
       name: ["generate-outreach", "label-inbound", "draft-response"][
         i % 3
@@ -523,13 +659,34 @@ function createObjects(
 
     traces.push(trace);
 
-    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const configArray = configParams.get(projectId) ?? [];
     const randomIndex = Math.floor(Math.random() * 3);
     const config =
       configArray.length >= randomIndex - 1 && configArray[randomIndex];
-    const { name: annotationScoreName, id: configId } = config || {
+    const {
+      name: annotationScoreName,
+      id: configId,
+      dataType,
+      categories,
+    } = config || {
       name: "manual-score",
       id: undefined,
+      dataType: ScoreDataType.NUMERIC,
+      categories: null,
+    };
+
+    const value = Math.floor(Math.random() * 2);
+    const scoreNumericAndStringValue = {
+      ...(dataType === ScoreDataType.NUMERIC && { value }),
+      ...(dataType === ScoreDataType.CATEGORICAL && {
+        value,
+        stringValue: categories?.find((category) => category.value === value)
+          ?.label,
+      }),
+      ...(dataType === ScoreDataType.BOOLEAN && {
+        value,
+        stringValue: value === 1 ? "True" : "False",
+      }),
     };
 
     const traceScores = [
@@ -538,12 +695,13 @@ function createObjects(
             {
               traceId: trace.id,
               name: annotationScoreName,
-              value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
+              createdAt: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
-              dataType: ScoreDataType.NUMERIC,
+              dataType,
+              ...scoreNumericAndStringValue,
               ...(configId ? { configId } : {}),
             },
           ]
@@ -555,14 +713,39 @@ function createObjects(
               name: "sentiment",
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
+              createdAt: traceTs,
               source: ScoreSource.API,
               projectId,
               dataType: ScoreDataType.NUMERIC,
             },
           ]
         : []),
+      ...(Math.random() < 0.8
+        ? [
+            {
+              traceId: trace.id,
+              name: "Completeness",
+              timestamp: traceTs,
+              createdAt: traceTs,
+              source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.CATEGORICAL,
+              stringValue:
+                Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+            },
+          ]
+        : []),
     ];
 
+    if (Math.random() > 0.9)
+      comments.push({
+        projectId: trace.projectId,
+        objectId: trace.id,
+        objectType: "TRACE",
+        content: "Trace comment content",
+        ...(Math.random() > 0.5 ? { authorUserId: `user-${i}` } : {}),
+      });
+
     scores.push(...traceScores);
 
     const existingSpanIds: string[] = [];
@@ -581,6 +764,7 @@ function createObjects(
         type: ObservationType.SPAN,
         id: `span-${v4()}`,
         startTime: spanTsStart,
+        createdAt: spanTsStart,
         endTime: spanTsEnd,
         name: `span-${i}-${j}`,
         metadata: {
@@ -652,6 +836,7 @@ function createObjects(
           type: ObservationType.GENERATION,
           id: `generation-${v4()}`,
           startTime: generationTsStart,
+          createdAt: generationTsStart,
           endTime: generationTsEnd,
           completionStartTime:
             Math.random() > 0.5 ? generationTsCompletionStart : undefined,
@@ -667,7 +852,7 @@ function createObjects(
                   },
                   {
                     role: "user",
-                    content: "How can i create a React component?",
+                    content: "How can i create a *React* component?",
                   },
                 ]
               : {
@@ -685,37 +870,8 @@ function createObjects(
                     },
                   ],
                 },
-          output: {
-            completion: `Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.
-
-              1.  **Functional Component**:
-              
-              A functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:
-              
-              
-              'import React from 'react';  function Greeting(props) {   return <h1>Hello, {props.name}</h1>; }  export default Greeting;'
-              
-              To use this component in another file, you can do:
-              
-              
-              'import Greeting from './Greeting';  function App() {   return (     <div>       <Greeting name="John" />     </div>   ); }  export default App;'
-              
-              2.  **Class Component**:
-              
-              You can also define components as classes in React. These have some additional features compared to functional components:
-              
-              
-              'import React, { Component } from 'react';  class Greeting extends Component {   render() {     return <h1>Hello, {this.props.name}</h1>;   } }  export default Greeting;'
-              
-              And here's how to use this component:
-              
-              
-              'import Greeting from './Greeting';  class App extends Component {   render() {     return (       <div>         <Greeting name="John" />       </div>     );   } }  export default App;'
-              
-              With the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.
-              
-              Remember to import React at the top of your file whenever you're creating a component, because JSX transpiles to 'React.createElement' calls under the hood.`,
-          },
+          output:
+            "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.\n\n**Image**\n\n![Languse Example Image](https://static.langfuse.com/langfuse-dev/langfuse-example-image.jpeg)\n\n1.  **Functional Component**:\n\nA functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:\n\n```javascript\nimport React from 'react';\nfunction Greeting(props) {\n  return <h1>Hello, {props.name}</h1>;\n}\nexport default Greeting;\n```\n\nTo use this component in another file, you can do:\n\n```javascript\nimport Greeting from './Greeting';\nfunction App() {\n  return (\n    <div>\n      <Greeting name=\"John\" />\n    </div>\n  );\n}\nexport default App;\n```\n\n2.  **Class Component**:\n\nYou can also define components as classes in React. These have some additional features compared to functional components:\n\n```javascript\nimport React, { Component } from 'react';\nclass Greeting extends Component {\n  render() {\n    return <h1>Hello, {this.props.name}</h1>;\n  }\n}\nexport default Greeting;\n```\n\nAnd here's how to use this component:\n\n```javascript\nimport Greeting from './Greeting';\nclass App extends Component {\n  render() {\n    return (\n      <div>\n        <Greeting name=\"John\" />\n      </div>\n    );\n  }\n}\nexport default App;\n```\n\nWith the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.\n\nRemember to import React at the top of your file whenever you're creating a component, because JSX transpiles to `React.createElement` calls under the hood.",
           model: model,
           internalModel: model,
           modelParameters: {
@@ -751,6 +907,8 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
+            createdAt: traceTs,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -760,6 +918,16 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
+            createdAt: traceTs,
+          });
+
+        if (Math.random() > 0.8)
+          comments.push({
+            projectId: trace.projectId,
+            objectId: generation.id,
+            objectType: "OBSERVATION",
+            content: "Observation comment content",
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -775,6 +943,7 @@ function createObjects(
             type: ObservationType.EVENT,
             id: `event-${v4()}`,
             startTime: eventTs,
+            createdAt: eventTs,
             name: `event-${i}-${j}-${k}-${l}`,
             metadata: {
               user: `user-${i}@langfuse.com`,
@@ -799,6 +968,7 @@ function createObjects(
     configs,
     sessions: uniqueSessions,
     events,
+    comments,
   };
 }
 
@@ -981,8 +1151,15 @@ async function generatePrompts(project: Project) {
 }
 
 async function generateConfigsForProject(projects: Project[]) {
-  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
-    new Map();
+  const projectIdsToConfigs: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  > = new Map();
 
   await Promise.all(
     projects.map(async (project) => {
@@ -994,7 +1171,12 @@ async function generateConfigsForProject(projects: Project[]) {
 }
 
 async function generateConfigs(project: Project) {
-  const configNameAndId: { name: string; id: string }[] = [];
+  const configNameAndId: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[] = [];
 
   const configs = [
     {
@@ -1051,7 +1233,12 @@ async function generateConfigs(project: Project) {
         id: config.id,
       },
     });
-    configNameAndId.push({ name: config.name, id: config.id });
+    configNameAndId.push({
+      name: config.name,
+      id: config.id,
+      dataType: config.dataType,
+      categories: config.categories ?? null,
+    });
   }
 
   return configNameAndId;

packages/shared/scripts/cleanup.sql

@@ -0,0 +1,7 @@
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '_prisma_migrations') THEN
+  DELETE FROM _prisma_migrations
+  WHERE migration_name IN ('20240606090858_pricings_add_latest_gemini_models', '20240530212419_model_price_anthropic_via_google_vertex', '20240604133340_backfill_manual_scores');
+ END IF;
+END $$;

packages/shared/src/constants.ts

@@ -6,4 +6,5 @@ export enum ModelUsageUnit {
   Seconds = "SECONDS",
   Milliseconds = "MILLISECONDS",
   Images = "IMAGES",
+  Requests = "REQUESTS",
 }

packages/shared/src/encryption/index.ts

@@ -1,7 +1,7 @@
 import crypto from "crypto";
 import { env } from "../env";
 
-const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const ENCRYPTION_KEY: string | undefined = env.ENCRYPTION_KEY; // Must be 256 bits (32 bytes, 64 hex characters)
 const IV_LENGTH: number = 16; // For AES, this is always 16
 
 // Alternatively: openssl rand -hex 32
@@ -13,17 +13,16 @@ export function keyGen() {
  * Encrypts the given plain text using AES-256-GCM algorithm.
  *
  * @param {string} plainText - The text to encrypt.
- * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
  * @returns {string} The encrypted data in hex format, including IV and authentication tag.
  */
-export function encrypt(
-  plainText: string,
-  keyHex: string = ENCRYPTION_KEY
-): string {
+export function encrypt(plainText: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
   const cipher = crypto.createCipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   let encrypted = cipher.update(plainText, "utf8", "hex");
@@ -34,7 +33,10 @@ export function encrypt(
   return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
 }
 
-export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+export function decrypt(text: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const [ivHex, encryptedHex, authTagHex] = text.split(":");
   if (!ivHex || !encryptedHex || !authTagHex) {
     throw new Error("Invalid or corrupted cipher format");
@@ -46,7 +48,7 @@ export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
 
   const decipher = crypto.createDecipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   decipher.setAuthTag(authTag);

packages/shared/src/env.ts

@@ -1,6 +1,21 @@
 import { z } from "zod";
 
 const EnvSchema = z.object({
+  NODE_ENV: z
+    .enum(["development", "test", "production"])
+    .default("development"),
+  REDIS_HOST: z.string().nullish(),
+  REDIS_PORT: z.coerce
+    .number({
+      description:
+        ".env files convert numbers to strings, therefoore we have to enforce them to be numbers",
+    })
+    .positive()
+    .max(65536, `options.port should be >= 0 and < 65536`)
+    .default(6379)
+    .nullable(),
+  REDIS_AUTH: z.string().nullish(),
+  REDIS_CONNECTION_STRING: z.string().nullish(),
   ENCRYPTION_KEY: z
     .string()
     .length(
@@ -8,6 +23,20 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
     )
     .optional(),
+  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
+  CLICKHOUSE_URL: z.string().url().optional(),
+  CLICKHOUSE_USER: z.string().optional(),
+  CLICKHOUSE_PASSWORD: z.string().optional(),
+  LANGFUSE_INGESTION_FLUSH_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(10000),
+  LANGFUSE_INGESTION_FLUSH_ATTEMPTS: z.coerce.number().positive().default(3),
+  LANGFUSE_INGESTION_BUFFER_TTL_SECONDS: z.coerce
+    .number()
+    .positive()
+    .default(60 * 10),
 });
 
 export const env = EnvSchema.parse(process.env);

packages/shared/src/errors/InternalServerError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InternalServerError extends BaseError {
+  constructor(description = "Internal Server Error") {
+    super("InternalServerError", 500, description, true);
+  }
+}

packages/shared/src/errors/InvalidRequestError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InvalidRequestError extends BaseError {
+  constructor(description = "Invalid Request Error") {
+    super("InvalidRequestError", 400, description, true);
+  }
+}

packages/shared/src/errors/ValidationError.ts

@@ -1,7 +0,0 @@
-import { BaseError } from "./BaseError";
-
-export class ValidationError extends BaseError {
-  constructor(description = "Validation Error") {
-    super("ValidationError", 400, description, true);
-  }
-}

packages/shared/src/errors/index.ts

@@ -1,7 +1,8 @@
 export { BaseError } from "./BaseError";
 export { LangfuseNotFoundError } from "./NotFoundError";
-export { ValidationError } from "./ValidationError";
+export { InvalidRequestError } from "./InvalidRequestError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
+export { InternalServerError } from "./InternalServerError";

packages/shared/src/features/annotation/types.ts

@@ -0,0 +1,59 @@
+import z from "zod";
+import { type ScoreDataType } from "../../db";
+
+const NUMERIC: ScoreDataType = "NUMERIC";
+const CATEGORICAL: ScoreDataType = "CATEGORICAL";
+const BOOLEAN: ScoreDataType = "BOOLEAN";
+
+export const availableDataTypes = [NUMERIC, CATEGORICAL, BOOLEAN] as const;
+
+const NumericData = z.object({
+  value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
+});
+
+const CategoricalData = z.object({
+  value: z.number().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
+
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
+
+const CreateAnnotationScoreBase = z.object({
+  name: z.string(),
+  projectId: z.string(),
+  traceId: z.string(),
+  configId: z.string().optional(),
+  observationId: z.string().optional(),
+  comment: z.string().nullish(),
+});
+
+const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
+  id: z.string(),
+});
+
+/**
+ * CreateAnnotationScoreData is only used for annotation scores created via the UI.
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const CreateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  CreateAnnotationScoreBase.merge(NumericData),
+  CreateAnnotationScoreBase.merge(CategoricalData),
+  CreateAnnotationScoreBase.merge(BooleanData),
+]);
+
+/**
+ * UpdateAnnotationScoreData is only used for annotation scores updated via the UI
+ * For langfuse score types please refer to `web/src/features/public-api/types/scores.ts`
+ */
+export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
+  UpdateAnnotationScoreBase.merge(NumericData),
+  UpdateAnnotationScoreBase.merge(CategoricalData),
+  UpdateAnnotationScoreBase.merge(BooleanData),
+]);

packages/shared/src/features/batchExport/types.ts

@@ -0,0 +1,73 @@
+import z from "zod";
+
+import { BatchExport } from "@prisma/client";
+
+import { singleFilter } from "../../interfaces/filters";
+import { orderBy } from "../../interfaces/orderBy";
+
+export enum BatchExportStatus {
+  QUEUED = "QUEUED",
+  PROCESSING = "PROCESSING",
+  COMPLETED = "COMPLETED",
+  FAILED = "FAILED",
+}
+
+export enum BatchExportFileFormat {
+  JSON = "JSON",
+  CSV = "CSV",
+}
+
+export enum BatchExportTableName {
+  Sessions = "sessions",
+}
+
+export const exportOptions: Record<
+  BatchExportFileFormat,
+  {
+    label: string;
+    extension: string;
+    fileType: string;
+  }
+> = {
+  CSV: { label: "CSV", extension: "csv", fileType: "text/csv" },
+  JSON: { label: "JSON", extension: "json", fileType: "application/json" },
+} as const;
+
+export const BatchExportQuerySchema = z.object({
+  tableName: z.nativeEnum(BatchExportTableName),
+  filter: z.array(singleFilter).nullable(),
+  orderBy,
+  limit: z.number().optional(),
+  page: z.number().optional(),
+});
+
+export type BatchExportQueryType = z.infer<typeof BatchExportQuerySchema>;
+
+export const CreateBatchExportSchema = z.object({
+  projectId: z.string(),
+  name: z.string(),
+  query: BatchExportQuerySchema,
+  format: z.nativeEnum(BatchExportFileFormat),
+});
+
+export const BatchExportSchema = z.object({
+  id: z.string(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  projectId: z.string(),
+  userId: z.string(),
+  finishedAt: z.date().nullable(),
+  expiresAt: z.date().nullable(),
+  name: z.string(),
+  status: z.nativeEnum(BatchExportStatus),
+  query: BatchExportQuerySchema,
+  format: z.nativeEnum(BatchExportFileFormat),
+  url: z.string().nullable(),
+  log: z.string().nullable(),
+});
+
+// Ensure that zod type matches the Prisma type
+export type BatchExportType =
+  z.infer<typeof BatchExportSchema> extends BatchExport
+    ? z.infer<typeof BatchExportSchema>
+    : never;

packages/shared/src/features/comments/types.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+const COMMENT_OBJECT_TYPES = [
+  "TRACE",
+  "OBSERVATION",
+  "SESSION",
+  "PROMPT",
+] as const;
+
+export const CreateCommentData = z.object({
+  projectId: z.string(),
+  content: z.string().trim().min(1).max(500),
+  objectId: z.string(),
+  objectType: z.enum(COMMENT_OBJECT_TYPES),
+});
+
+export const DeleteCommentData = z.object({
+  projectId: z.string(),
+  commentId: z.string(),
+  objectId: z.string(),
+  objectType: z.enum(COMMENT_OBJECT_TYPES),
+});

packages/shared/src/features/evals/types.ts

@@ -80,6 +80,8 @@ export const availableEvalVariables = [
 export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
   [LLMAdapter.OpenAI]: [
     "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4o-2024-08-06", // cheaper than gpt-4o
     "gpt-4-turbo-preview",
     "gpt-3.5-turbo",
   ] as const,

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -0,0 +1,236 @@
+import { z } from "zod";
+
+import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
+
+import { isPresent } from "../../utils/typeChecks";
+import {
+  jsonSchema,
+  paginationMetaResponseZod,
+  paginationZod,
+} from "../../utils/zod";
+
+/**
+ * Types to use across codebase
+ */
+export type ConfigCategory = z.infer<typeof Category>;
+export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
+
+const validateCategories = (
+  categories: ConfigCategory[],
+  ctx: z.RefinementCtx
+) => {
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories) {
+    if (uniqueNames.has(category.label)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category label: ${category.label}, category labels must be unique`,
+      });
+      return;
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category value: ${category.value}, category values must be unique`,
+      });
+      return;
+    }
+    uniqueValues.add(category.value);
+  }
+};
+
+/**
+ * Objects
+ */
+export const Category = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+const Categories = z.array(Category);
+
+const NumericScoreConfig = z.object({
+  maxValue: z.number().nullish(),
+  minValue: z.number().nullish(),
+  dataType: z.literal("NUMERIC"),
+  categories: z.undefined().nullish(),
+});
+
+const CategoricalScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("CATEGORICAL"),
+  categories: jsonSchema.superRefine((categories, ctx) => {
+    const parseResult = Categories.safeParse(categories);
+    if (!parseResult.success) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "Category must be an array of objects with label value pairs, where labels and values are unique.",
+      });
+      return;
+    }
+
+    validateCategories(categories as ConfigCategory[], ctx);
+  }),
+});
+
+const BooleanScoreConfig = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("BOOLEAN"),
+  categories: z
+    .array(Category)
+    .length(2, "Boolean data type must have exactly 2 categories.")
+    .refine((categories) => {
+      const expectedCategories = [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ];
+      return categories.every(
+        (category, index) =>
+          category.label === expectedCategories[index].label &&
+          category.value === expectedCategories[index].value
+      );
+    }),
+});
+
+const ScoreConfigBase = z.object({
+  id: z.string(),
+  name: z.string().min(1).max(35),
+  isArchived: z.boolean(),
+  description: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  projectId: z.string(),
+});
+
+const ScoreConfigPostBase = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+});
+
+const ValidatedScoreConfigSchema = z
+  .union([
+    ScoreConfigBase.merge(NumericScoreConfig),
+    ScoreConfigBase.merge(
+      z.object({
+        maxValue: z.undefined().nullish(),
+        minValue: z.undefined().nullish(),
+        dataType: z.literal("CATEGORICAL"),
+        categories: Categories.superRefine(validateCategories),
+      })
+    ),
+    ScoreConfigBase.merge(BooleanScoreConfig),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+/**
+ * Use this function when pulling a list of score configs from the database before using in the application to ensure type safety.
+ * All score configs are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scoreConfigs
+ * @returns list of validated score configs
+ */
+export const filterAndValidateDbScoreConfigList = (
+  scoreConfigs: ScoreConfigDbType[],
+  onParseError?: (error: z.ZodError) => void
+): ValidatedScoreConfig[] =>
+  scoreConfigs.reduce((acc, ts) => {
+    const result = ValidatedScoreConfigSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as ValidatedScoreConfig[]);
+
+/**
+ * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
+ * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
+ * @param scoreConfig
+ * @returns validated score config
+ * @throws error if score fails validation
+ */
+export const validateDbScoreConfig = (
+  scoreConfig: ScoreConfigDbType
+): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
+
+/**
+ * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
+ * This function will NOT throw an error by default. The score is expected to pass the validation.
+ * @param scoreConfig
+ * @returns score config validation object:
+ * - success: true if the score config passes validation
+ * - data: the validated score config if success is true
+ * - error: the error object if success is false
+ */
+export const validateDbScoreConfigSafe = (scoreConfig: ScoreConfigDbType) =>
+  ValidatedScoreConfigSchema.safeParse(scoreConfig);
+
+/**
+ * Endpoints
+ */
+
+// GET /score-configs/{configId}
+export const GetScoreConfigQuery = z.object({
+  configId: z.string(),
+});
+
+export const GetScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// POST /score-configs
+export const PostScoreConfigBody = z
+  .union([
+    ScoreConfigPostBase.merge(CategoricalScoreConfig),
+    ScoreConfigPostBase.merge(NumericScoreConfig),
+    ScoreConfigPostBase.merge(
+      z.object({
+        dataType: z.literal("BOOLEAN"),
+        categories: z.undefined().nullish(),
+      })
+    ),
+  ])
+  .superRefine((data, ctx) => {
+    if (data.dataType === "NUMERIC") {
+      if (
+        isPresent(data.maxValue) &&
+        isPresent(data.minValue) &&
+        data.maxValue <= data.minValue
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: "Maximum value must be greater than Minimum value",
+        });
+      }
+    }
+  });
+
+export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
+
+// GET /score-configs
+export const GetScoreConfigsQuery = z.object({
+  ...paginationZod,
+});
+
+export const GetScoreConfigsResponse = z.object({
+  data: z.array(ValidatedScoreConfigSchema),
+  meta: paginationMetaResponseZod,
+});

packages/shared/src/features/scores/scoreTypes.ts

@@ -0,0 +1,298 @@
+import { z } from "zod";
+
+import { Score } from "@prisma/client";
+
+import { isPresent, stringDateTime } from "../../utils/typeChecks";
+import {
+  NonEmptyString,
+  paginationMetaResponseZod,
+  paginationZod,
+} from "../../utils/zod";
+import { Category as ConfigCategory } from "./scoreConfigTypes";
+
+/**
+ * Types to use across codebase
+ */
+export type APIScore = z.infer<typeof APIScoreSchema>;
+
+/**
+ * Helpers
+ */
+
+const ScoreSource = ["API", "EVAL", "ANNOTATION"] as const;
+const ScoreDataType = ["NUMERIC", "CATEGORICAL", "BOOLEAN"] as const;
+
+const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
+
+const NumericData = z.object({
+  value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
+});
+
+const CategoricalData = z.object({
+  value: z.number().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
+
+const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
+
+const ScoreBase = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  projectId: z.string(),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  authorUserId: z.string().nullish(),
+  comment: z.string().nullish(),
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  configId: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+const BaseScoreBody = z.object({
+  id: z.string().nullish(),
+  name: NonEmptyString,
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+  comment: z.string().nullish(),
+});
+
+/**
+ * Objects
+ */
+
+export const APIScoreSchema = z.discriminatedUnion("dataType", [
+  ScoreBase.merge(NumericData),
+  ScoreBase.merge(CategoricalData),
+  ScoreBase.merge(BooleanData),
+]);
+
+/**
+ * Validation objects
+ */
+export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number(),
+      dataType: z.literal("NUMERIC"),
+    })
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.string(),
+      dataType: z.literal("CATEGORICAL"),
+    })
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number().refine((val) => val === 0 || val === 1, {
+        message: "Value must be either 0 or 1",
+      }),
+      dataType: z.literal("BOOLEAN"),
+    })
+  ),
+]);
+
+const ScorePropsAgainstConfigNumeric = z
+  .object({
+    value: z.number(),
+    maxValue: z.number().optional(),
+    minValue: z.number().optional(),
+    dataType: z.literal("NUMERIC"),
+  })
+  .superRefine((data, ctx) => {
+    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
+      });
+    }
+    if (isPresent(data.minValue) && data.value <= data.minValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value is below minimum value of ${data.minValue} defined in config`,
+      });
+    }
+  });
+
+const ScorePropsAgainstConfigCategorical = z
+  .object({
+    value: z.string(),
+    categories: z.array(ConfigCategory),
+    dataType: z.literal("CATEGORICAL"),
+  })
+  .superRefine((data, ctx) => {
+    if (!data.categories.some(({ label }) => label === data.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value ${data.value} does not map to a valid category. Pass a valid category value.`,
+      });
+    }
+  });
+
+export const ScorePropsAgainstConfig = z.union([
+  ScorePropsAgainstConfigNumeric,
+  ScorePropsAgainstConfigCategorical,
+  z.object({
+    value: z.number().refine((val) => val === 0 || val === 1, {
+      message: "Value must be either 0 or 1",
+    }),
+    dataType: z.literal("BOOLEAN"),
+  }),
+]);
+
+/**
+ * Transformations
+ */
+
+/**
+ * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scores
+ * @returns list of validated scores
+ */
+export const filterAndValidateDbScoreList = (
+  scores: Score[],
+  onParseError?: (error: z.ZodError) => void
+): APIScore[] =>
+  scores.reduce((acc, ts) => {
+    const result = APIScoreSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      console.error("Score parsing error: ", result.error);
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as APIScore[]);
+
+/**
+ * Use this function when pulling a single score from the database before using in the application to ensure type safety.
+ * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
+ * @param score
+ * @returns validated score
+ * @throws error if score fails validation
+ */
+export const validateDbScore = (score: Score): APIScore =>
+  APIScoreSchema.parse(score);
+
+/**
+ * Endpoints
+ */
+
+// POST /scores
+/**
+ * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
+ */
+export const PostScoresBody = z.discriminatedUnion("dataType", [
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number(),
+      dataType: z.literal("NUMERIC"),
+      configId: z.string().nullish(),
+    })
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.string(),
+      dataType: z.literal("CATEGORICAL"),
+      configId: z.string().nullish(),
+    })
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number().refine((value) => value === 0 || value === 1, {
+        message:
+          "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
+      }),
+      dataType: z.literal("BOOLEAN"),
+      configId: z.string().nullish(),
+    })
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.union([z.string(), z.number()]),
+      dataType: z.undefined(),
+      configId: z.string().nullish(),
+    })
+  ),
+]);
+
+export const PostScoresResponse = z.object({ id: z.string() });
+
+// GET /scores
+export const GetScoresQuery = z.object({
+  ...paginationZod,
+  userId: z.string().nullish(),
+  dataType: z.enum(ScoreDataType).nullish(),
+  configId: z.string().nullish(),
+  name: z.string().nullish(),
+  fromTimestamp: stringDateTime,
+  toTimestamp: stringDateTime,
+  source: z.enum(ScoreSource).nullish(),
+  value: z.coerce.number().nullish(),
+  operator: z.enum(operators).nullish(),
+  scoreIds: z
+    .string()
+    .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
+    .refine((arr) => arr.every((id) => typeof id === "string"), {
+      message: "Each score ID must be a string",
+    })
+    .nullish(),
+});
+
+// LegacyGetScoreResponseDataV1 is only used for response of GET /scores list endpoint
+const LegacyGetScoreResponseDataV1 = z.intersection(
+  APIScoreSchema,
+  z.object({
+    trace: z.object({
+      userId: z.string().nullish(),
+    }),
+  })
+);
+export const GetScoresResponse = z.object({
+  data: z.array(LegacyGetScoreResponseDataV1),
+  meta: paginationMetaResponseZod,
+});
+
+export const legacyFilterAndValidateV1GetScoreList = (
+  scores: unknown[],
+  onParseError?: (error: z.ZodError) => void
+): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
+  scores.reduce(
+    (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
+      const result = LegacyGetScoreResponseDataV1.safeParse(ts);
+      if (result.success) {
+        acc.push(result.data);
+      } else {
+        console.error("Score parsing error: ", result.error);
+        onParseError?.(result.error);
+      }
+      return acc;
+    },
+    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[]
+  );
+
+// GET /scores/{scoreId}
+export const GetScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const GetScoreResponse = APIScoreSchema;
+
+// DELETE /scores/{scoreId}
+export const DeleteScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const DeleteScoreResponse = z.object({
+  message: z.string(),
+});