chore: release v2.60.2

push

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+{
+	"name": "langfuse dev",
+    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+        "version": "latest",
+        "moby": true
+        }
+    },
+    "forwardPorts": [3000],
+
+    "onCreateCommand": "npm install -g pnpm@8.15.5",
+    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+}

.env.dev.example

@@ -34,10 +34,6 @@ S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 S3_REGION=
 
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Set during docker build of application
 # Used to disable environment verification at build time
 # DOCKER_BUILD=1
@@ -47,5 +43,5 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
-# openssl rand -base64 32 used only here
+# openssl rand -hex 32 used only here
 ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.local.example

@@ -13,11 +13,6 @@ NEXTAUTH_URL="http://localhost:3000"
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"
 
-
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379

.env.prod.example

@@ -9,6 +9,8 @@
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
+# optional, set to true to disable automated database migrations on Docker start
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED= 
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
@@ -23,6 +25,7 @@ SALT="salt"
 # Use CSP headers to enforce HTTPS, optional
 LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
+
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
@@ -34,6 +37,9 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Enable experimental features, optional
 # LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
+# Disable automated database migrations on Docker start, optional
+# LANFUSE_WEB_MIGRATION_DISABLED=true
+
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
@@ -43,6 +49,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_GOOGLE_CLIENT_ID=
 # AUTH_GOOGLE_CLIENT_SECRET=
 # AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GOOGLE_ALLOWED_DOMAINS=langfuse.com,google.com # optional allowlist of workspace domains that can sign in via Google
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
@@ -58,6 +65,16 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_CLIENT_ID=
+# AUTH_CUSTOM_CLIENT_SECRET=
+# AUTH_CUSTOM_ISSUER=
+# AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -71,6 +88,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # S3_SECRET_ACCESS_KEY=
 # S3_BUCKET_NAME=
 # S3_REGION=
+# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
 
 # Exports are streamed to S3 in pages to avoid memory issues
 # The page size can be adjusted if needed to optimize performance
@@ -96,7 +114,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
-# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
 
 # NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
 # NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
@@ -113,15 +131,17 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_SENTRY_ORG=
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
+# Used to determine the Sentry sample rate
+# LANGFUSE_TRACING_SAMPLE_RATE=
+
+# NewRelic
+# NEW_RELIC_API_KEY= 
+# OTLP_ENDPOINT=
 
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 
@@ -137,4 +157,5 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Admin API
 # ADMIN_API_KEY=
 
+
 ### END Langfuse Cloud Config

.github/workflows/pipeline.yml

@@ -40,25 +40,35 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build Web Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./web/Dockerfile
-          push: false
+      - name: Build and run both images from compose
+        run: |
+          docker compose -f docker-compose.build.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Build Worker Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: false
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Ensure no unhealthy status
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
 
   tests-web:
     runs-on: ubuntu-latest
+    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -89,6 +99,9 @@ jobs:
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
 
       - name: Seed DB
         run: |
@@ -106,9 +119,11 @@ jobs:
 
   tests-worker:
     runs-on: ubuntu-latest
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -139,6 +154,7 @@ jobs:
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
 
       - name: Seed DB
         run: |
@@ -176,6 +192,7 @@ jobs:
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
 
       - name: Seed DB
@@ -242,6 +259,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
@@ -265,6 +290,9 @@ jobs:
           push: true
           tags: ${{ steps.meta-web.outputs.tags }}
           labels: ${{ steps.meta-web.outputs.labels }}
+          platforms: |
+            linux/amd64
+            linux/arm64
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
@@ -289,3 +317,6 @@ jobs:
           push: true
           tags: ${{ steps.meta-worker.outputs.tags }}
           labels: ${{ steps.meta-worker.outputs.labels }}
+          platforms: |
+            linux/amd64
+            linux/arm64

.github/workflows/porter_stack_web_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - production
+name: Deploy to web
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: web
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_web_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to web (Staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_web_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to web (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - production
+name: Deploy to worker (EU temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to worker (staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_worker_us.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (US)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4038"
+          PORTER_DEPLOYMENT_TARGET_ID: 260cd04e-5dc8-4b22-a291-ad4453f981f7
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4038 }}

.github/workflows/porter_stack_worker_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/snyk.yml

@@ -0,0 +1,39 @@
+name: Snyk Container
+on: push
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker compose -f docker-compose.build.yml up -d
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-server
+          args: --file=web/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: web
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-worker
+          args: --file=worker/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: worker

.gitignore

@@ -1,4 +1,5 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+**/newrelic_agent.log
 
 # dependencies
 /node_modules

.husky/pre-commit

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Commit will proceed
+    else
+        echo "Commit to $protected_branch branch has been canceled."
+        exit 1 # Commit will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the commit
+exit 0

.husky/pre-push

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to push to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Push will proceed
+    else
+        echo "Push to $protected_branch branch has been canceled."
+        exit 1 # Push will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the push
+exit 0

CONTRIBUTING.md

@@ -55,7 +55,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 > [!NOTE]
 > Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
 
 ```mermaid
 flowchart TB
@@ -151,6 +151,8 @@ Requirements
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
 - Docker to run the database locally
 
+**Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
+
 **Steps**
 
 1. Fork the repository and clone it locally
@@ -191,6 +193,16 @@ Requirements
     pnpm run dev
    ```
 
+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -290,7 +302,77 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
-## Using secrets stored in 1Password
+## Theming
+
+At Langfuse, we utilize CSS variables to manage our theme settings across the platform.
+
+Our approach leverages separate CSS variables for backgrounds (--background) and foregrounds (--foreground), fully adhering to the [shadcn/ui](https://ui.shadcn.com/docs/theming) color conventions. The background suffix can be omitted if the variable is used for the background color of the component. We recommend using HSL values for these colors to enhance consistency and customization. There is no need to manually handle dark mode styling with "dark:" prefixes, as next-themes automatically manages the theme switching.
+
+Given the following CSS variables:
+
+```
+--primary: 222.2 47.4% 11.2%; // e.g. background-color
+--primary-foreground: 210 40% 98%; // e.g. text-color
+```
+
+The background color of the following component will be `hsl(var(--primary))` and the foreground color will be `hsl(var(--primary-foreground))`.
+
+```
+<div class="bg-primary text-primary-foreground">Hello</div>
+```
+
+### Color Variables
+
+| Variable                 | Description                                                        | Examples                         |
+| ------------------------ | ------------------------------------------------------------------ | -------------------------------- |
+| --background             | Background color                                                   | Default background color of body |
+| --foreground             | Foreground color                                                   | Default text color of body       |
+| --muted                  | Muted background color                                             | TabsList, Skeleton and Switch    |
+| --muted-foreground       | Muted foreground color                                             |                                  |
+| --popover                | Popover background color                                           | DropdownMenu, HoverCard, Popover |
+| --popover-foreground     | Popover foreground color                                           |                                  |
+| --card                   | Card background color                                              | Card                             |
+| --card-foreground        | Card foreground color                                              |                                  |
+| --border                 | Border color                                                       | Default border color             |
+| --input                  | Input field border color                                           | Input, Select, Textarea          |
+| --primary                | Primary button background colors                                   | Button variant="primary"         |
+| --primary-foreground     | Primary button foreground color                                    |                                  |
+| --secondary              | Secondary button background color                                  | Button variant="secondary"       |
+| --secondary-foreground   | Secondary button foreground color                                  |                                  |
+| --accent                 | Used for accents such as hover effects                             | DropdownMenuItem, SelectItem     |
+| --accent-foreground      | Used for texts on hover effects                                    | DropdownMenuItem, SelectItem     |
+| --destructive            | Destructive action color for background                            | Button variant="destructive"     |
+| --destructive-foreground | Destructive action color for text                                  |                                  |
+| --ring                   | Focus ring color                                                   | MultiSelect                      |
+| --primary-accent         | Primary accent color used for branding                             | Layout                           |
+| --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
+| --muted-green            | Muted green for Event label                                        | ObservationTree                  |
+| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
+| --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
+| --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
+| --accent-dark-green      | Dark green accent for border of output and assistant messages      | CodeJsonViewer and IOPReview     |
+| --light-red              | Light red for error background                                     | level-color and StatusBadge      |
+| --dark-red               | Dark red for error text and error badge dot color                  | level-color and ErrorPage        |
+| --light-yellow           | Light yellow for warning background                                | LevelColor                       |
+| --dark-yellow            | Dark yellow for warning text                                       | LevelColor                       |
+| --light-green            | Light green for success status badge background                    | StatusBadge                      |
+| --dark-green             | Dark green for success status badge text and dot                   | StatusBadge                      |
+| --light-blue             | Light blue for background of Staging label                         | LangfuseLogo                     |
+| --dark-blue              | Dark blue for text and border of Staging label                     | LangfuseLogo                     |
+| --accent-light-blue      | Light blue accent for table link hover effect                      | TableLink                        |
+| --accent-dark-blue       | Dark blue accent for table link text                               | TableLink                        |
+
+### Adding New Colors
+
+1. Global Definitions: Add new CSS variable definitions in the global.css file.
+2. Tailwind Configuration: Reflect these new colors in the tailwind.config.js to maintain alignment with Tailwind's utility classes.
+
+By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
+
+## Maintainers
+
+### Using secrets stored in 1Password
 
 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
 

README.md

@@ -24,6 +24,8 @@
    </div>
    <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
+   <span>We're hiring. <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in Backend Engineering, Product Engineering, and Developer Relations.</span>
+   <br/>
    <br/>
    <div>
       <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
@@ -106,22 +108,26 @@ See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Lan
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                 |
-| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
+| Integration                                              | Supports                   | Description                                                                                                                                      |
+| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
 
-External projects/packages that integrate with Langfuse:
+Packages that integrate with Langfuse:
 
-| Name                                                            | Description                                                                                                                                      |
-| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
-| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
+| Name                                                       | Description                                                                                                             |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
+| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
+| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
+| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
 
 ## Questions and feedback
 
@@ -153,7 +159,7 @@ This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICE
 
 ### GET API to export your data
 
-[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics). You can also access them conveniently via the SDKs ([docs](https://langfuse.com/docs/query-traces)).
 
 ### Security & Privacy
 

careers

@@ -0,0 +1,14 @@
+_We are Hiring_
+Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+
+_Open Roles_
+
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
+_More Info_
+
+- https://langfuse.com/careers
+- https://langfuse.com/docs
+- https://langfuse.com/changelog

docker-compose.build.yml

@@ -5,6 +5,8 @@ services:
     build:
       dockerfile: ./web/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis
@@ -20,11 +22,18 @@ services:
       - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis
@@ -41,6 +50,11 @@ services:
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:7.2.4

docker-compose.dev.yml

@@ -1,5 +1,3 @@
-version: "3.5"
-
 services:
   redis:
     image: redis:7.2.4
@@ -10,7 +8,7 @@ services:
       - 6379:6379
 
   db:
-    image: postgres
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     command: ["postgres", "-c", "log_statement=all"]
     environment:

docker-compose.yml

@@ -1,10 +1,9 @@
-version: "3.5"
-
 services:
   langfuse-server:
-    image: ghcr.io/langfuse/langfuse:latest
+    image: langfuse/langfuse:2
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     ports:
       - "3000:3000"
     environment:
@@ -18,6 +17,11 @@ services:
   db:
     image: postgres
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres

ee/package.json

@@ -33,7 +33,7 @@
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

ee/src/sso/index.ts

@@ -2,14 +2,16 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
-import { prisma } from "@langfuse/shared/src/db";
+import { type SsoConfig, prisma } from "@langfuse/shared/src/db";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { env } from "../env";
+import { CustomSSOProvider } from "@langfuse/shared/src/server";
 
 // Local cache for SSO configurations
 let cachedSsoConfigs: {
@@ -25,14 +27,36 @@ let cachedSsoConfigs: {
 async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
   if (!isEeAvailable) return [];
   const CACHE_TTL = 60 * 1000; // 1 minute
+  const DB_MAX_WAIT = 2000; // 2 seconds
+  const DB_TIMEOUT = 3000; // 3 seconds
 
   // Set/refresh the cache if it's empty or expired
   if (
     cachedSsoConfigs === null ||
     Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
   ) {
+    // findMany with custom timeout via $transaction
+    let dbConfigs: SsoConfig[] = [];
+    try {
+      dbConfigs = await prisma.$transaction(
+        async (prisma) => prisma.ssoConfig.findMany(),
+        {
+          maxWait: DB_MAX_WAIT,
+          timeout: DB_TIMEOUT,
+        }
+      );
+    } catch (e) {
+      // cache empty array to prevent repeated DB calls on error
+      cachedSsoConfigs = {
+        data: [],
+        timestamp: Date.now(),
+      };
+
+      // caught and logged in the caller
+      throw e;
+    }
+
     // transform into zod object
-    const dbConfigs = await prisma.ssoConfig.findMany();
     const parsedSsoConfigs = dbConfigs
       .map((v) => {
         try {
@@ -144,6 +168,21 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "custom")
+    return CustomSSOProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+      authorization: {
+        params: { scope: provider.authConfig.scope ?? "openid email profile" },
+      },
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,39 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const CustomProviderSchema = base.extend({
+  authProvider: z.literal("custom"),
+  authConfig: z
+    .object({
+      name: z.string(),
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      scope: z.string().nullish(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +104,8 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
+  CustomProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/client/definition/score.yml

@@ -17,15 +17,131 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
-  Score:
+      dataType:
+        type: optional<ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
+      source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+  ScoreSource:
+    enum:
+      - ANNOTATION
+      - API
+      - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - CATEGORICAL
+      - BOOLEAN
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores

fern/apis/server/definition/commons.yml

@@ -111,18 +111,103 @@ types:
       totalCost:
         docs: USD total cost, defaults to input+output
         type: optional<double>
-
-  Score:
+  ScoreConfig:
+    docs: Configuration for a score
+    properties:
+      id: string
+      name: string
+      createdAt: datetime
+      updatedAt: datetime
+      projectId: string
+      dataType: ScoreDataType
+      isArchived:
+        type: boolean
+        docs: Whether the score config is archived. Defaults to false
+      minValue:
+        type: optional<double>
+        docs: Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      categories:
+        type: optional<list<ConfigCategory>>
+        docs: Configures custom categories for categorical scores
+      description: optional<string>
+  ConfigCategory:
+    properties:
+      value: double
+      label: string
+  NumericScore:
     properties:
       id: string
       traceId: string
       name: string
-      value: double
+      value:
+        type: double
+        docs: The numeric value of the score
       source: ScoreSource
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
-  DatasetCore:
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+  BooleanScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal
+  CategoricalScore:
+    properties:
+      id: string
+      traceId: string
+      name: string
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+      source: ScoreSource
+      observationId: optional<string>
+      timestamp: datetime
+      comment: optional<string>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and stringValue must map to a config category
+  Score:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScore
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScore
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScore
+        docs: "Score with BOOLEAN data type"
+
+  CreateScoreValue:
+    discriminated: false
+    union:
+      - string
+      - double
+    docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
+  Dataset:
     properties:
       id: string
       name: string
@@ -131,21 +216,6 @@ types:
       projectId: string
       createdAt: datetime
       updatedAt: datetime
-  DatasetWithReferences: # GET datasets
-    extends: DatasetCore
-    properties:
-      items:
-        type: list<string>
-        docs: list of dataset item ids
-      runs:
-        type: list<string>
-        docs: list of dataset run names
-  Dataset: # GET datasets/[name]
-    docs: Dataset including all items
-    extends: DatasetCore
-    properties:
-      items: list<DatasetItem>
-      runs: list<string>
   DatasetItem:
     properties:
       id: string
@@ -179,7 +249,43 @@ types:
       datasetName: string
       createdAt: datetime
       updatedAt: datetime
+  DatasetRunWithItems:
+    extends: DatasetRun
+    properties:
       datasetRunItems: list<DatasetRunItem>
+  Model:
+    docs: Model definition used for transforming usage into USD cost and/or tokenization.
+    properties:
+      id: string
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total unit. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>
+      isLangfuseManaged:
+        type: boolean
 
   # Utilities
   ModelUsageUnit:
@@ -209,9 +315,14 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - BOOLEAN
+      - CATEGORICAL
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
@@ -19,6 +20,23 @@ service:
         id:
           type: string
       response: commons.DatasetItem
+    list:
+      docs: Get dataset items
+      method: GET
+      path: /dataset-items
+      request:
+        name: GetDatasetItemsRequest
+        query-parameters:
+          datasetName: optional<string>
+          sourceTraceId: optional<string>
+          sourceObservationId: optional<string>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetItems
 
 types:
   CreateDatasetItemRequest:
@@ -31,7 +49,11 @@ types:
       sourceObservationId: optional<string>
       id:
         type: optional<string>
-        docs: Dataset items are upserted on their id
+        docs: Dataset items are upserted on their id. Id needs to be globally unique and cannot be reused across datasets.
       status:
         type: optional<commons.DatasetStatus>
         docs: Defaults to ACTIVE for newly created items
+  PaginatedDatasetItems:
+    properties:
+      data: list<commons.DatasetItem>
+      meta: pagination.MetaResponse

fern/apis/server/definition/datasets.yml

@@ -9,42 +9,66 @@ service:
     list:
       method: GET
       docs: Get all datasets
-      path: /datasets
+      path: /v2/datasets
       request:
         name: GetDatasetsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
       response: PaginatedDatasets
     get:
       method: GET
-      docs: Get a dataset and its items
-      path: /datasets/{datasetName}
+      docs: Get a dataset
+      path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
       response: commons.Dataset
     create:
       method: POST
       docs: Create a dataset
-      path: /datasets
+      path: /v2/datasets
       request: CreateDatasetRequest
       response: commons.Dataset
-    getRuns:
+    getRun:
       method: GET
       docs: Get a dataset run and its items
       path: /datasets/{datasetName}/runs/{runName}
       path-parameters:
         datasetName: string
         runName: string
-      response: commons.DatasetRun
+      response: commons.DatasetRunWithItems
+    getRuns:
+      method: GET
+      docs: Get dataset runs
+      path: /datasets/{datasetName}/runs
+      path-parameters:
+        datasetName: string
+      request:
+        name: GetDatasetRunsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetRuns
 
 types:
   PaginatedDatasets:
     properties:
-      data: list<commons.DatasetWithReferences>
+      data: list<commons.Dataset>
       meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
       description: optional<string>
       metadata: optional<unknown>
+  PaginatedDatasetRuns:
+    properties:
+      data: list<commons.DatasetRun>
+      meta: pagination.MetaResponse

fern/apis/server/definition/ingestion.yml

@@ -7,28 +7,57 @@ service:
   base-path: /api/public
   endpoints:
     batch:
-      docs: Batched ingestion for Langfuse Tracing
+      docs: |
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
+
+        Notes:
+        - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
+        - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
       path: /ingestion
       request:
         name: IngestionRequest
         body:
           properties:
-            batch: list<IngestionEvent>
-      response: IngestionResponse # will be reportet as 200 response, but endpoint returns 207
+            batch:
+              type: list<IngestionEvent>
+              docs: "Batch of tracing events to be ingested. Discriminated by attribute `type`."
+            metadata:
+              type: optional<unknown>
+              docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
+      response:
+        type: IngestionResponse
+        status-code: 207
 
 types:
   IngestionEvent:
     discriminant: "type"
     union:
-      trace-create: TraceEvent
-      score-create: ScoreEvent
-      event-create: CreateEventEvent
-      generation-create: CreateGenerationEvent
-      generation-update: UpdateGenerationEvent
-      span-create: CreateSpanEvent
-      span-update: UpdateSpanEvent
-      sdk-log: SDKLogEvent
+      trace-create:
+        type: TraceEvent
+        docs: Creates a new trace. Upserts on id for updates if trace with id exists.
+      score-create:
+        type: ScoreEvent
+        docs: Creates a new score. Upserts on id for updates if score with id exists.
+      span-create:
+        type: CreateSpanEvent
+        docs: Creates a new span.
+      span-update:
+        type: UpdateSpanEvent
+        docs: Updates span based on id.
+      generation-create:
+        type: CreateGenerationEvent
+        docs: Creates a new generation.
+      generation-update:
+        type: UpdateGenerationEvent
+        docs: Updates a generation based on id.
+      event-create:
+        type: CreateEventEvent
+        docs: Creates an event.
+
+      sdk-log:
+        type: SDKLogEvent
+        docs: Langfuse SDKs only, used for debugging purposes.
 
       # both are legacy
       observation-create:
@@ -156,15 +185,66 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, the score name must equal the config name and scores must comply with the config's range and data type. For categorical scores, the value must map to a config category. Numeric scores might be constrained by the score config's max and min values
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
 
   BaseEvent:
     properties:
-      id: string
-      timestamp: string
-      metadata: unknown
+      id:
+        type: string
+        docs: UUID v4 that identifies the event
+      timestamp:
+        type: string
+        docs: "Datetime (ISO 8601) of event creation in client. Should be as close to actual event creation in client as possible, this timestamp will be used for ordering of events in future release. Resolution: milliseconds (required), microseconds (optimal)."
+      metadata:
+        type: optional<unknown>
+        docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
 
   TraceEvent:
     extends: BaseEvent

fern/apis/server/definition/metrics.yml

@@ -48,12 +48,25 @@ types:
       date: date
       countTraces: integer
       countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
       model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD

fern/apis/server/definition/models.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      method: POST
+      docs: Create a model
+      path: /models
+      request: CreateModelRequest
+      response: commons.Model
+    list:
+      method: GET
+      docs: Get all models
+      path: /models
+      request:
+        name: GetModelsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedModels
+    get:
+      method: GET
+      docs: Get a model
+      path: /models/{id}
+      path-parameters:
+        id: string
+      response: commons.Model
+    delete:
+      method: DELETE
+      docs: Delete a model. Cannot delete models managed by Langfuse. You can create your own definition with the same modelName to override the definition though.
+      path: /models/{id}
+      path-parameters:
+        id: string
+
+types:
+  PaginatedModels:
+    properties:
+      data: list<commons.Model>
+      meta: pagination.MetaResponse
+  CreateModelRequest:
+    properties:
+      modelName:
+        docs: "Name of the model definition. If multiple with the same name exist, they are applied in the following order: (1) custom over built-in, (2) newest according to startTime where model.startTime<observation.startTime"
+        type: string
+      matchPattern:
+        docs: "Regex pattern which matches this model definition to generation.model. Useful in case of fine-tuned models. If you want to exact match, use `(?i)^modelname$`"
+        type: string
+      startDate:
+        docs: Apply only to generations which are newer than this ISO date.
+        type: optional<date>
+      unit:
+        docs: Unit used by this model.
+        type: commons.ModelUsageUnit
+      inputPrice:
+        docs: Price (USD) per input unit
+        type: optional<double>
+      outputPrice:
+        docs: Price (USD) per output unit
+        type: optional<double>
+      totalPrice:
+        docs: Price (USD) per total units. Cannot be set if input or output price is set.
+        type: optional<double>
+      tokenizerId:
+        docs: Optional. Tokenizer to be applied to observations which match to this model. See docs for more details.
+        type: optional<string>
+      tokenizerConfig:
+        docs: Optional. Configuration for the selected tokenizer. Needs to be JSON. See docs for more details.
+        type: optional<unknown>

fern/apis/server/definition/prompts.yml

@@ -35,8 +35,12 @@ service:
           name: optional<string>
           label: optional<string>
           tag: optional<string>
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
 
       response: PromptMetaListResponse
 
@@ -70,14 +74,24 @@ types:
       name: string
       prompt: list<ChatMessage>
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   CreateTextPromptRequest:
     properties:
       name: string
       prompt: string
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   Prompt:
     union:
@@ -89,7 +103,12 @@ types:
       name: string
       version: integer
       config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
 
   ChatMessage:
     properties:

fern/apis/server/definition/score-configs.yml

@@ -0,0 +1,58 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      docs: Create a score configuration (config). Score configs are used to define the structure of scores
+      method: POST
+      path: /score-configs
+      request: CreateScoreConfigRequest
+      response: commons.ScoreConfig
+    get:
+      docs: Get all score configs
+      method: GET
+      path: /score-configs
+      request:
+        name: GetScoreConfigsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
+      response: ScoreConfigs
+    get-by-id:
+      docs: Get a score config
+      method: GET
+      path: /score-configs/{configId}
+      path-parameters:
+        configId:
+          type: string
+          docs: The unique langfuse identifier of a score config
+      response: commons.ScoreConfig
+types:
+  ScoreConfigs:
+    properties:
+      data: list<commons.ScoreConfig>
+      meta: pagination.MetaResponse
+  CreateScoreConfigRequest:
+    properties:
+      name: string
+      dataType: commons.ScoreDataType
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/score.yml

@@ -25,13 +25,17 @@ service:
           limit:
             type: optional<integer>
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId: optional<string>
-          name: optional<string>
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
-            type: commons.ScoreSource
+            type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
           operator:
             type: optional<string>
@@ -39,6 +43,15 @@ service:
           value:
             type: optional<double>
             docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
       response: Scores
     get-by-id:
       docs: Get a score
@@ -63,9 +76,55 @@ types:
       id: optional<string>
       traceId: string
       name: string
-      value: double
+      value:
+        type: commons.CreateScoreValue
+        docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      dataType:
+        type: optional<commons.ScoreDataType>
+        docs: The data type of the score. When passing a configId this field is inferred. Otherwise, this field must be passed or will default to numeric.
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. The unique langfuse identifier of a score config. When passing this field, the dataType and stringValue fields are automatically populated.
+    examples:
+      - value:
+          name: "novelty"
+          value: 0.9
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "consistency"
+          value: 1.2
+          dataType: "NUMERIC"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "accuracy"
+          value: 0.9
+          dataType: "NUMERIC"
+          configId: "9203-4567-89ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "toxicity"
+          value: "not toxic"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "correctness"
+          value: "partially correct"
+          dataType: "CATEGORICAL"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "hallucination"
+          value: 0
+          dataType: "BOOLEAN"
+          traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "helpfulness"
+          value: 1
+          dataType: "BOOLEAN"
+          configId: "1234-5678-90ab-cdef"
+          traceId: "cdef-1234-5678-90ab"
+
   Scores:
     properties:
       data: list<commons.Score>

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
       path: /sessions/{sessionId}
       path-parameters:

fern/apis/server/definition/trace.yml

@@ -30,6 +30,7 @@ service:
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
             docs: Retrieve only traces newer than this datetime (ISO 8601).

fern/apis/server/definition/utils/pagination.yml

@@ -1,4 +1,13 @@
 types:
+  # to be used as copy/paste template for query strings
+  # PaginationRequest:
+  #   properties:
+  #     page:
+  #       type: optional<integer>
+  #       docs: page number, starts at 1
+  #     limit:
+  #       type: optional<integer>
+  #       docs: limit of items per page
   MetaResponse:
     properties:
       page:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.21.0"
+  "version": "0.30.3"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.43.2",
+  "version": "2.60.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -17,26 +17,36 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "models:migrate": "turbo run models:migrate",
-    "release": "dotenv -e ../.env -- release-it"
+    "release": "dotenv -e ../.env -- release-it",
+    "prepare": "husky"
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
-    "dotenv-cli": "^7.4.1",
+    "braces": "3.0.3",
+    "dotenv-cli": "^7.4.2",
+    "husky": "^9.0.11",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.1",
+    "release-it": "^17.3.0",
     "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {
       "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
+      "tagName": "v${version}",
+      "commitArgs": [
+        "--no-verify"
+      ],
+      "pushArgs": [
+        "--no-verify"
+      ]
     },
     "plugins": {
       "@release-it/bumper": {
@@ -69,5 +79,8 @@
         "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
       }
     }
+  },
+  "dependencies": {
+    "newrelic": "^11.22.0"
   }
 }

packages/config-eslint/package.json

@@ -9,7 +9,7 @@
   ],
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",

packages/shared/package.json

@@ -17,9 +17,9 @@
       "import": "./dist/src/db.js",
       "require": "./dist/src/db.js"
     },
-    "./src/server/auth": {
-      "import": "./dist/src/server/auth.js",
-      "require": "./dist/src/server/auth.js"
+    "./src/server": {
+      "import": "./dist/src/server/index.js",
+      "require": "./dist/src/server/index.js"
     },
     "./encryption": {
       "import": "./dist/src/encryption/index.js",
@@ -44,14 +44,22 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.16",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@langchain/anthropic": "^0.1.21",
     "@langchain/core": "^0.1.61",
-    "@langchain/openai": "^0.0.28",
+    "@langchain/openai": "^0.0.33",
     "@prisma/client": "^5.13.0",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.36",
+    "langchain": "^0.2.6",
+    "lodash": "^4.17.21",
+    "next-auth": "^4.24.7",
+    "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -59,17 +67,18 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.5",
+    "@types/nodemailer": "^6.4.15",
+    "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
     "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.0.3",
+    "nodemon": "^3.1.3",
     "prettier": "^3.2.5",
     "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
@@ -78,5 +87,9 @@
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
     "vitest": "^1.5.3"
+  },
+  "peerDependencies": {
+    "@types/react": "^18.2.79",
+    "react": "^18.0.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -25,22 +25,17 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -103,6 +98,21 @@ export type AuditLog = {
     before: string | null;
     after: string | null;
 };
+export type BatchExport = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    user_id: string;
+    finished_at: Timestamp | null;
+    expires_at: Timestamp | null;
+    name: string;
+    status: string;
+    query: unknown;
+    format: string;
+    url: string | null;
+    log: string | null;
+};
 export type CronJobs = {
     name: string;
     last_run: Timestamp | null;
@@ -157,6 +167,7 @@ export type EvalTemplate = {
     version: number;
     prompt: string;
     model: string;
+    provider: string;
     model_params: unknown;
     vars: Generated<string[]>;
     output_schema: unknown;
@@ -204,8 +215,12 @@ export type LlmApiKeys = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     provider: string;
+    adapter: string;
     display_secret_key: string;
     secret_key: string;
+    base_url: string | null;
+    custom_models: Generated<string[]>;
+    with_default_models: Generated<boolean>;
     project_id: string;
 };
 export type MembershipInvitation = {
@@ -246,8 +261,10 @@ export type Observation = {
     status_message: string | null;
     version: string | null;
     created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
+    internal_model_id: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
     output: unknown | null;
@@ -258,6 +275,9 @@ export type Observation = {
     input_cost: string | null;
     output_cost: string | null;
     total_cost: string | null;
+    calculated_input_cost: string | null;
+    calculated_output_cost: string | null;
+    calculated_total_cost: string | null;
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
@@ -266,7 +286,7 @@ export type ObservationView = {
     trace_id: string | null;
     project_id: string;
     type: ObservationType;
-    start_time: Generated<Timestamp>;
+    start_time: Timestamp;
     end_time: Timestamp | null;
     name: string | null;
     metadata: unknown | null;
@@ -274,7 +294,8 @@ export type ObservationView = {
     level: Generated<ObservationLevel>;
     status_message: string | null;
     version: string | null;
-    created_at: Generated<Timestamp>;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     model: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
@@ -293,6 +314,7 @@ export type ObservationView = {
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
     latency: number | null;
+    time_to_first_token: number | null;
 };
 export type PosthogIntegration = {
     project_id: string;
@@ -302,14 +324,6 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -344,11 +358,30 @@ export type Score = {
     timestamp: Generated<Timestamp>;
     project_id: string;
     name: string;
-    value: number;
+    value: number | null;
     source: ScoreSource;
+    author_user_id: string | null;
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -406,6 +439,8 @@ export type TraceView = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Timestamp;
+    updated_at: Timestamp;
     duration: number | null;
 };
 export type User = {
@@ -429,6 +464,7 @@ export type DB = {
     Account: Account;
     api_keys: ApiKey;
     audit_logs: AuditLog;
+    batch_exports: BatchExport;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;
@@ -444,10 +480,10 @@ export type DB = {
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240513082205_observations_view_add_time_to_first_token/migration.sql

@@ -0,0 +1,61 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240522081254_scores_add_author_user_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "author_user_id" TEXT;

packages/shared/prisma/migrations/20240522095738_scores_add_author_user_id_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_author_user_id_idx" ON "scores"("author_user_id");

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/migrations/20240528214726_add_cursor_new_columns_observations/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214727_add_cursor_new_columns_scores/migration.sql

@@ -0,0 +1,4 @@
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214728_add_cursor_index_01/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "audit_logs_updated_at_idx" ON "audit_logs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_02/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_created_at_idx" ON "dataset_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_03/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_updated_at_idx" ON "dataset_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_04/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_created_at_idx" ON "dataset_run_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_05/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_updated_at_idx" ON "dataset_run_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_06/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_created_at_idx" ON "dataset_runs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_07/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_updated_at_idx" ON "dataset_runs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_08/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_created_at_idx" ON "datasets"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_09/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_updated_at_idx" ON "datasets"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_10/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_updated_at_idx" ON "job_executions"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_11/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_updated_at_idx" ON "observations"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_12/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_created_at_idx" ON "prompts"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_13/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_updated_at_idx" ON "prompts"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_14/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_created_at_idx" ON "score_configs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_15/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_updated_at_idx" ON "score_configs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_16/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_created_at_idx" ON "scores"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_17/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_updated_at_idx" ON "scores"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_18/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "trace_sessions_updated_at_idx" ON "trace_sessions"("updated_at");

packages/shared/prisma/migrations/20240603212024_dataset_items_add_index_source_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");

packages/shared/prisma/migrations/20240604133338_scores_add_index_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");

packages/shared/prisma/migrations/20240604133339_score_data_type_add_boolean/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';

packages/shared/prisma/migrations/20240606093356_drop_unused_pricings_table/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";

packages/shared/prisma/migrations/20240606133011_remove_trace_fkey_datasetrunitems/migration.sql

@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";

packages/shared/prisma/migrations/20240607090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,8 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+-- remove model if added by faulty migration, context: https://github.com/langfuse/langfuse/issues/2266
+DELETE FROM models WHERE id in ('clx30djsn0000w9mzebiv41we', 'clx30hkrx0000w9mz7lqi0ial');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240607212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240611105521_llm_api_keys_custom_endpoints/migration.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+  ALTER TABLE "llm_api_keys" 
+    ADD COLUMN "base_url" TEXT,
+    ADD COLUMN "adapter" TEXT,
+    ADD COLUMN "custom_models" TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+    ADD COLUMN "with_default_models" BOOLEAN NOT NULL DEFAULT true;
+
+  UPDATE "llm_api_keys"
+    SET "adapter" = "provider";
+
+  ALTER TABLE "llm_api_keys" 
+    ALTER COLUMN "adapter" SET NOT NULL;
+
+  ALTER TABLE "eval_templates"
+    ADD COLUMN "provider" TEXT;
+  
+  UPDATE "eval_templates"
+    SET "provider" = 'openai';
+  
+  ALTER TABLE "eval_templates"
+    ALTER COLUMN "provider" SET NOT NULL;
+
+COMMIT;

packages/shared/prisma/migrations/20240611113517_backfill_manual_score_configs/migration.sql

@@ -0,0 +1,51 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	md5(random()::text || clock_timestamp()::text || s.project_id::text)::uuid AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND config_id IS NULL
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;

packages/shared/prisma/migrations/20240612101858_add_index_observations_project_id_prompt_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_prompt_id_idx" ON "observations"("project_id", "prompt_id");

packages/shared/prisma/migrations/20240617094803_observations_remove_prompt_fk_constraint/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "observations" DROP CONSTRAINT "observations_prompt_id_fkey";

packages/shared/prisma/migrations/20240618134129_add_batch_exports_table/migration.sql

@@ -0,0 +1,27 @@
+-- CreateTable
+CREATE TABLE "batch_exports" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "finished_at" TIMESTAMP(3),
+    "expires_at" TIMESTAMP(3),
+    "name" TEXT NOT NULL,
+    "status" TEXT NOT NULL,
+    "query" JSONB NOT NULL,
+    "format" TEXT NOT NULL,
+    "url" TEXT,
+    "log" TEXT,
+
+    CONSTRAINT "batch_exports_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "batch_exports_project_id_user_id_idx" ON "batch_exports"("project_id", "user_id");
+
+-- CreateIndex
+CREATE INDEX "batch_exports_status_idx" ON "batch_exports"("status");
+
+-- AddForeignKey
+ALTER TABLE "batch_exports" ADD CONSTRAINT "batch_exports_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240618164950_drop_observations_parent_observation_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_parent_observation_id_idx";

packages/shared/prisma/migrations/20240618164951_drop_observations_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_updated_at_idx";

packages/shared/prisma/migrations/20240618164952_drop_scores_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "scores_updated_at_idx";

packages/shared/prisma/migrations/20240618164953_drop_traces_external_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_external_id_idx";

packages/shared/prisma/migrations/20240618164954_drop_traces_release_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_release_idx";

packages/shared/prisma/migrations/20240618164955_drop_traces_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_updated_at_idx";

packages/shared/prisma/migrations/20240618164956_create_traces_project_id_timestamp_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "traces_project_id_timestamp_idx" ON "traces"("project_id", "timestamp");

packages/shared/prisma/migrations/20240624133412_models_add_anthropic_3_5_sonnet/migration.sql

@@ -0,0 +1,15 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id
+)
+VALUES
+  -- add 3.5 sonnet model
+  ('clxt0n0m60000pumz1j5b7zsf', NULL, 'claude-3-5-sonnet-20240620', '(?i)^(claude-3-5-sonnet(-|@)?20240620)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude')

packages/shared/prisma/migrations/20240625103957_observations_add_calculated_cost_columns/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "calculated_input_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_output_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_total_cost" DECIMAL(65,30),
+ADD COLUMN     "internal_model_id" TEXT;

packages/shared/prisma/migrations/20240625103958_fix_model_match_gpt4_vision/migration.sql

@@ -0,0 +1,17 @@
+/*
+Previously, the pattern did not allow for model specifications like "1106" or any other 4-digit block. 
+This led to missing models on the generation-update call where the exact model name that was used was provided (including the 4-digit block).
+The new pattern allows for:
+
+- "gpt-4-vision-preview" as set on generation-create
+- "gpt-4-1106-vision-preview" as set on generation-update
+
+*/
+
+UPDATE
+	"models"
+SET
+	"match_pattern" = '(?i)^(gpt-4(-\d{4})?-vision-preview)$'
+WHERE
+	"id" = 'clrkvx5gp000108juaogs54ea'
+	AND "model_name" = 'gpt-4-turbo-vision';

packages/shared/prisma/migrations/20240703214747_models_anthropic_aws_bedrock/migration.sql

@@ -0,0 +1,9 @@
+-- Add AWS Bedrock model names for Anthropic models
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku-20240307|anthropic\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet-20240229|anthropic\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus-20240229|anthropic\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-5-sonnet-20240620|anthropic\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$' WHERE "id" = 'clxt0n0m60000pumz1j5b7zsf';

packages/shared/prisma/migrations/20240704103900_observations_view_read_from_calculated/migration.sql

@@ -0,0 +1,88 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240528214726_add_cursor_new_columns_observations and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.unit,
+    o.prompt_id,
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation