chore: release v2.58.0

something

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+{
+	"name": "langfuse dev",
+    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+        "version": "latest",
+        "moby": true
+        }
+    },
+    "forwardPorts": [3000],
+
+    "onCreateCommand": "npm install -g pnpm@8.15.5",
+    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+}

.env.dev.example

@@ -34,10 +34,6 @@ S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 S3_REGION=
 
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Set during docker build of application
 # Used to disable environment verification at build time
 # DOCKER_BUILD=1

.env.local.example

@@ -13,11 +13,6 @@ NEXTAUTH_URL="http://localhost:3000"
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"
 
-
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379

.env.prod.example

@@ -9,6 +9,8 @@
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
+# optional, set to true to disable automated database migrations on Docker start
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED= 
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
@@ -23,6 +25,7 @@ SALT="salt"
 # Use CSP headers to enforce HTTPS, optional
 LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
+
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
@@ -34,6 +37,9 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Enable experimental features, optional
 # LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
+# Disable automated database migrations on Docker start, optional
+# LANFUSE_WEB_MIGRATION_DISABLED=true
+
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
@@ -43,6 +49,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_GOOGLE_CLIENT_ID=
 # AUTH_GOOGLE_CLIENT_SECRET=
 # AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GOOGLE_ALLOWED_DOMAINS=langfuse.com,google.com # optional allowlist of workspace domains that can sign in via Google
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
@@ -62,7 +69,12 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
-
+# AUTH_CUSTOM_CLIENT_ID=
+# AUTH_CUSTOM_CLIENT_SECRET=
+# AUTH_CUSTOM_ISSUER=
+# AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -76,6 +88,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # S3_SECRET_ACCESS_KEY=
 # S3_BUCKET_NAME=
 # S3_REGION=
+# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
 
 # Exports are streamed to S3 in pages to avoid memory issues
 # The page size can be adjusted if needed to optimize performance
@@ -118,15 +131,13 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_SENTRY_ORG=
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
+# Used to determine the Sentry sample rate
+# LANGFUSE_TRACING_SAMPLE_RATE=
 
 # Cloudflare Turnstile
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 

.github/workflows/pipeline.yml

@@ -64,9 +64,11 @@ jobs:
 
   tests-web:
     runs-on: ubuntu-latest
+    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -97,6 +99,9 @@ jobs:
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
 
       - name: Seed DB
         run: |
@@ -114,9 +119,11 @@ jobs:
 
   tests-worker:
     runs-on: ubuntu-latest
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
+        postgres-version: [12, 15]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -147,6 +154,7 @@ jobs:
         run: |
           docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
 
       - name: Seed DB
         run: |
@@ -184,6 +192,7 @@ jobs:
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
 
       - name: Seed DB
@@ -250,6 +259,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
@@ -273,6 +290,9 @@ jobs:
           push: true
           tags: ${{ steps.meta-web.outputs.tags }}
           labels: ${{ steps.meta-web.outputs.labels }}
+          platforms: |
+            linux/amd64
+            linux/arm64
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
@@ -297,3 +317,6 @@ jobs:
           push: true
           tags: ${{ steps.meta-worker.outputs.tags }}
           labels: ${{ steps.meta-worker.outputs.labels }}
+          platforms: |
+            linux/amd64
+            linux/arm64

.github/workflows/porter_stack_web_eu_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - production
+name: Deploy to web
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: web
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_web_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to web (Staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_web_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to web (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: web
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -1,8 +1,8 @@
 "on":
   push:
     branches:
-    - main
-name: Deploy to worker
+    - production
+name: Deploy to worker (EU temp)
 jobs:
   porter-deploy:
     runs-on: ubuntu-latest
@@ -18,12 +18,12 @@ jobs:
       timeout-minutes: 30
       run: exec porter apply
       env:
-        PORTER_CLUSTER: "3959"
-        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
-        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_CLUSTER: "4060"
+        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
+        PORTER_HOST: https://dashboard.porter.run
         PORTER_PR_NUMBER: ${{ github.event.number }}
         PORTER_PROJECT: "12565"
         PORTER_REPO_NAME: ${{ github.event.repository.name }}
         PORTER_STACK_NAME: worker
         PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_staging.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - main
+name: Deploy to worker (staging)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4037"
+          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_worker_us.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (US)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4038"
+          PORTER_DEPLOYMENT_TARGET_ID: 260cd04e-5dc8-4b22-a291-ad4453f981f7
+          PORTER_HOST: https://dashboard.getporter.dev
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4038 }}

.github/workflows/porter_stack_worker_us_temp.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+      - production
+name: Deploy to worker (US temp)
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Setup porter
+        uses: porter-dev/setup-porter@v0.1.0
+      - name: Deploy stack
+        timeout-minutes: 30
+        run: exec porter apply
+        env:
+          PORTER_CLUSTER: "4054"
+          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_PROJECT: "12565"
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_STACK_NAME: worker
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/snyk.yml

@@ -0,0 +1,39 @@
+name: Snyk Container
+on: push
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker compose -f docker-compose.build.yml up -d
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-server
+          args: --file=web/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: web
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-worker
+          args: --file=worker/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: worker

.husky/pre-commit

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Commit will proceed
+    else
+        echo "Commit to $protected_branch branch has been canceled."
+        exit 1 # Commit will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the commit
+exit 0

.husky/pre-push

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to push to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Push will proceed
+    else
+        echo "Push to $protected_branch branch has been canceled."
+        exit 1 # Push will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the push
+exit 0

CONTRIBUTING.md

@@ -55,7 +55,7 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 > [!NOTE]
 > Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
-> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x.
 
 ```mermaid
 flowchart TB
@@ -151,6 +151,8 @@ Requirements
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
 - Docker to run the database locally
 
+**Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
+
 **Steps**
 
 1. Fork the repository and clone it locally

README.md

@@ -24,6 +24,8 @@
    </div>
    <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
+   <span>We're hiring. <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in Backend Engineering, Product Engineering, and Developer Relations.</span>
+   <br/>
    <br/>
    <div>
       <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>

docker-compose.build.yml

@@ -5,6 +5,8 @@ services:
     build:
       dockerfile: ./web/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis
@@ -30,6 +32,8 @@ services:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
+      args:
+        - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
     depends_on:
       - db
       - redis

docker-compose.dev.yml

@@ -1,5 +1,3 @@
-version: "3.5"
-
 services:
   redis:
     image: redis:7.2.4
@@ -10,7 +8,7 @@ services:
       - 6379:6379
 
   db:
-    image: postgres
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     command: ["postgres", "-c", "log_statement=all"]
     environment:

docker-compose.yml

@@ -1,8 +1,6 @@
-version: "3.5"
-
 services:
   langfuse-server:
-    image: ghcr.io/langfuse/langfuse:latest
+    image: langfuse/langfuse:2
     depends_on:
       db:
         condition: service_healthy

ee/src/sso/index.ts

@@ -11,6 +11,7 @@ import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { env } from "../env";
+import { CustomSSOProvider } from "@langfuse/shared/src/server";
 
 // Local cache for SSO configurations
 let cachedSsoConfigs: {
@@ -173,6 +174,15 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "custom")
+    return CustomSSOProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+      authorization: {
+        params: { scope: provider.authConfig.scope ?? "openid email profile" },
+      },
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -76,12 +76,27 @@ export const CognitoProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CustomProviderSchema = base.extend({
+  authProvider: z.literal("custom"),
+  authConfig: z
+    .object({
+      name: z.string(),
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      scope: z.string().nullish(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
 export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -90,6 +105,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   OktaProviderSchema,
   AzureAdProviderSchema,
   CognitoProviderSchema,
+  CustomProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/commons.yml

@@ -111,7 +111,24 @@ types:
       totalCost:
         docs: USD total cost, defaults to input+output
         type: optional<double>
-
+  ScoreConfig:
+    docs: Configuration for a score
+    properties:
+      id: string
+      name: string
+      createdAt: datetime
+      updatedAt: datetime
+      projectId: string
+      dataType: ScoreDataType
+      isArchived: boolean
+      minValue: optional<double>
+      maxValue: optional<double>
+      categories: optional<list<ConfigCategory>>
+      description: optional<string>
+  ConfigCategory:
+    properties:
+      value: double
+      label: string
   Score:
     properties:
       id: string
@@ -122,7 +139,7 @@ types:
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
-  DatasetCore:
+  Dataset:
     properties:
       id: string
       name: string
@@ -131,21 +148,6 @@ types:
       projectId: string
       createdAt: datetime
       updatedAt: datetime
-  DatasetWithReferences: # GET datasets
-    extends: DatasetCore
-    properties:
-      items:
-        type: list<string>
-        docs: list of dataset item ids
-      runs:
-        type: list<string>
-        docs: list of dataset run names
-  Dataset: # GET datasets/[name]
-    docs: Dataset including all items
-    extends: DatasetCore
-    properties:
-      items: list<DatasetItem>
-      runs: list<string>
   DatasetItem:
     properties:
       id: string
@@ -179,6 +181,9 @@ types:
       datasetName: string
       createdAt: datetime
       updatedAt: datetime
+  DatasetRunWithItems:
+    extends: DatasetRun
+    properties:
       datasetRunItems: list<DatasetRunItem>
 
   # Utilities
@@ -212,6 +217,11 @@ types:
       - ANNOTATION
       - API
       - EVAL
+  ScoreDataType:
+    enum:
+      - NUMERIC
+      - BOOLEAN
+      - CATEGORICAL
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
@@ -19,6 +20,23 @@ service:
         id:
           type: string
       response: commons.DatasetItem
+    list:
+      docs: Get dataset items
+      method: GET
+      path: /dataset-items
+      request:
+        name: GetDatasetItemsRequest
+        query-parameters:
+          datasetName: optional<string>
+          sourceTraceId: optional<string>
+          sourceObservationId: optional<string>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetItems
 
 types:
   CreateDatasetItemRequest:
@@ -31,7 +49,11 @@ types:
       sourceObservationId: optional<string>
       id:
         type: optional<string>
-        docs: Dataset items are upserted on their id
+        docs: Dataset items are upserted on their id. Id needs to be globally unique and cannot be reused across datasets.
       status:
         type: optional<commons.DatasetStatus>
         docs: Defaults to ACTIVE for newly created items
+  PaginatedDatasetItems:
+    properties:
+      data: list<commons.DatasetItem>
+      meta: pagination.MetaResponse

fern/apis/server/definition/datasets.yml

@@ -9,42 +9,66 @@ service:
     list:
       method: GET
       docs: Get all datasets
-      path: /datasets
+      path: /v2/datasets
       request:
         name: GetDatasetsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
       response: PaginatedDatasets
     get:
       method: GET
       docs: Get a dataset and its items
-      path: /datasets/{datasetName}
+      path: /v2/datasets/{datasetName}
       path-parameters:
         datasetName: string
       response: commons.Dataset
     create:
       method: POST
       docs: Create a dataset
-      path: /datasets
+      path: /v2/datasets
       request: CreateDatasetRequest
       response: commons.Dataset
-    getRuns:
+    getRun:
       method: GET
       docs: Get a dataset run and its items
       path: /datasets/{datasetName}/runs/{runName}
       path-parameters:
         datasetName: string
         runName: string
-      response: commons.DatasetRun
+      response: commons.DatasetRunWithItems
+    getRuns:
+      method: GET
+      docs: Get a dataset runs
+      path: /datasets/{datasetName}/runs
+      path-parameters:
+        datasetName: string
+      request:
+        name: GetDatasetRunsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedDatasetRuns
 
 types:
   PaginatedDatasets:
     properties:
-      data: list<commons.DatasetWithReferences>
+      data: list<commons.Dataset>
       meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
       description: optional<string>
       metadata: optional<unknown>
+  PaginatedDatasetRuns:
+    properties:
+      data: list<commons.DatasetRun>
+      meta: pagination.MetaResponse

fern/apis/server/definition/ingestion.yml

@@ -7,28 +7,57 @@ service:
   base-path: /api/public
   endpoints:
     batch:
-      docs: Batched ingestion for Langfuse Tracing
+      docs: |
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
+
+        Notes:
+        - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
+        - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
       path: /ingestion
       request:
         name: IngestionRequest
         body:
           properties:
-            batch: list<IngestionEvent>
-      response: IngestionResponse # will be reportet as 200 response, but endpoint returns 207
+            batch:
+              type: list<IngestionEvent>
+              docs: "Batch of tracing events to be ingested. Discriminated by attribute `type`."
+            metadata:
+              type: optional<unknown>
+              docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
+      response:
+        type: IngestionResponse
+        status-code: 207
 
 types:
   IngestionEvent:
     discriminant: "type"
     union:
-      trace-create: TraceEvent
-      score-create: ScoreEvent
-      event-create: CreateEventEvent
-      generation-create: CreateGenerationEvent
-      generation-update: UpdateGenerationEvent
-      span-create: CreateSpanEvent
-      span-update: UpdateSpanEvent
-      sdk-log: SDKLogEvent
+      trace-create:
+        type: TraceEvent
+        docs: Creates a new trace. Upserts on id for updates if trace with id exists.
+      score-create:
+        type: ScoreEvent
+        docs: Creates a new score. Upserts on id for updates if score with id exists.
+      span-create:
+        type: CreateSpanEvent
+        docs: Creates a new span.
+      span-update:
+        type: UpdateSpanEvent
+        docs: Updates span based on id.
+      generation-create:
+        type: CreateGenerationEvent
+        docs: Creates a new generation.
+      generation-update:
+        type: UpdateGenerationEvent
+        docs: Updates a generation based on id.
+      event-create:
+        type: CreateEventEvent
+        docs: Creates an event.
+
+      sdk-log:
+        type: SDKLogEvent
+        docs: Langfuse SDKs only, used for debugging purposes.
 
       # both are legacy
       observation-create:
@@ -162,9 +191,15 @@ types:
 
   BaseEvent:
     properties:
-      id: string
-      timestamp: string
-      metadata: unknown
+      id:
+        type: string
+        docs: UUID v4 that identifies the event
+      timestamp:
+        type: string
+        docs: "Datetime (ISO 8601) of event creation in client. Should be as close to actual event creation in client as possible, this timestamp will be used for ordering of events in future release. Resolution: milliseconds (required), microseconds (optimal)."
+      metadata:
+        type: optional<unknown>
+        docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
 
   TraceEvent:
     extends: BaseEvent

fern/apis/server/definition/prompts.yml

@@ -35,8 +35,12 @@ service:
           name: optional<string>
           label: optional<string>
           tag: optional<string>
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
 
       response: PromptMetaListResponse
 

fern/apis/server/definition/score-configs.yml

@@ -0,0 +1,36 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get all score configs
+      method: GET
+      path: /score-configs
+      request:
+        name: GetScoreConfigsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+      response: ScoreConfigs
+    get-by-id:
+      docs: Get a score config
+      method: GET
+      path: /score-configs/{configId}
+      path-parameters:
+        configId:
+          type: string
+          docs: The unique langfuse identifier of a score config
+      response: commons.ScoreConfig
+types:
+  ScoreConfigs:
+    properties:
+      data: list<commons.ScoreConfig>
+      meta: pagination.MetaResponse

fern/apis/server/definition/score.yml

@@ -31,7 +31,7 @@ service:
             type: optional<datetime>
             docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
-            type: commons.ScoreSource
+            type: optional<commons.ScoreSource>
             docs: Retrieve only scores from a specific source.
           operator:
             type: optional<string>
@@ -39,6 +39,9 @@ service:
           value:
             type: optional<double>
             docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
       response: Scores
     get-by-id:
       docs: Get a score

fern/apis/server/definition/utils/pagination.yml

@@ -1,4 +1,13 @@
 types:
+  # to be used as copy/paste template for query strings
+  # PaginationRequest:
+  #   properties:
+  #     page:
+  #       type: optional<integer>
+  #       docs: page number, starts at 1
+  #     limit:
+  #       type: optional<integer>
+  #       docs: limit of items per page
   MetaResponse:
     properties:
       page:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.21.0"
+  "version": "0.30.3"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.49.2",
+  "version": "2.58.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -17,26 +17,36 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "models:migrate": "turbo run models:migrate",
-    "release": "dotenv -e ../.env -- release-it"
+    "release": "dotenv -e ../.env -- release-it",
+    "prepare": "husky"
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
+    "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
+    "husky": "^9.0.11",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.1",
+    "release-it": "^17.3.0",
     "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {
       "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
+      "tagName": "v${version}",
+      "commitArgs": [
+        "--no-verify"
+      ],
+      "pushArgs": [
+        "--no-verify"
+      ]
     },
     "plugins": {
       "@release-it/bumper": {

packages/shared/package.json

@@ -17,9 +17,9 @@
       "import": "./dist/src/db.js",
       "require": "./dist/src/db.js"
     },
-    "./src/server/auth": {
-      "import": "./dist/src/server/auth.js",
-      "require": "./dist/src/server/auth.js"
+    "./src/server": {
+      "import": "./dist/src/server/index.js",
+      "require": "./dist/src/server/index.js"
     },
     "./encryption": {
       "import": "./dist/src/encryption/index.js",
@@ -44,15 +44,22 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@langchain/anthropic": "^0.1.21",
     "@langchain/core": "^0.1.61",
     "@langchain/openai": "^0.0.33",
     "@prisma/client": "^5.13.0",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.36",
+    "langchain": "^0.2.6",
     "lodash": "^4.17.21",
+    "next-auth": "^4.24.7",
+    "nodemailer": "^6.9.13",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -60,8 +67,9 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.11.29",
+    "@types/nodemailer": "^6.4.15",
     "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
@@ -70,7 +78,7 @@
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
     "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.0.3",
+    "nodemon": "^3.1.3",
     "prettier": "^3.2.5",
     "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
@@ -79,5 +87,9 @@
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
     "vitest": "^1.5.3"
+  },
+  "peerDependencies": {
+    "@types/react": "^18.2.79",
+    "react": "^18.0.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -98,6 +98,21 @@ export type AuditLog = {
     before: string | null;
     after: string | null;
 };
+export type BatchExport = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    user_id: string;
+    finished_at: Timestamp | null;
+    expires_at: Timestamp | null;
+    name: string;
+    status: string;
+    query: unknown;
+    format: string;
+    url: string | null;
+    log: string | null;
+};
 export type CronJobs = {
     name: string;
     last_run: Timestamp | null;
@@ -152,6 +167,7 @@ export type EvalTemplate = {
     version: number;
     prompt: string;
     model: string;
+    provider: string;
     model_params: unknown;
     vars: Generated<string[]>;
     output_schema: unknown;
@@ -199,8 +215,12 @@ export type LlmApiKeys = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     provider: string;
+    adapter: string;
     display_secret_key: string;
     secret_key: string;
+    base_url: string | null;
+    custom_models: Generated<string[]>;
+    with_default_models: Generated<boolean>;
     project_id: string;
 };
 export type MembershipInvitation = {
@@ -244,6 +264,7 @@ export type Observation = {
     updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
+    internal_model_id: string | null;
     modelParameters: unknown | null;
     input: unknown | null;
     output: unknown | null;
@@ -254,6 +275,9 @@ export type Observation = {
     input_cost: string | null;
     output_cost: string | null;
     total_cost: string | null;
+    calculated_input_cost: string | null;
+    calculated_output_cost: string | null;
+    calculated_total_cost: string | null;
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
@@ -437,6 +461,7 @@ export type DB = {
     Account: Account;
     api_keys: ApiKey;
     audit_logs: AuditLog;
+    batch_exports: BatchExport;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;

packages/shared/prisma/migrations/20240530212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -1,7 +0,0 @@
--- Google Vertex uses @ to separate model name and version
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
-
-UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -1,5 +0,0 @@
--- does not include pricing yet, will be added as soon as it is calculated at ingestion time
-
-INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
-
-INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240607090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,8 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+-- remove model if added by faulty migration, context: https://github.com/langfuse/langfuse/issues/2266
+DELETE FROM models WHERE id in ('clx30djsn0000w9mzebiv41we', 'clx30hkrx0000w9mz7lqi0ial');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240607212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240611105521_llm_api_keys_custom_endpoints/migration.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+  ALTER TABLE "llm_api_keys" 
+    ADD COLUMN "base_url" TEXT,
+    ADD COLUMN "adapter" TEXT,
+    ADD COLUMN "custom_models" TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+    ADD COLUMN "with_default_models" BOOLEAN NOT NULL DEFAULT true;
+
+  UPDATE "llm_api_keys"
+    SET "adapter" = "provider";
+
+  ALTER TABLE "llm_api_keys" 
+    ALTER COLUMN "adapter" SET NOT NULL;
+
+  ALTER TABLE "eval_templates"
+    ADD COLUMN "provider" TEXT;
+  
+  UPDATE "eval_templates"
+    SET "provider" = 'openai';
+  
+  ALTER TABLE "eval_templates"
+    ALTER COLUMN "provider" SET NOT NULL;
+
+COMMIT;

packages/shared/prisma/migrations/20240611113517_backfill_manual_score_configs/migration.sql

@@ -10,7 +10,7 @@ INSERT INTO score_configs (id,
 		max_value,
 		description)
 SELECT
-	gen_random_uuid () AS id,
+	md5(random()::text || clock_timestamp()::text || s.project_id::text)::uuid AS id,
 	s.project_id,
 	'manual-score',
 	'NUMERIC',
@@ -24,6 +24,7 @@ FROM ( SELECT DISTINCT
 		scores
 	WHERE
 		name = 'manual-score'
+		AND config_id IS NULL
 		AND source = 'ANNOTATION') s
 WHERE
 	NOT EXISTS (
@@ -47,4 +48,4 @@ FROM
 WHERE
 	scores.project_id = pc.project_id
 	AND scores.name = 'manual-score';
-COMMIT;
+COMMIT;

packages/shared/prisma/migrations/20240612101858_add_index_observations_project_id_prompt_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_prompt_id_idx" ON "observations"("project_id", "prompt_id");

packages/shared/prisma/migrations/20240617094803_observations_remove_prompt_fk_constraint/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "observations" DROP CONSTRAINT "observations_prompt_id_fkey";

packages/shared/prisma/migrations/20240618134129_add_batch_exports_table/migration.sql

@@ -0,0 +1,27 @@
+-- CreateTable
+CREATE TABLE "batch_exports" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "finished_at" TIMESTAMP(3),
+    "expires_at" TIMESTAMP(3),
+    "name" TEXT NOT NULL,
+    "status" TEXT NOT NULL,
+    "query" JSONB NOT NULL,
+    "format" TEXT NOT NULL,
+    "url" TEXT,
+    "log" TEXT,
+
+    CONSTRAINT "batch_exports_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "batch_exports_project_id_user_id_idx" ON "batch_exports"("project_id", "user_id");
+
+-- CreateIndex
+CREATE INDEX "batch_exports_status_idx" ON "batch_exports"("status");
+
+-- AddForeignKey
+ALTER TABLE "batch_exports" ADD CONSTRAINT "batch_exports_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240618164950_drop_observations_parent_observation_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_parent_observation_id_idx";

packages/shared/prisma/migrations/20240618164951_drop_observations_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "observations_updated_at_idx";

packages/shared/prisma/migrations/20240618164952_drop_scores_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "scores_updated_at_idx";

packages/shared/prisma/migrations/20240618164953_drop_traces_external_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_external_id_idx";

packages/shared/prisma/migrations/20240618164954_drop_traces_release_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_release_idx";

packages/shared/prisma/migrations/20240618164955_drop_traces_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "traces_updated_at_idx";

packages/shared/prisma/migrations/20240618164956_create_traces_project_id_timestamp_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "traces_project_id_timestamp_idx" ON "traces"("project_id", "timestamp");

packages/shared/prisma/migrations/20240624133412_models_add_anthropic_3_5_sonnet/migration.sql

@@ -0,0 +1,15 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id
+)
+VALUES
+  -- add 3.5 sonnet model
+  ('clxt0n0m60000pumz1j5b7zsf', NULL, 'claude-3-5-sonnet-20240620', '(?i)^(claude-3-5-sonnet(-|@)?20240620)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude')

packages/shared/prisma/migrations/20240625103957_observations_add_calculated_cost_columns/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "calculated_input_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_output_cost" DECIMAL(65,30),
+ADD COLUMN     "calculated_total_cost" DECIMAL(65,30),
+ADD COLUMN     "internal_model_id" TEXT;

packages/shared/prisma/migrations/20240625103958_fix_model_match_gpt4_vision/migration.sql

@@ -0,0 +1,17 @@
+/*
+Previously, the pattern did not allow for model specifications like "1106" or any other 4-digit block. 
+This led to missing models on the generation-update call where the exact model name that was used was provided (including the 4-digit block).
+The new pattern allows for:
+
+- "gpt-4-vision-preview" as set on generation-create
+- "gpt-4-1106-vision-preview" as set on generation-update
+
+*/
+
+UPDATE
+	"models"
+SET
+	"match_pattern" = '(?i)^(gpt-4(-\d{4})?-vision-preview)$'
+WHERE
+	"id" = 'clrkvx5gp000108juaogs54ea'
+	AND "model_name" = 'gpt-4-turbo-vision';

packages/shared/prisma/schema.prisma

@@ -115,6 +115,7 @@ model Project {
     PosthogIntegration PosthogIntegration[]
     Score              Score[]
     scoreConfig        ScoreConfig[]
+    BatchExport        BatchExport[]
 
     @@map("projects")
 }
@@ -144,9 +145,13 @@ model LlmApiKeys {
     createdAt DateTime @default(now()) @map("created_at")
     updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
 
-    provider         String
-    displaySecretKey String @map("display_secret_key")
-    secretKey        String @map("secret_key")
+    provider          String
+    adapter           String // This controls the interface that is used to connect with the LLM, e.g. 'openai' or 'anthropic'
+    displaySecretKey  String   @map("display_secret_key")
+    secretKey         String   @map("secret_key")
+    baseURL           String?  @map("base_url")
+    customModels      String[] @default([]) @map("custom_models")
+    withDefaultModels Boolean  @default(true) @map("with_default_models")
 
     projectId String  @map("project_id")
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -237,14 +242,12 @@ model Trace {
     JobExecution JobExecution[]
 
     @@index([projectId])
+    @@index([projectId, timestamp])
     @@index([sessionId])
     @@index([name])
     @@index([userId])
     @@index([id, userId])
-    @@index([externalId])
     @@index(timestamp)
-    @@index(release)
-    @@index(updatedAt)
     @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
@@ -296,24 +299,33 @@ model Observation {
     updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
-    model               String?
-    internalModel       String?       @map("internal_model")
-    modelParameters     Json?
-    input               Json?
-    output              Json?
-    promptTokens        Int           @default(0) @map("prompt_tokens")
-    completionTokens    Int           @default(0) @map("completion_tokens")
-    totalTokens         Int           @default(0) @map("total_tokens")
-    unit                String?
-    inputCost           Decimal?      @map("input_cost")
-    outputCost          Decimal?      @map("output_cost")
-    totalCost           Decimal?      @map("total_cost")
+    model           String?
+    internalModel   String? @map("internal_model")
+    internalModelId String? @map("internal_model_id")
+
+    modelParameters  Json?
+    input            Json?
+    output           Json?
+    promptTokens     Int     @default(0) @map("prompt_tokens")
+    completionTokens Int     @default(0) @map("completion_tokens")
+    totalTokens      Int     @default(0) @map("total_tokens")
+    unit             String?
+
+    // User provided cost at ingestion
+    inputCost  Decimal? @map("input_cost")
+    outputCost Decimal? @map("output_cost")
+    totalCost  Decimal? @map("total_cost")
+
+    // Calculated cost
+    calculatedInputCost  Decimal? @map("calculated_input_cost")
+    calculatedOutputCost Decimal? @map("calculated_output_cost")
+    calculatedTotalCost  Decimal? @map("calculated_total_cost")
+
     completionStartTime DateTime?     @map("completion_start_time")
     project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
 
-    promptId String? @map("prompt_id")
-    prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
+    promptId String? @map("prompt_id") // no fk constraint, prompt can be deleted
 
     @@unique([id, projectId])
     @@index([projectId, internalModel, startTime, unit])
@@ -324,11 +336,10 @@ model Observation {
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
-    @@index(updatedAt)
     @@index(projectId)
-    @@index(parentObservationId)
     @@index(model)
     @@index(internalModel)
+    @@index([projectId, promptId])
     @@index(promptId)
     @@index([projectId, startTime, type])
     @@map("observations")
@@ -424,7 +435,6 @@ model Score {
     @@index([observationId], type: Hash)
     @@index([source])
     @@index([createdAt])
-    @@index([updatedAt])
     @@map("scores")
 }
 
@@ -585,15 +595,14 @@ model Prompt {
 
     createdBy String @map("created_by")
 
-    prompt      Json
-    name        String
-    version     Int
-    type        String        @default("text")
-    isActive    Boolean?      @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-    config      Json          @default("{}")
-    tags        String[]      @default([])
-    labels      String[]      @default([])
-    Observation Observation[]
+    prompt   Json
+    name     String
+    version  Int
+    type     String   @default("text")
+    isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+    config   Json     @default("{}")
+    tags     String[] @default([])
+    labels   String[] @default([])
 
     @@unique([projectId, name, version])
     @@index([projectId, name, version])
@@ -664,6 +673,7 @@ model EvalTemplate {
     version          Int
     prompt           String
     model            String
+    provider         String
     modelParams      Json               @map("model_params")
     vars             String[]           @default([])
     outputSchema     Json               @map("output_schema")
@@ -777,3 +787,27 @@ model PosthogIntegration {
     @@index([projectId])
     @@map("posthog_integrations")
 }
+
+model BatchExport {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String  @map("user_id")
+
+    finishedAt DateTime? @map("finished_at")
+    expiresAt  DateTime? @map("expires_at")
+
+    name   String
+    status String
+    query  Json
+    format String
+    url    String?
+    log    String?
+
+    @@index([projectId, userId])
+    @@index([status])
+    @@map("batch_exports")
+}

packages/shared/prisma/seed.ts

@@ -12,7 +12,7 @@ import { parseArgs } from "node:util";
 import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
-import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server";
 import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
@@ -184,6 +184,7 @@ async function main() {
           secretKey: encrypt(OPENAI_API_KEY),
           displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
           provider: "openai",
+          adapter: "openai",
         },
       });
     } else {
@@ -209,6 +210,7 @@ async function main() {
           "Please evaluate the toxicity of the following text {{input}} {{output}}",
         model: "gpt-3.5-turbo",
         vars: ["input", "output"],
+        provider: "openai",
         outputSchema: {
           score: "provide a score between 0 and 1",
           reasoning: "one sentence reasoning for the score",
@@ -375,11 +377,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Sessions ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -393,11 +394,10 @@ async function uploadObjects(
     );
   });
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Traces ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -411,11 +411,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Observations ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -429,11 +428,10 @@ async function uploadObjects(
   });
 
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Events ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 
@@ -446,11 +444,10 @@ async function uploadObjects(
     );
   });
   for (let i = 0; i < promises.length; i++) {
-    process.stdout.clearLine(0);
-    process.stdout.cursorTo(0);
-    process.stdout.write(
-      `Seeding of Scores ${(i / promises.length) * 100}% complete`
-    );
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      console.log(
+        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`
+      );
     await promises[i];
   }
 }

packages/shared/scripts/cleanup.sql

@@ -0,0 +1,7 @@
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '_prisma_migrations') THEN
+  DELETE FROM _prisma_migrations
+  WHERE migration_name IN ('20240606090858_pricings_add_latest_gemini_models', '20240530212419_model_price_anthropic_via_google_vertex', '20240604133340_backfill_manual_scores');
+ END IF;
+END $$;

packages/shared/src/errors/InvalidRequestError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class InvalidRequestError extends BaseError {
+  constructor(description = "Invalid Request Error") {
+    super("InvalidRequestError", 400, description, true);
+  }
+}

packages/shared/src/errors/ValidationError.ts

@@ -1,7 +0,0 @@
-import { BaseError } from "./BaseError";
-
-export class ValidationError extends BaseError {
-  constructor(description = "Validation Error") {
-    super("ValidationError", 400, description, true);
-  }
-}

packages/shared/src/errors/index.ts

@@ -1,6 +1,6 @@
 export { BaseError } from "./BaseError";
 export { LangfuseNotFoundError } from "./NotFoundError";
-export { ValidationError } from "./ValidationError";
+export { InvalidRequestError } from "./InvalidRequestError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";

packages/shared/src/features/annotation/types.ts

@@ -0,0 +1,15 @@
+import z from "zod";
+import { ScoreConfig } from "../../db";
+
+const configCategory = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+export const categoriesList = z.array(configCategory);
+
+export type ConfigCategory = z.infer<typeof configCategory>;
+
+export type CastedConfig = Omit<ScoreConfig, "categories"> & {
+  categories: ConfigCategory[] | null;
+};

packages/shared/src/features/batchExport/types.ts

@@ -0,0 +1,73 @@
+import z from "zod";
+
+import { BatchExport } from "@prisma/client";
+
+import { singleFilter } from "../../interfaces/filters";
+import { orderBy } from "../../interfaces/orderBy";
+
+export enum BatchExportStatus {
+  QUEUED = "QUEUED",
+  PROCESSING = "PROCESSING",
+  COMPLETED = "COMPLETED",
+  FAILED = "FAILED",
+}
+
+export enum BatchExportFileFormat {
+  JSON = "JSON",
+  CSV = "CSV",
+}
+
+export enum BatchExportTableName {
+  Sessions = "sessions",
+}
+
+export const exportOptions: Record<
+  BatchExportFileFormat,
+  {
+    label: string;
+    extension: string;
+    fileType: string;
+  }
+> = {
+  CSV: { label: "CSV", extension: "csv", fileType: "text/csv" },
+  JSON: { label: "JSON", extension: "json", fileType: "application/json" },
+} as const;
+
+export const BatchExportQuerySchema = z.object({
+  tableName: z.nativeEnum(BatchExportTableName),
+  filter: z.array(singleFilter).nullable(),
+  orderBy,
+  limit: z.number().optional(),
+  page: z.number().optional(),
+});
+
+export type BatchExportQueryType = z.infer<typeof BatchExportQuerySchema>;
+
+export const CreateBatchExportSchema = z.object({
+  projectId: z.string(),
+  name: z.string(),
+  query: BatchExportQuerySchema,
+  format: z.nativeEnum(BatchExportFileFormat),
+});
+
+export const BatchExportSchema = z.object({
+  id: z.string(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  projectId: z.string(),
+  userId: z.string(),
+  finishedAt: z.date().nullable(),
+  expiresAt: z.date().nullable(),
+  name: z.string(),
+  status: z.nativeEnum(BatchExportStatus),
+  query: BatchExportQuerySchema,
+  format: z.nativeEnum(BatchExportFileFormat),
+  url: z.string().nullable(),
+  log: z.string().nullable(),
+});
+
+// Ensure that zod type matches the Prisma type
+export type BatchExportType =
+  z.infer<typeof BatchExportSchema> extends BatchExport
+    ? z.infer<typeof BatchExportSchema>
+    : never;

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, UIModelParams } from "../..";
+import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,38 +77,15 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels: UIModelParams[] = [
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-3.5-turbo", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4-turbo-preview", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4o", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-] as const;
-
-export const EvalModelNames = z.enum([
-  "gpt-3.5-turbo",
-  "gpt-4-turbo-preview",
-  "gpt-4o",
-]);
+export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
+  [LLMAdapter.OpenAI]: [
+    "gpt-4o",
+    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+  ] as const,
+  [LLMAdapter.Anthropic]: [] as const,
+  [LLMAdapter.Azure]: [] as const,
+};
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/filterToPrisma.ts

@@ -1,8 +1,5 @@
 import { Prisma } from "@prisma/client";
-import {
-  ColumnDefinition,
-  type TableNames,
-} from "./interfaces/tableDefinition";
+import { ColumnDefinition, type TableNames } from "./tableDefinitions";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
 

packages/shared/src/index.ts

@@ -1,11 +1,12 @@
 export * from "./constants";
+export * from "./queries";
 export * from "./queues";
-export * from "./interfaces/exportTypes";
 export * from "./interfaces/filters";
 export * from "./interfaces/orderBy";
-export * from "./interfaces/tableDefinition";
+export * from "./tableDefinitions";
 export * from "./types";
 export * from "./filterToPrisma";
+export * from "./orderByToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
 export * from "./observationsTable";
@@ -20,6 +21,10 @@ export * from "./server/llm/fetchLLMCompletion";
 
 // evals
 export * from "./features/evals/types";
+export * from "./features/batchExport/types";
+
+// annotation
+export * from "./features/annotation/types";
 
 // export db types only
 export * from "@prisma/client";

packages/shared/src/interfaces/exportTypes.ts

@@ -1,19 +0,0 @@
-export const exportFileFormats = ["CSV", "JSON", "OPENAI-JSONL"] as const;
-export type ExportFileFormats = (typeof exportFileFormats)[number];
-
-export const exportOptions: Record<
-  ExportFileFormats,
-  {
-    label: string;
-    extension: string;
-    fileType: string;
-  }
-> = {
-  CSV: { label: "CSV", extension: "csv", fileType: "text/csv" },
-  JSON: { label: "JSON", extension: "json", fileType: "application/json" },
-  "OPENAI-JSONL": {
-    label: "OpenAI JSONL (fine-tuning)",
-    extension: "jsonl",
-    fileType: "application/json",
-  },
-} as const;

packages/shared/src/interfaces/orderBy.ts

@@ -6,3 +6,5 @@ export const orderBy = z
     order: z.enum(["ASC", "DESC"]),
   })
   .nullable();
+
+export type OrderByState = z.infer<typeof orderBy>;

packages/shared/src/observationsTable.ts

@@ -2,7 +2,7 @@ import { ObservationLevel } from "@prisma/client";
 import {
   type OptionsDefinition,
   type ColumnDefinition,
-} from "./interfaces/tableDefinition";
+} from "./tableDefinitions/types";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [

packages/shared/src/orderByToPrisma.ts

@@ -1,8 +1,10 @@
-import { type OrderByState } from "@/src/features/orderBy/types";
-import { type ColumnDefinition } from "@langfuse/shared";
-import { Prisma } from "@langfuse/shared";
 import { z } from "zod";
 
+import { Prisma } from "@prisma/client";
+
+import type { ColumnDefinition } from "./tableDefinitions/types";
+import type { OrderByState } from "./interfaces/orderBy";
+
 /**
  * Convert orderBy to SQL ORDER BY clause
  * @param orderBy orderBy state
@@ -11,7 +13,7 @@ import { z } from "zod";
  */
 export function orderByToPrismaSql(
   orderBy: OrderByState,
-  tableColumns: ColumnDefinition[],
+  tableColumns: ColumnDefinition[]
 ): Prisma.Sql {
   if (!orderBy) {
     return Prisma.sql`ORDER BY t.timestamp DESC NULLS LAST`;
@@ -20,7 +22,7 @@ export function orderByToPrismaSql(
   const col = tableColumns.find(
     // TODO: Only use id instead of name.
     // It's less error-prone & decouples data fetching from the human-readable UI labels
-    (c) => c.name === orderBy.column || c.id === orderBy.column,
+    (c) => c.name === orderBy.column || c.id === orderBy.column
   );
 
   if (!col) {
@@ -38,6 +40,6 @@ export function orderByToPrismaSql(
 
   // Both column and order are safe, can use raw SQL
   return Prisma.raw(
-    `ORDER BY ${col.internal} ${order.data} ${orderBy.order === "DESC" ? "NULLS LAST" : "NULLS FIRST"}`,
+    `ORDER BY ${col.internal} ${order.data} ${orderBy.order === "DESC" ? "NULLS LAST" : "NULLS FIRST"}`
   );
 }

packages/shared/src/queries/getSessionTableSQL.ts

@@ -0,0 +1,88 @@
+import { z } from "zod";
+
+import { Prisma } from "@prisma/client";
+import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
+import { singleFilter } from "../interfaces/filters";
+import { orderBy } from "../interfaces/orderBy";
+import { orderByToPrismaSql } from "../orderByToPrisma";
+import { sessionsViewCols } from "../tableDefinitions/index";
+
+const GetSessionTableSQLParamsSchema = z.object({
+  projectId: z.string(),
+  filter: z.array(singleFilter).nullable(),
+  orderBy: orderBy,
+  page: z.number(),
+  limit: z.number(),
+});
+type GetSessionTableSQLParams = z.infer<typeof GetSessionTableSQLParamsSchema>;
+
+export const getSessionTableSQL = (
+  params: GetSessionTableSQLParams
+): Prisma.Sql => {
+  const { projectId, filter, orderBy, page, limit } =
+    GetSessionTableSQLParamsSchema.parse(params);
+
+  const filterCondition = tableColumnsToSqlFilterAndPrefix(
+    filter ?? [],
+    sessionsViewCols,
+    "sessions"
+  );
+  const orderByCondition = orderByToPrismaSql(orderBy, sessionsViewCols);
+
+  const sql = Prisma.sql`
+      SELECT
+        s.id,
+        s. "created_at" AS "createdAt",
+        s.bookmarked,
+        s.public,
+        t. "userIds",
+        t. "countTraces",
+        o. "sessionDuration",
+        o. "totalCost" AS "totalCost",
+        o. "inputCost" AS "inputCost",
+        o. "outputCost" AS "outputCost",
+        o. "promptTokens" AS "promptTokens",
+        o. "completionTokens" AS "completionTokens",
+        o. "totalTokens" AS "totalTokens",
+        (count(*) OVER ())::int AS "totalCount"
+      FROM
+        trace_sessions AS s
+        LEFT JOIN LATERAL (
+          SELECT
+            t.session_id,
+            MAX(t. "timestamp") AS "max_timestamp",
+            MIN(t. "timestamp") AS "min_timestamp",
+            array_agg(t.id) AS "traceIds",
+            array_agg(DISTINCT t.user_id) AS "userIds",
+            count(t.id)::int AS "countTraces"
+          FROM
+            traces t
+          WHERE
+            t.project_id = ${projectId}
+            AND t.session_id = s.id
+          GROUP BY
+            t.session_id) AS t ON TRUE
+        LEFT JOIN LATERAL (
+          SELECT
+            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
+            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
+            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
+            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
+            SUM(o.prompt_tokens) AS "promptTokens",
+            SUM(o.completion_tokens) AS "completionTokens",
+            SUM(o.total_tokens) AS "totalTokens"
+          FROM
+            observations_view o
+          WHERE
+            o.project_id = ${projectId}
+            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
+      WHERE
+        s. "project_id" = ${projectId}
+        ${filterCondition}
+      ${orderByCondition}
+      LIMIT ${limit}
+      OFFSET ${page * limit}
+    `;
+
+  return sql;
+};

packages/shared/src/queries/index.ts

@@ -0,0 +1 @@
+export { getSessionTableSQL } from "./getSessionTableSQL";

packages/shared/src/queues.ts

@@ -1,36 +1,75 @@
 import { z } from "zod";
 
-export const TraceUpsertEvent = z.object({
+export enum EventName {
+  TraceUpsert = "TraceUpsert",
+  BatchExport = "BatchExport",
+  EvaluationExecution = "EvaluationExecution",
+}
+
+export const BatchExportJobSchema = z.object({
+  projectId: z.string(),
+  batchExportId: z.string(),
+});
+export const TraceUpsertEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
-
 export const EvalExecutionEvent = z.object({
   projectId: z.string(),
   jobExecutionId: z.string(),
 });
 
+export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
+export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
+
+export const EventBodySchema = z.union([
+  z.object({
+    name: z.literal(EventName.TraceUpsert),
+    payload: z.array(TraceUpsertEventSchema),
+  }),
+  z.object({
+    name: z.literal(EventName.EvaluationExecution),
+    payload: EvalExecutionEvent,
+  }),
+  z.object({
+    name: z.literal(EventName.BatchExport),
+    payload: BatchExportJobSchema,
+  }),
+]);
+export type EventBodyType = z.infer<typeof EventBodySchema>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+  BatchExport = "batch-export-queue",
+  RepeatQueue = "repeat-queue",
 }
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
   EvaluationExecution = "evaluation-execution-job",
+  BatchExportJob = "batch-export-job",
+  EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
 }
 
 export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
     timestamp: Date;
     id: string;
-    payload: z.infer<typeof TraceUpsertEvent>;
+    payload: TraceUpsertEventType;
     name: QueueJobs.TraceUpsert;
   };
   [QueueName.EvaluationExecution]: {
     timestamp: Date;
     id: string;
-    payload: z.infer<typeof EvalExecutionEvent>;
+    payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
   };
+  [QueueName.BatchExport]: {
+    timestamp: Date;
+    id: string;
+    payload: BatchExportJobType;
+    name: QueueJobs.BatchExportJob;
+  };
 };

packages/shared/src/server/auth.ts

@@ -1,6 +1,7 @@
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
 
 export function generateSecretKey() {
   return `sk-lf-${randomUUID()}`;
@@ -48,3 +49,33 @@ export function createShaHash(privateKey: string, salt: string): string {
 
   return hash;
 }
+
+export interface CustomSSOUser extends Record<string, any> {
+  email: string;
+  id: string;
+  name: string;
+  verified: boolean;
+}
+
+export function CustomSSOProvider<P extends CustomSSOUser>(
+  options: OAuthUserConfig<P>
+): OAuthConfig<P> {
+  return {
+    id: "custom",
+    name: "CustomSSOProvider",
+    type: "oauth",
+    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
+    authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
+    checks: ["pkce", "state"],
+    idToken: true,
+    profile(profile) {
+      return {
+        id: profile.sub,
+        name: profile.name,
+        email: profile.email,
+        image: null,
+      };
+    },
+    options,
+  };
+}

packages/shared/src/server/index.ts

@@ -0,0 +1,9 @@
+export * from "./services/S3StorageService";
+export * from "./services/email/projectInvitation/sendProjectInvitationEmail";
+export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
+export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
+export * from "./auth";
+export * from "./llm/fetchLLMCompletion";
+export * from "./llm/types";
+export * from "./utils/DatabaseReadStream";
+export * from "./utils/transforms";

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -17,7 +17,7 @@ import {
   ChatMessageRole,
   LLMFunctionCall,
   ModelParams,
-  ModelProvider,
+  LLMAdapter,
 } from "./types";
 import zodToJsonSchema from "zod-to-json-schema";
 import { JsonOutputFunctionsParser } from "langchain/output_parsers";
@@ -27,7 +27,9 @@ type LLMCompletionParams = {
   modelParams: ModelParams;
   functionCall?: LLMFunctionCall;
   callbacks?: BaseCallbackHandler[];
+  baseURL?: string;
   apiKey?: string;
+  maxRetries?: number;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -57,7 +59,16 @@ export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   // the apiKey must never be printed to the console
-  const { messages, modelParams, streaming, callbacks, apiKey } = params;
+  const {
+    messages,
+    modelParams,
+    streaming,
+    callbacks,
+    apiKey,
+    baseURL,
+    maxRetries,
+  } = params;
+
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -68,16 +79,18 @@ export async function fetchLLMCompletion(
   });
 
   let chatModel: ChatOpenAI | ChatAnthropic;
-  if (modelParams.provider === ModelProvider.Anthropic) {
+  if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
+      anthropicApiUrl: baseURL,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      clientOptions: { maxRetries },
     });
-  } else if (modelParams.provider === ModelProvider.OpenAI) {
+  } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
       openAIApiKey: apiKey,
       modelName: modelParams.model,
@@ -85,15 +98,29 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks,
+      maxRetries,
+      configuration: {
+        baseURL,
+      },
+    });
+  } else if (modelParams.adapter === LLMAdapter.Azure) {
+    chatModel = new ChatOpenAI({
+      azureOpenAIApiKey: apiKey,
+      azureOpenAIBasePath: baseURL,
+      azureOpenAIApiDeploymentName: modelParams.model,
+      azureOpenAIApiVersion: "2024-02-01",
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+      maxRetries,
     });
   } else {
     // eslint-disable-next-line no-unused-vars
-    const _exhaustiveCheck: never = modelParams;
+    const _exhaustiveCheck: never = modelParams.adapter;
     throw new Error("This model provider is not supported.");
   }
 
-  console.log("Making LLM call with params: ", modelParams);
-
   if (params.functionCall) {
     const functionCallingModel = chatModel.bind({
       functions: [

packages/shared/src/server/llm/types.ts

@@ -1,3 +1,4 @@
+import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
@@ -9,9 +10,10 @@ export type ChatMessage = {
 
 export type ChatMessageWithId = ChatMessage & { id: string };
 
-export enum ModelProvider {
+export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
+  Azure = "azure",
 }
 
 export enum ChatMessageRole {
@@ -19,32 +21,30 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
 }
-export type ModelParams = AnthropicModelParams | OpenAIModelParams;
+
+export type ModelParams = {
+  provider: string;
+  adapter: LLMAdapter;
+  model: string;
+} & ModelConfig;
 
 type RecordWithEnabledFlag<T> = {
   [K in keyof T]: { value: T[K]; enabled: boolean };
 };
 export type UIModelParams = RecordWithEnabledFlag<
-  Required<AnthropicModelParams | OpenAIModelParams> & {
+  Required<ModelParams> & {
     maxTemperature: number;
   }
 >;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
-
 export const ZodModelConfig = z.object({
   max_tokens: z.coerce.number().optional(),
   temperature: z.coerce.number().optional(),
   top_p: z.coerce.number().optional(),
 });
 
-// OpenAI
-export type OpenAIModelParams = {
-  provider: ModelProvider.OpenAI;
-  model: OpenAIModel;
-} & ModelConfig;
-
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-05-13",
@@ -64,13 +64,8 @@ export const openAIModels = [
 
 export type OpenAIModel = (typeof openAIModels)[number];
 
-// Anthropic
-export type AnthropicModelParams = {
-  provider: ModelProvider.Anthropic;
-  model: AnthropicModel;
-} & ModelConfig;
-
 export const anthropicModels = [
+  "claude-3-5-sonnet-20240620",
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
   "claude-3-haiku-20240307",
@@ -81,8 +76,9 @@ export const anthropicModels = [
 
 export type AnthropicModel = (typeof anthropicModels)[number];
 export const supportedModels = {
-  [ModelProvider.Anthropic]: anthropicModels,
-  [ModelProvider.OpenAI]: openAIModels,
+  [LLMAdapter.Anthropic]: anthropicModels,
+  [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.Azure]: [],
 } as const;
 
 export type LLMFunctionCall = {
@@ -90,3 +86,26 @@ export type LLMFunctionCall = {
   description: string;
   parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
 };
+
+export const LLMApiKeySchema = z
+  .object({
+    id: z.string(),
+    projectId: z.string(),
+    createdAt: z.date(),
+    updatedAt: z.date(),
+    adapter: z.nativeEnum(LLMAdapter),
+    provider: z.string(),
+    displaySecretKey: z.string(),
+    secretKey: z.string(),
+    baseURL: z.string().nullable(),
+    customModels: z.array(z.string()),
+    withDefaultModels: z.boolean(),
+  })
+  // strict mode to prevent extra keys. Thorws error otherwise
+  // https://github.com/colinhacks/zod?tab=readme-ov-file#strict
+  .strict();
+
+export type LLMApiKey =
+  z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
+    ? z.infer<typeof LLMApiKeySchema>
+    : never;

packages/shared/src/server/services/S3StorageService.ts

@@ -1,5 +1,4 @@
 import type { Readable } from "stream";
-import { env } from "@/src/env.mjs";
 import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -8,44 +7,52 @@ type UploadFile = {
   fileName: string;
   fileType: string;
   data: Readable | string;
+  expiresInSeconds: number;
 };
 
-class S3StorageService {
+export class S3StorageService {
   private client: S3Client;
+  private bucketName: string;
 
-  constructor() {
-    if (!S3StorageService.getIsS3StorageConfigured(env)) {
-      throw new Error("S3 bucket is not configured");
-    }
+  constructor(params: {
+    accessKeyId: string;
+    secretAccessKey: string;
+    bucketName: string;
+    endpoint: string;
+    region: string;
+  }) {
+    const { accessKeyId, secretAccessKey, bucketName, endpoint, region } =
+      params;
 
     this.client = new S3Client({
       credentials: {
-        accessKeyId: env.S3_ACCESS_KEY_ID,
-        secretAccessKey: env.S3_SECRET_ACCESS_KEY,
+        accessKeyId,
+        secretAccessKey,
       },
-      endpoint: env.S3_ENDPOINT,
-      region: env.S3_REGION,
+      endpoint,
+      region,
     });
+    this.bucketName = bucketName;
   }
 
   public async uploadFile({
     fileName,
     fileType,
     data,
+    expiresInSeconds,
   }: UploadFile): Promise<{ signedUrl: string }> {
     try {
       await new Upload({
         client: this.client,
         params: {
-          Bucket: env.S3_BUCKET_NAME,
+          Bucket: this.bucketName,
           Key: fileName,
           Body: data,
           ContentType: fileType,
         },
       }).done();
 
-      const expiresInOneHour = 60 * 60;
-      const signedUrl = await this.getSignedUrl(fileName, expiresInOneHour);
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
 
       return { signedUrl };
     } catch (err) {
@@ -57,42 +64,20 @@ class S3StorageService {
 
   private async getSignedUrl(
     fileName: string,
-    ttlSeconds: number,
+    ttlSeconds: number
   ): Promise<string> {
     try {
       return await getSignedUrl(
         this.client,
         new GetObjectCommand({
-          Bucket: env.S3_BUCKET_NAME,
+          Bucket: this.bucketName,
           Key: fileName,
           ResponseContentDisposition: `attachment; filename="${fileName}"`,
         }),
-        { expiresIn: ttlSeconds },
+        { expiresIn: ttlSeconds }
       );
     } catch (err) {
       throw Error("Failed to generate signed URL");
     }
   }
-
-  static getIsS3StorageConfigured(
-    currentEnv: Env,
-  ): currentEnv is S3ConfiguredEnv {
-    return Boolean(
-      currentEnv.S3_BUCKET_NAME &&
-        currentEnv.S3_ACCESS_KEY_ID &&
-        currentEnv.S3_SECRET_ACCESS_KEY &&
-        currentEnv.S3_ENDPOINT &&
-        currentEnv.S3_REGION,
-    );
-  }
 }
-
-export { S3StorageService };
-
-type Env = typeof env;
-type S3ConfiguredEnv = Env & {
-  S3_ACCESS_KEY_ID: string;
-  S3_SECRET_ACCESS_KEY: string;
-  S3_ENDPOINT: string;
-  S3_REGION: string;
-};

packages/shared/src/server/services/email/batchExportSuccess/BatchExportSuccessEmailTemplate.tsx

@@ -0,0 +1,83 @@
+import * as React from "react";
+import {
+  Body,
+  Button,
+  Container,
+  Head,
+  Heading,
+  Hr,
+  Html,
+  Img,
+  Preview,
+  Section,
+  Tailwind,
+  Text,
+} from "@react-email/components";
+
+interface BatchExportSuccessTemplateProps {
+  userName: string;
+  receiverEmail: string;
+  downloadLink: string;
+  batchExportName: string;
+  expiresInHours: number;
+}
+
+export const BatchExportSuccessEmailTemplate = ({
+  receiverEmail,
+  downloadLink,
+  userName,
+  batchExportName,
+  expiresInHours,
+}: BatchExportSuccessTemplateProps) => {
+  const previewText = `Download your data export from Langfuse`;
+
+  return (
+    <Html>
+      <Head />
+      <Preview>{previewText}</Preview>
+      <Tailwind>
+        <Body className="mx-auto my-auto bg-background font-sans">
+          <Container className="mx-auto my-10 w-[465px] rounded border border-solid border-[#eaeaea] p-5">
+            <Section className="mt-8">
+              <Img
+                src="https://static.langfuse.com/langfuse_logo_transactional_email.png"
+                width="40"
+                height="40"
+                alt="Langfuse"
+                className="mx-auto my-0"
+              />
+            </Section>
+            <Heading className="mx-0 my-[30px] p-0 text-center text-2xl font-normal text-black">
+              Your data export is ready
+            </Heading>
+            <Text className="text-sm leading-6 text-black">
+              Hello <strong>{userName}</strong>
+            </Text>
+            <Text className="text-sm leading-6 text-black">
+              Your data export{" "}
+              <span className="font-mono">{batchExportName}</span> is ready to
+              download. The download link will expire in{" "}
+              <strong>{expiresInHours} hours</strong>.
+            </Text>
+            <Section className="mb-4 mt-8 text-center">
+              <Button
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
+                href={downloadLink}
+              >
+                Download Export
+              </Button>
+            </Section>
+            <Hr className="mx-0 my-[26px] w-full border border-solid border-[#eaeaea]" />
+            <Text className="text-xs leading-6 text-[#666666]">
+              This email was intended for{" "}
+              <span className="text-black">{receiverEmail}</span>. If you were
+              not expecting this email, please delete it.
+            </Text>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  );
+};
+
+export default BatchExportSuccessEmailTemplate;

packages/shared/src/server/services/email/batchExportSuccess/sendBatchExportSuccessEmail.ts

@@ -0,0 +1,56 @@
+import { createTransport } from "nodemailer";
+import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
+import { render } from "@react-email/render";
+
+import { BatchExportSuccessEmailTemplate } from "./BatchExportSuccessEmailTemplate";
+
+type SendBatchExportSuccessParams = {
+  env: Partial<
+    Record<"EMAIL_FROM_ADDRESS" | "SMTP_CONNECTION_URL", string | undefined>
+  >;
+  receiverEmail: string;
+  downloadLink: string;
+  userName: string;
+  batchExportName: string;
+  expiresInHours: number;
+};
+
+export const sendBatchExportSuccessEmail = async ({
+  env,
+  receiverEmail,
+  downloadLink,
+  userName,
+  batchExportName,
+  expiresInHours,
+}: SendBatchExportSuccessParams) => {
+  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
+    console.error("Missing environment variables for sending email.");
+
+    return;
+  }
+
+  try {
+    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
+    const htmlTemplate = render(
+      BatchExportSuccessEmailTemplate({
+        receiverEmail,
+        downloadLink,
+        userName,
+        batchExportName,
+        expiresInHours,
+      })
+    );
+
+    await mailer.sendMail({
+      to: receiverEmail,
+      from: {
+        address: env.EMAIL_FROM_ADDRESS,
+        name: "Langfuse",
+      },
+      subject: "Your data export is ready",
+      html: htmlTemplate,
+    });
+  } catch (error) {
+    console.error(error);
+  }
+};

packages/shared/src/server/services/email/passwordReset/sendResetPasswordVerificationRequest.tsx

@@ -0,0 +1,88 @@
+/*
+ * To be used in the `sendVerificationRequest` function of the `email` provider of NextAuth.js.
+ */
+
+import * as React from "react";
+import {
+  Body,
+  Button,
+  Container,
+  Head,
+  Heading,
+  Html,
+  Img,
+  Preview,
+  Section,
+  Tailwind,
+  Text,
+} from "@react-email/components";
+import { createTransport } from "nodemailer";
+import { render } from "@react-email/render";
+import { type SendVerificationRequestParams } from "next-auth/providers/email";
+
+interface ResetPasswordTemplateProps {
+  url: string;
+}
+
+const ResetPasswordTemplate = ({ url }: ResetPasswordTemplateProps) => {
+  const previewText = "Reset your Langfuse password";
+  return (
+    <Html>
+      <Head />
+      <Preview>{previewText}</Preview>
+      <Tailwind>
+        <Body className="mx-auto my-auto bg-background font-sans">
+          <Container className="mx-auto my-10 w-[465px] rounded border border-solid border-[#eaeaea] p-5">
+            <Section className="mt-8">
+              <Img
+                src="https://static.langfuse.com/langfuse_logo_transactional_email.png"
+                width="40"
+                height="40"
+                alt="Langfuse"
+                className="mx-auto my-0"
+              />
+            </Section>
+            <Heading className="mx-0 my-[30px] p-0 text-center text-xl font-normal text-black">
+              Forgot your Langfuse password?
+              <br />
+              It happens to the best of us.
+            </Heading>
+            <Section className="mb-8 mt-8 text-center">
+              <Button
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
+                href={url}
+              >
+                Reset your password
+              </Button>
+            </Section>
+            <Text className="text-center text-xs leading-6 text-[#666666]">
+              If you do not want to change your password or didn&apos;t request
+              a reset, you can ignore and delete this email.
+            </Text>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  );
+};
+
+export async function sendResetPasswordVerificationRequest(
+  params: SendVerificationRequestParams
+) {
+  const { identifier, url, provider } = params;
+  const transport = createTransport(provider.server);
+  const htmlTemplate = render(<ResetPasswordTemplate url={url} />);
+  const result = await transport.sendMail({
+    to: identifier,
+    from: provider.from,
+    subject: `Forgot your password?`,
+    text: `To reset your Langfuse password, please confirm your email:\n${url}\n\nIf you do not want to change your password or didn't request a reset, you can ignore and delete this email.`,
+    html: htmlTemplate,
+  });
+  const failed = result.rejected.concat(result.pending).filter(Boolean);
+  if (failed.length) {
+    throw new Error(`Email(s) (${failed.join(", ")}) could not be sent`);
+  }
+}
+
+export default ResetPasswordTemplate;

packages/shared/src/server/services/email/projectInvitation/ProjectInvitationEmailTemplate.tsx

@@ -15,14 +15,13 @@ import {
   Text,
 } from "@react-email/components";
 
-import { env } from "@/src/env.mjs";
-
 interface ProjectInvitationTemplateProps {
   invitedByUsername: string;
   invitedByUserEmail: string;
   projectName: string;
-  recieverEmail: string;
+  receiverEmail: string;
   inviteLink: string;
+  emailFromAddress: string;
   langfuseCloudRegion?: string;
 }
 
@@ -30,8 +29,9 @@ export const ProjectInvitationTemplate = ({
   invitedByUsername,
   invitedByUserEmail,
   projectName,
-  recieverEmail,
+  receiverEmail,
   inviteLink,
+  emailFromAddress,
   langfuseCloudRegion,
 }: ProjectInvitationTemplateProps) => {
   const previewText = `Join ${invitedByUsername} on Langfuse`;
@@ -72,7 +72,7 @@ export const ProjectInvitationTemplate = ({
             </Text>
             <Section className="mb-4 mt-8 text-center">
               <Button
-                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-background no-underline"
+                className="rounded bg-black px-5 py-3 text-center text-xs font-semibold text-white no-underline"
                 href={inviteLink}
               >
                 Accept Invitation
@@ -90,10 +90,10 @@ export const ProjectInvitationTemplate = ({
             <Hr className="mx-0 my-[26px] w-full border border-solid border-[#eaeaea]" />
             <Text className="text-xs leading-6 text-[#666666]">
               This invitation was intended for{" "}
-              <span className="text-black">{recieverEmail}</span>. This invite
+              <span className="text-black">{receiverEmail}</span>. This invite
               was sent from{" "}
-              <span className="text-black">{env.EMAIL_FROM_ADDRESS}</span>. If
-              you were not expecting this invitation, you can ignore this email.
+              <span className="text-black">{emailFromAddress}</span>. If you
+              were not expecting this invitation, you can ignore this email.
             </Text>
           </Container>
         </Body>

packages/shared/src/server/services/email/projectInvitation/sendProjectInvitationEmail.ts

@@ -0,0 +1,85 @@
+import { createTransport } from "nodemailer";
+import { parseConnectionUrl } from "nodemailer/lib/shared/index.js";
+import { render } from "@react-email/render";
+
+import ProjectInvitationTemplate from "./ProjectInvitationEmailTemplate";
+
+const langfuseUrls = {
+  US: "https://us.cloud.langfuse.com",
+  EU: "https://cloud.langfuse.com",
+  STAGING: "https://staging.langfuse.com",
+};
+
+type SendProjectInvitationParams = {
+  env: Partial<
+    Record<
+      | "EMAIL_FROM_ADDRESS"
+      | "SMTP_CONNECTION_URL"
+      | "NEXT_PUBLIC_LANGFUSE_CLOUD_REGION"
+      | "NEXTAUTH_URL",
+      string | undefined
+    >
+  >;
+  to: string;
+  inviterName: string;
+  inviterEmail: string;
+  projectName: string;
+};
+
+export const sendProjectInvitationEmail = async ({
+  env,
+  to,
+  inviterName,
+  inviterEmail,
+  projectName,
+}: SendProjectInvitationParams) => {
+  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
+    console.error(
+      "Missing environment variables for sending project invitation email."
+    );
+    return;
+  }
+
+  const getAuthURL = () =>
+    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "US" ||
+    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU" ||
+    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
+      ? langfuseUrls[env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION]
+      : env.NEXTAUTH_URL;
+
+  const authUrl = getAuthURL();
+  if (!authUrl) {
+    console.error(
+      "Missing NEXTAUTH_URL or NEXT_PUBLIC_LANGFUSE_CLOUD_REGION environment variable."
+    );
+    return;
+  }
+
+  try {
+    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
+
+    const htmlTemplate = render(
+      ProjectInvitationTemplate({
+        invitedByUsername: inviterName,
+        invitedByUserEmail: inviterEmail,
+        projectName: projectName,
+        receiverEmail: to,
+        inviteLink: authUrl,
+        emailFromAddress: env.EMAIL_FROM_ADDRESS,
+        langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+      })
+    );
+
+    await mailer.sendMail({
+      to,
+      from: {
+        address: env.EMAIL_FROM_ADDRESS,
+        name: "Langfuse",
+      },
+      subject: `${inviterName} invited you to join "${projectName}"`,
+      html: htmlTemplate,
+    });
+  } catch (error) {
+    console.error(error);
+  }
+};

packages/shared/src/server/utils/DatabaseReadStream.ts

@@ -26,9 +26,10 @@ export class DatabaseReadStream<EntityType> extends Readable {
     // the delegate function takes care of querying the database in a paginated manner
     private queryDelegate: (
       pageSize: number,
-      offset: number,
+      offset: number
     ) => Promise<Array<EntityType>>,
     private pageSize: number,
+    private maxRecords?: number
   ) {
     super({ objectMode: true }); // Set object mode to true to allow pushing objects to the stream rather than strings or buffers
 
@@ -43,10 +44,19 @@ export class DatabaseReadStream<EntityType> extends Readable {
     this.isReading = true;
 
     try {
+      // Stop reading if the maximum number of records has been reached
+      if (this.maxRecords && this.offset >= this.maxRecords) {
+        this.hasNextPage = false;
+        this.push(null); // Signal end of stream
+
+        return;
+      }
+
       const rows = await this.queryDelegate(this.pageSize, this.offset);
+
       if (rows.length > 0) {
         rows.forEach((row) => this.push(row));
-        this.offset += rows.length;
+        this.offset += this.pageSize;
       } else {
         this.hasNextPage = false;
         this.push(null); // Signal end of stream

packages/shared/src/server/utils/transforms/index.ts

@@ -0,0 +1,13 @@
+import { Transform } from "stream";
+
+import { BatchExportFileFormat } from "../../../features/batchExport/types";
+import { transformStreamToCsv } from "./transformStreamToCsv";
+import { transformStreamToJson } from "./transformStreamToJson";
+
+export const streamTransformations: Record<
+  BatchExportFileFormat,
+  () => Transform
+> = {
+  CSV: transformStreamToCsv,
+  JSON: transformStreamToJson,
+};

packages/shared/src/server/utils/transforms/stringify.ts

@@ -0,0 +1,5 @@
+export const stringify = (data: any): string => {
+  return JSON.stringify(data, (key, value) =>
+    typeof value === "bigint" ? Number.parseInt(value.toString()) : value
+  );
+};

packages/shared/src/server/utils/transforms/transformStreamToCsv.ts

@@ -0,0 +1,39 @@
+import { Transform, type TransformCallback } from "stream";
+import { stringify } from "./stringify";
+
+export function transformStreamToCsv(): Transform {
+  let isFirstChunk = true;
+  let headers: string[] = [];
+
+  return new Transform({
+    objectMode: true,
+    transform(
+      row: Record<string, any>,
+      encoding: BufferEncoding,
+      callback: TransformCallback
+    ): void {
+      if (isFirstChunk) {
+        // Extract headers from the first object
+        headers = Object.keys(row);
+        this.push(headers.join(",") + "\n");
+        isFirstChunk = false;
+      }
+
+      // Convert the object to a CSV line and push it
+      const csvRow = headers.map((header) => {
+        const field = row[header];
+        const str = stringify(field);
+
+        if (str.startsWith('"') && str.endsWith('"')) {
+          return str;
+        } else {
+          return `"${str?.replace(/"/g, '""') ?? ""}"`;
+        }
+      });
+
+      this.push(csvRow.join(",") + "\n");
+
+      callback();
+    },
+  });
+}

packages/shared/src/server/utils/transforms/transformStreamToJson.ts

@@ -0,0 +1,39 @@
+import { Transform, type TransformCallback } from "stream";
+import { stringify } from "./stringify";
+
+export function transformStreamToJson(): Transform {
+  let isFirstElement = true;
+
+  return new Transform({
+    objectMode: true,
+
+    transform(
+      row: any,
+      encoding: BufferEncoding,
+      callback: TransformCallback
+    ): void {
+      if (isFirstElement) {
+        this.push("["); // Push the opening bracket for the first element
+        isFirstElement = false; // Reset the flag after the first element
+      } else {
+        this.push(","); // For subsequent elements, prepend a comma
+      }
+
+      this.push(stringify(row)); // Push the current row as a JSON string
+
+      callback();
+    },
+
+    // 'final' is called when there is no more data to be consumed, but before the stream is finished.
+    final(callback: TransformCallback): void {
+      if (isFirstElement) {
+        // If no rows were processed, the opening bracket has not been pushed yet.
+        this.push("[]"); // Push an empty array to ensure valid JSON.
+      } else {
+        this.push("]"); // Close JSON array
+      }
+
+      callback();
+    },
+  });
+}

packages/shared/src/tableDefinitions/index.ts

@@ -0,0 +1,2 @@
+export * from "./sessionsView";
+export * from "./types";

packages/shared/src/tableDefinitions/sessionsView.ts

@@ -1,7 +1,7 @@
 import {
   type OptionsDefinition,
   type ColumnDefinition,
-} from "@langfuse/shared";
+} from "../tableDefinitions/types";
 
 export const sessionsViewCols: ColumnDefinition[] = [
   { name: "⭐️", id: "bookmarked", type: "boolean", internal: "s.bookmarked" },
@@ -85,7 +85,7 @@ export type SessionOptions = {
 };
 
 export function sessionsTableColsWithOptions(
-  options?: SessionOptions,
+  options?: SessionOptions
 ): ColumnDefinition[] {
   return sessionsViewCols.map((col) => {
     if (col.id === "userIds") {

packages/shared/src/utils/zod.ts

@@ -58,7 +58,32 @@ export const optionalPaginationZod = {
     .optional(),
 };
 
+export const queryStringZod = z
+  .string()
+  .transform((val) => decodeURIComponent(val));
+
+export const paginationMetaResponseZod = z.object({
+  page: z.number().int().positive(),
+  limit: z.number().int().positive(),
+  totalItems: z.number().int().nonnegative(),
+  totalPages: z.number().int().nonnegative(),
+});
+
 export const noHtmlRegex = /<[^>]*>/;
 export const noHtmlCheck = (value: string) => !noHtmlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);
+
+/**
+ * Validates an object against a Zod schema and helps with IDE type warnings.
+ *
+ * @param schema - The Zod schema to validate against.
+ * @param object - The object to be validated.
+ * @returns The parsed object if validation is successful.
+ */
+export const validateZodSchema = <T extends z.ZodTypeAny>(
+  schema: T,
+  object: z.infer<T>
+): z.infer<T> => {
+  return schema.parse(object);
+};

packages/shared/tsconfig.json

@@ -1,6 +1,7 @@
 {
   "extends": "@repo/typescript-config/base.json",
   "compilerOptions": {
+    "jsx": "react",
     "moduleResolution": "NodeNext",
     "module": "NodeNext",
     "lib": ["ES2020"],

turbo.json

@@ -9,7 +9,9 @@
     "start": {
       "dependsOn": ["^start"]
     },
-    "db:migrate": {},
+    "db:migrate": {
+      "cache": false
+    },
     "db:deploy": {},
     "db:reset": {},
     "db:push": {},

web/Dockerfile

@@ -1,10 +1,10 @@
-FROM node:20-alpine AS alpine
+FROM --platform=${BUILDPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
+RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
-FROM alpine AS base
+FROM --platform=${BUILDPLATFORM:-linux/amd64} alpine AS base
 RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
@@ -12,7 +12,7 @@ RUN corepack enable
 RUN corepack prepare pnpm@8.15.5 --activate
 
 
-FROM base AS pruner
+FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS pruner
 
 WORKDIR /app
 
@@ -20,7 +20,7 @@ COPY . .
 RUN turbo prune --scope=web --docker
 
 
-FROM base AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} base AS builder
 
 WORKDIR /app
 
@@ -33,6 +33,15 @@ RUN pnpm install --frozen-lockfile
 
 ENV DOCKER_BUILD 1
 
+# pass public variables in build step
+ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+ARG NEXT_PUBLIC_DEMO_PROJECT_ID
+ARG NEXT_PUBLIC_SIGN_UP_DISABLED
+ARG NEXT_PUBLIC_TURNSTILE_SITE_KEY
+ARG NEXT_PUBLIC_POSTHOG_KEY
+ARG NEXT_PUBLIC_POSTHOG_HOST
+ARG NEXT_PUBLIC_CRISP_WEBSITE_ID
+ARG LANFUSE_WEB_MIGRATION_DISABLED
 
 # Copy source code of isolated subworkspace
 COPY --from=pruner /app/out/full/ .
@@ -48,7 +57,7 @@ ENV NEXT_TELEMETRY_DISABLED 1
 RUN turbo run build --filter=web
 
 # Production image, copy all the files and run next
-FROM base AS runner
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS runner
 
 RUN apk add --no-cache dumb-init
 
@@ -78,6 +87,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/web/public ./web/public
 COPY --from=builder --chown=nextjs:nodejs /app/packages/shared/prisma ./packages/shared/prisma
 
 COPY --chown=nextjs:nodejs ./web/entrypoint.sh ./web/entrypoint.sh
+COPY --chown=nextjs:nodejs ./packages/shared/scripts/cleanup.sql ./packages/shared/scripts/cleanup.sql
 RUN chmod +x ./web/entrypoint.sh
 
 USER nextjs

web/entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
+# Run cleanup script before running migrations
 # Check if DATABASE_URL is not set
 if [ -z "$DATABASE_URL" ]; then
     # Check if all required variables are provided
@@ -18,8 +19,13 @@ if [ -z "$DIRECT_URL" ]; then
     export DIRECT_URL=$DATABASE_URL
 fi
 
-# Apply migrations
-prisma migrate deploy --schema=./packages/shared/prisma/schema.prisma
+# Always execute the scripts, except when disabled.
+if [ "$LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED" != "true" ]; then
+    prisma db execute --url "$DIRECT_URL" --file "./packages/shared/scripts/cleanup.sql"
+
+    # Apply migrations
+    prisma migrate deploy --schema=./packages/shared/prisma/schema.prisma
+fi
 status=$?
 
 # If migration fails (returns non-zero exit status), exit script with that status

web/next.config.mjs

@@ -11,8 +11,8 @@ import { env } from "./src/env.mjs";
  * img-src https to allow loading images from SSO providers
  */
 const cspHeader = `
-  default-src 'self' https://ph.langfuse.com https://*.posthog.com wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  default-src 'self' https://*.langfuse.com https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
+  script-src 'self' 'unsafe-eval' https://*.langfuse.com https://*.crisp.chat https://challenges.cloudflare.com https://*.sentry.io https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
   style-src 'self' 'unsafe-inline' https://*.crisp.chat;
   img-src 'self' https: blob: data:;
   font-src 'self' https://*.crisp.chat;
@@ -29,6 +29,9 @@ const cspHeader = `
 const nextConfig = {
   transpilePackages: ["@langfuse/shared"],
   reactStrictMode: true,
+  experimental: {
+    instrumentationHook: true,
+  },
 
   /**
    * If you have `experimental: { appDir: true }` set, then you must comment the below `i18n` config
@@ -106,26 +109,9 @@ const nextConfig = {
 
     return config;
   },
-  sentry: {
-    // See the sections below for information on the following options:
-    //   'Configure Source Maps':
-    //     - disableServerWebpackPlugin
-    //     - disableClientWebpackPlugin
-    //     - hideSourceMaps
-    hideSourceMaps: true,
-    //     - widenClientFileUpload
-    //   'Configure Legacy Browser Support':
-    //     - transpileClientSDK
-    //   'Configure Serverside Auto-instrumentation':
-    //     - autoInstrumentServerFunctions
-    //     - excludeServerRoutes
-    //   'Configure Tunneling':
-    //     - tunnelRoute
-    tunnelRoute: "/api/monitoring-tunnel",
-  },
 };
 
-const sentryWebpackPluginOptions = {
+const sentryOptions = {
   // Additional config options for the Sentry Webpack plugin. Keep in mind that
   // the following options are set automatically, and overriding them is not
   // recommended:
@@ -139,6 +125,22 @@ const sentryWebpackPluginOptions = {
 
   // For all available options, see:
   // https://github.com/getsentry/sentry-webpack-plugin#options.
+
+  // See the sections below for information on the following options:
+  //   'Configure Source Maps':
+  //     - disableServerWebpackPlugin
+  //     - disableClientWebpackPlugin
+  //     - hideSourceMaps
+  hideSourceMaps: true,
+  //     - widenClientFileUpload
+  //   'Configure Legacy Browser Support':
+  //     - transpileClientSDK
+  //   'Configure Serverside Auto-instrumentation':
+  //     - autoInstrumentServerFunctions
+  //     - excludeServerRoutes
+  //   'Configure Tunneling':
+  //     - tunnelRoute
+  tunnelRoute: "/api/monitoring-tunnel",
 };
 
-export default withSentryConfig(nextConfig, sentryWebpackPluginOptions);
+export default withSentryConfig(nextConfig, sentryOptions);

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.49.2",
+  "version": "2.58.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -17,17 +17,15 @@
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
+    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts",
+    "generationCost:backfill": "dotenv -e ../.env -- tsx scripts/observations-backfill-calculated-cost.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.568.0",
-    "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
     "@codemirror/lint": "^6.8.0",
     "@headlessui/react": "1.7.18",
-    "@headlessui/tailwindcss": "0.2.0",
+    "@headlessui/tailwindcss": "0.2.1",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
     "@langchain/anthropic": "^0.1.21",
@@ -36,6 +34,7 @@
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
+    "@mui/x-tree-view": "^7.6.2",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -58,15 +57,13 @@
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-toggle-group": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.18",
-    "@react-email/render": "^0.0.14",
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.113.0",
-    "@sentry/node": "^7.113.0",
-    "@sentry/profiling-node": "^7.113.0",
-    "@sentry/types": "^7.113.0",
+    "@sentry/nextjs": "^8.13.0",
+    "@sentry/node": "^8.13.0",
+    "@sentry/profiling-node": "^8.13.0",
+    "@sentry/types": "^8.13.0",
     "@t3-oss/env-nextjs": "^0.10.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -77,7 +74,7 @@
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
@@ -88,18 +85,17 @@
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.4",
+    "dompurify": "^3.1.5",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.36",
+    "langchain": "^0.2.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
     "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "next-themes": "^0.3.0",
-    "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
     "prisma": "^5.13.0",
@@ -124,16 +120,15 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
     "@tailwindcss/forms": "^0.5.7",
-    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/jest-dom": "^6.4.6",
     "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
     "@types/node": "20.10.5",
-    "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
@@ -149,7 +144,7 @@
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.14",
-    "tailwindcss": "^3.4.3",
+    "tailwindcss": "^3.4.4",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.2",

web/public/generated/api/openapi.yml

@@ -71,6 +71,77 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetItemRequest'
+    get:
+      description: Get dataset items
+      operationId: datasetItems_list
+      tags:
+        - DatasetItems
+      parameters:
+        - name: datasetName
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: sourceTraceId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: sourceObservationId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedDatasetItems'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/dataset-items/{id}:
     get:
       description: Get a dataset item
@@ -162,7 +233,7 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRunItemRequest'
-  /api/public/datasets:
+  /api/public/v2/datasets:
     get:
       description: Get all datasets
       operationId: datasets_list
@@ -171,12 +242,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -259,7 +332,7 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRequest'
-  /api/public/datasets/{datasetName}:
+  /api/public/v2/datasets/{datasetName}:
     get:
       description: Get a dataset and its items
       operationId: datasets_get
@@ -307,7 +380,7 @@ paths:
   /api/public/datasets/{datasetName}/runs/{runName}:
     get:
       description: Get a dataset run and its items
-      operationId: datasets_getRuns
+      operationId: datasets_getRun
       tags:
         - Datasets
       parameters:
@@ -327,7 +400,66 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/DatasetRun'
+                $ref: '#/components/schemas/DatasetRunWithItems'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/datasets/{datasetName}/runs:
+    get:
+      description: Get a dataset runs
+      operationId: datasets_getRuns
+      tags:
+        - Datasets
+      parameters:
+        - name: datasetName
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: page
+          in: query
+          description: page number, starts at 1
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: limit of items per page
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedDatasetRuns'
         '400':
           description: ''
           content:
@@ -397,7 +529,21 @@ paths:
           description: ''
   /api/public/ingestion:
     post:
-      description: Batched ingestion for Langfuse Tracing
+      description: >-
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via
+        the API, such as to build your own Langfuse client implementation, this
+        is the only API route you need to implement.
+
+
+        Notes:
+
+
+        - Batch sizes are limited to 3.5 MB in total. You need to adjust the
+        number of events per batch accordingly.
+
+        - The API does not return a 4xx status code for input errors. Instead,
+        it responds with a 207 status code, which includes a list of the
+        encountered errors.
       operationId: ingestion_batch
       tags:
         - Ingestion
@@ -446,6 +592,14 @@ paths:
                   type: array
                   items:
                     $ref: '#/components/schemas/IngestionEvent'
+                  description: >-
+                    Batch of tracing events to be ingested. Discriminated by
+                    attribute `type`.
+                metadata:
+                  nullable: true
+                  description: >-
+                    Optional. Metadata field used by the Langfuse SDKs for
+                    debugging.
               required:
                 - batch
   /api/public/metrics/daily:
@@ -818,12 +972,14 @@ paths:
             nullable: true
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -906,6 +1062,108 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreatePromptRequest'
+  /api/public/score-configs:
+    get:
+      description: Get all score configs
+      operationId: scoreConfigs_get
+      tags:
+        - ScoreConfigs
+      parameters:
+        - name: page
+          in: query
+          description: Page number, starts at 1.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreConfigs'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/score-configs/{configId}:
+    get:
+      description: Get a score config
+      operationId: scoreConfigs_get-by-id
+      tags:
+        - ScoreConfigs
+      parameters:
+        - name: configId
+          in: path
+          description: The unique langfuse identifier of a score config
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreConfig'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/scores:
     post:
       description: Create a score
@@ -997,9 +1255,10 @@ paths:
         - name: source
           in: query
           description: Retrieve only scores from a specific source.
-          required: true
+          required: false
           schema:
             $ref: '#/components/schemas/ScoreSource'
+            nullable: true
         - name: operator
           in: query
           description: Retrieve only scores with <operator> value.
@@ -1015,6 +1274,13 @@ paths:
             type: number
             format: double
             nullable: true
+        - name: scoreIds
+          in: query
+          description: Comma-separated list of score IDs to limit the results to.
+          required: false
+          schema:
+            type: string
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1595,6 +1861,63 @@ components:
           format: double
           nullable: true
           description: USD total cost, defaults to input+output
+    ScoreConfig:
+      title: ScoreConfig
+      type: object
+      description: Configuration for a score
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        createdAt:
+          type: string
+          format: date-time
+        updatedAt:
+          type: string
+          format: date-time
+        projectId:
+          type: string
+        dataType:
+          $ref: '#/components/schemas/ScoreDataType'
+        isArchived:
+          type: boolean
+        minValue:
+          type: number
+          format: double
+          nullable: true
+        maxValue:
+          type: number
+          format: double
+          nullable: true
+        categories:
+          type: array
+          items:
+            $ref: '#/components/schemas/ConfigCategory'
+          nullable: true
+        description:
+          type: string
+          nullable: true
+      required:
+        - id
+        - name
+        - createdAt
+        - updatedAt
+        - projectId
+        - dataType
+        - isArchived
+    ConfigCategory:
+      title: ConfigCategory
+      type: object
+      properties:
+        value:
+          type: number
+          format: double
+        label:
+          type: string
+      required:
+        - value
+        - label
     Score:
       title: Score
       type: object
@@ -1626,8 +1949,8 @@ components:
         - value
         - source
         - timestamp
-    DatasetCore:
-      title: DatasetCore
+    Dataset:
+      title: Dataset
       type: object
       properties:
         id:
@@ -1653,43 +1976,6 @@ components:
         - projectId
         - createdAt
         - updatedAt
-    DatasetWithReferences:
-      title: DatasetWithReferences
-      type: object
-      properties:
-        items:
-          type: array
-          items:
-            type: string
-          description: list of dataset item ids
-        runs:
-          type: array
-          items:
-            type: string
-          description: list of dataset run names
-      required:
-        - items
-        - runs
-      allOf:
-        - $ref: '#/components/schemas/DatasetCore'
-    Dataset:
-      title: Dataset
-      type: object
-      description: Dataset including all items
-      properties:
-        items:
-          type: array
-          items:
-            $ref: '#/components/schemas/DatasetItem'
-        runs:
-          type: array
-          items:
-            type: string
-      required:
-        - items
-        - runs
-      allOf:
-        - $ref: '#/components/schemas/DatasetCore'
     DatasetItem:
       title: DatasetItem
       type: object
@@ -1781,10 +2067,6 @@ components:
         updatedAt:
           type: string
           format: date-time
-        datasetRunItems:
-          type: array
-          items:
-            $ref: '#/components/schemas/DatasetRunItem'
       required:
         - id
         - name
@@ -1792,7 +2074,18 @@ components:
         - datasetName
         - createdAt
         - updatedAt
+    DatasetRunWithItems:
+      title: DatasetRunWithItems
+      type: object
+      properties:
+        datasetRunItems:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetRunItem'
+      required:
         - datasetRunItems
+      allOf:
+        - $ref: '#/components/schemas/DatasetRun'
     ModelUsageUnit:
       title: ModelUsageUnit
       type: string
@@ -1837,6 +2130,13 @@ components:
         - ANNOTATION
         - API
         - EVAL
+    ScoreDataType:
+      title: ScoreDataType
+      type: string
+      enum:
+        - NUMERIC
+        - BOOLEAN
+        - CATEGORICAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
       type: object
@@ -1858,13 +2158,28 @@ components:
         id:
           type: string
           nullable: true
-          description: Dataset items are upserted on their id
+          description: >-
+            Dataset items are upserted on their id. Id needs to be globally
+            unique and cannot be reused across datasets.
         status:
           $ref: '#/components/schemas/DatasetStatus'
           nullable: true
           description: Defaults to ACTIVE for newly created items
       required:
         - datasetName
+    PaginatedDatasetItems:
+      title: PaginatedDatasetItems
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetItem'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     CreateDatasetRunItemRequest:
       title: CreateDatasetRunItemRequest
       type: object
@@ -1899,7 +2214,7 @@ components:
         data:
           type: array
           items:
-            $ref: '#/components/schemas/DatasetWithReferences'
+            $ref: '#/components/schemas/Dataset'
         meta:
           $ref: '#/components/schemas/utilsMetaResponse'
       required:
@@ -1918,6 +2233,19 @@ components:
           nullable: true
       required:
         - name
+    PaginatedDatasetRuns:
+      title: PaginatedDatasetRuns
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetRun'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     HealthResponse:
       title: HealthResponse
       type: object
@@ -1964,8 +2292,19 @@ components:
                 type:
                   type: string
                   enum:
-                    - event-create
-            - $ref: '#/components/schemas/CreateEventEvent'
+                    - span-create
+            - $ref: '#/components/schemas/CreateSpanEvent'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - span-update
+            - $ref: '#/components/schemas/UpdateSpanEvent'
           required:
             - type
         - type: object
@@ -1997,19 +2336,8 @@ components:
                 type:
                   type: string
                   enum:
-                    - span-create
-            - $ref: '#/components/schemas/CreateSpanEvent'
-          required:
-            - type
-        - type: object
-          allOf:
-            - type: object
-              properties:
-                type:
-                  type: string
-                  enum:
-                    - span-update
-            - $ref: '#/components/schemas/UpdateSpanEvent'
+                    - event-create
+            - $ref: '#/components/schemas/CreateEventEvent'
           required:
             - type
         - type: object
@@ -2332,13 +2660,20 @@ components:
       properties:
         id:
           type: string
+          description: UUID v4 that identifies the event
         timestamp:
           type: string
-        metadata: {}
+          description: >-
+            Datetime (ISO 8601) of event creation in client. Should be as close
+            to actual event creation in client as possible, this timestamp will
+            be used for ordering of events in future release. Resolution:
+            milliseconds (required), microseconds (optimal).
+        metadata:
+          nullable: true
+          description: Optional. Metadata field used by the Langfuse SDKs for debugging.
       required:
         - id
         - timestamp
-        - metadata
     TraceEvent:
       title: TraceEvent
       type: object
@@ -2798,6 +3133,19 @@ components:
         - prompt
       allOf:
         - $ref: '#/components/schemas/BasePrompt'
+    ScoreConfigs:
+      title: ScoreConfigs
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/ScoreConfig'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object

web/public/generated/postman/collection.json

@@ -106,6 +106,57 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get dataset items",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/dataset-items?datasetName=&sourceTraceId=&sourceObservationId=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "dataset-items"
+              ],
+              "query": [
+                {
+                  "key": "datasetName",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "sourceTraceId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "sourceObservationId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -160,25 +211,26 @@
           "request": {
             "description": "Get all datasets",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets?page=&limit=",
+              "raw": "{{baseUrl}}/api/public/v2/datasets?page=&limit=",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets"
               ],
               "query": [
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 }
               ],
               "variable": []
@@ -196,13 +248,14 @@
           "request": {
             "description": "Get a dataset and its items",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets/:datasetName",
+              "raw": "{{baseUrl}}/api/public/v2/datasets/:datasetName",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets",
                 ":datasetName"
               ],
@@ -228,13 +281,14 @@
           "request": {
             "description": "Create a dataset",
             "url": {
-              "raw": "{{baseUrl}}/api/public/datasets",
+              "raw": "{{baseUrl}}/api/public/v2/datasets",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "datasets"
               ],
               "query": [],
@@ -257,7 +311,7 @@
         },
         {
           "_type": "endpoint",
-          "name": "Get Runs",
+          "name": "Get Run",
           "request": {
             "description": "Get a dataset run and its items",
             "url": {
@@ -293,6 +347,50 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Runs",
+          "request": {
+            "description": "Get a dataset runs",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/datasets/:datasetName/runs?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "datasets",
+                ":datasetName",
+                "runs"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "page number, starts at 1"
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "limit of items per page"
+                }
+              ],
+              "variable": [
+                {
+                  "key": "datasetName",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -337,7 +435,7 @@
           "_type": "endpoint",
           "name": "Batch",
           "request": {
-            "description": "Batched ingestion for Langfuse Tracing",
+            "description": "Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.\n\nNotes:\n\n- Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.\n- The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.",
             "url": {
               "raw": "{{baseUrl}}/api/public/ingestion",
               "host": [
@@ -356,7 +454,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
+              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ],\n    \"metadata\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -658,12 +756,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 }
               ],
               "variable": []
@@ -711,6 +809,81 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Score Configs",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get all score configs",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": "Page number, starts at 1."
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get By Id",
+          "request": {
+            "description": "Get a score config",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/score-configs/:configId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "score-configs",
+                ":configId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "configId",
+                  "value": "",
+                  "description": "The unique langfuse identifier of a score config"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -755,7 +928,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -804,6 +977,11 @@
                   "key": "value",
                   "value": "",
                   "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "scoreIds",
+                  "value": "",
+                  "description": "Comma-separated list of score IDs to limit the results to."
                 }
               ],
               "variable": []

web/scripts/observations-backfill-calculated-cost.ts

@@ -0,0 +1,236 @@
+import "dotenv/config";
+
+import { z } from "zod";
+import { prisma, Prisma } from "@langfuse/shared/src/db";
+
+const BackfillCalculatedGenerationArgsSchema = z
+  .object({
+    batchSize: z.coerce.number().optional().default(5_000),
+    maxRowsToProcess: z.coerce.number().optional().default(Infinity), // Default to process all rows
+    maxDate: z.coerce.date().optional().default(new Date()), // Default to today
+  })
+  .strict();
+
+const backfillCalculatedGenerationCost = async () => {
+  let previousTimeout;
+  try {
+    const args = parseArgs(process.argv.slice(2));
+    const { batchSize, maxRowsToProcess, maxDate } = args;
+
+    log("Starting backfillCalculatedGenerationCost with params", args);
+
+    // Set the statement timeout
+    const newTimeout = "19min";
+    previousTimeout = await updateStatementTimeout(newTimeout, previousTimeout);
+
+    // Drop column if it exists and add temporary column
+    await addTemporaryColumnIfNotExists();
+
+    let currentDateCutoff = maxDate.toISOString();
+    let totalRowsProcessed = 0;
+
+    log("Starting batch update loop...");
+
+    // Step 3: Batch update in a loop
+    while (true) {
+      log(`Starting batch update for generations before: ${currentDateCutoff}`);
+      const startDate = Date.now();
+      const batchUpdate = await prisma.$queryRaw<
+        { start_time: Date }[]
+      >(Prisma.sql`
+        WITH batch AS (
+            SELECT o.id,
+                o.start_time,
+                o.prompt_tokens,
+                o.completion_tokens,
+                o.total_tokens,
+                o.input_cost,
+                o.output_cost,
+                o.total_cost,
+                m.id AS model_id,
+                m.input_price,
+                m.output_price,
+                m.total_price
+            FROM observations o
+            LEFT JOIN LATERAL (
+                SELECT models.id,
+                    models.input_price,
+                    models.output_price,
+                    models.total_price
+                FROM models
+                WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+                  AND models.model_name = o.internal_model
+                  AND (models.start_date < o.start_time OR models.start_date IS NULL)
+                  AND o.unit = models.unit
+                ORDER BY models.project_id, models.start_date DESC NULLS LAST
+                LIMIT 1
+            ) m ON true
+            WHERE
+                start_time <= ${currentDateCutoff}::TIMESTAMP WITH TIME ZONE AT TIME ZONE 'UTC'
+              	AND (internal_model IS NOT NULL
+                      OR input_cost IS NOT NULL
+                      OR output_cost IS NOT NULL
+                      OR total_cost IS NOT NULL) 
+            ORDER BY
+              start_time DESC
+            LIMIT ${batchSize}
+        ),
+        updated_batch AS (
+            UPDATE observations o
+            SET calculated_input_cost = COALESCE(batch.input_cost, batch.prompt_tokens::numeric * batch.input_price),
+                calculated_output_cost = COALESCE(batch.output_cost, batch.completion_tokens::numeric * batch.output_price),
+                calculated_total_cost = COALESCE(
+                    batch.total_cost,
+                    CASE
+                        WHEN batch.total_price IS NOT NULL AND batch.total_tokens IS NOT NULL THEN batch.total_price * batch.total_tokens::numeric
+                        ELSE batch.prompt_tokens::numeric * batch.input_price + batch.completion_tokens::numeric * batch.output_price
+                    END
+                ),
+                internal_model_id = batch.model_id,
+                tmp_has_calculated_cost = TRUE
+            FROM batch
+            WHERE o.id = batch.id
+            RETURNING o.id
+        )
+        -- Get the last id of the updated batch
+        SELECT start_time FROM batch LIMIT 1 OFFSET ${batchSize - 1};
+      `);
+
+      log(`Batch update completed in ${Date.now() - startDate} ms`);
+
+      if (!batchUpdate[0]?.start_time) {
+        log(
+          `No more rows to process, breaking loop after ${totalRowsProcessed.toLocaleString()} rows processed.`,
+        );
+
+        break;
+      }
+
+      currentDateCutoff = batchUpdate[0]?.start_time.toISOString();
+      totalRowsProcessed += batchSize;
+
+      log(
+        `Total rows processed after increment: ${totalRowsProcessed.toLocaleString()} rows`,
+      );
+      if (maxRowsToProcess && totalRowsProcessed >= maxRowsToProcess) {
+        log(
+          `Max rows to process reached: ${maxRowsToProcess.toLocaleString()}, breaking loop.`,
+        );
+
+        break;
+      }
+    }
+
+    log("✅ Finished batch update loop.");
+
+    // Drop the temporary column
+    log("Dropping temporary column...");
+    await prisma.$executeRaw`ALTER TABLE observations DROP COLUMN IF EXISTS tmp_has_calculated_cost;`;
+    log("✅ Dropped temporary column");
+
+    log("✅ Finished backfillCalculatedGenerationCost");
+  } catch (err) {
+    console.error("Error executing script", err);
+  } finally {
+    // Reset the statement timeout to two minutes
+    await prisma.$executeRawUnsafe(
+      `SET statement_timeout = '${previousTimeout}';`,
+    );
+    log(
+      `Reset statement_timeout to ${previousTimeout}. Current statement_timeout: ${JSON.stringify(
+        await prisma.$queryRaw(Prisma.sql`SHOW statement_timeout;`),
+      )}`,
+    );
+
+    // Disconnect from the database
+    await prisma.$disconnect();
+    log("Disconnected from the database.");
+  }
+};
+
+function parseArgs(args: string[]) {
+  try {
+    const namedArgs: Record<string, string | boolean> = {};
+    for (let i = 0; i < args.length; i++) {
+      if (args[i].startsWith("--")) {
+        const key = args[i].slice(2);
+        const value =
+          args[i + 1] && !args[i + 1].startsWith("--") ? args[i + 1] : true;
+        namedArgs[key] = value;
+        if (value !== true) i++; // Skip the next argument if it was used as a value
+      }
+    }
+
+    return BackfillCalculatedGenerationArgsSchema.parse(namedArgs);
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      console.error("Validation error:", error.errors);
+    } else {
+      console.error("An unexpected error occurred:", error);
+    }
+
+    process.exit(1);
+  }
+}
+
+async function addTemporaryColumnIfNotExists() {
+  const columnExists = await prisma.$queryRaw<{ column_exists: boolean }[]>(
+    Prisma.sql`
+            SELECT EXISTS (
+                SELECT 1
+                FROM information_schema.columns
+                WHERE table_name = 'observations'
+                AND column_name = 'tmp_has_calculated_cost'
+            ) AS column_exists;
+        `,
+  );
+  if (!columnExists[0]?.column_exists) {
+    await prisma.$executeRaw`ALTER TABLE observations ADD COLUMN tmp_has_calculated_cost BOOLEAN DEFAULT FALSE;`;
+    log("✅ Added temporary column tmp_has_calculated_cost");
+  } else {
+    log(
+      "⚠️ Temporary column tmp_has_calculated_cost already exists. Continuing...",
+    );
+  }
+}
+
+type StatementTimeout = {
+  statement_timeout: string;
+};
+
+async function updateStatementTimeout(
+  newTimeout: string,
+  previousTimeout: any,
+) {
+  const [{ statement_timeout: previousTimeoutRead }] = await prisma.$queryRaw<
+    StatementTimeout[]
+  >(Prisma.sql`SHOW statement_timeout;`);
+
+  log(`Current statement_timeout read from DB: ${previousTimeoutRead}`);
+
+  if (!previousTimeoutRead || previousTimeoutRead === newTimeout) {
+    // If the statement_timeout is already set to 19 minutes, assume it was set by this script and reset it to 2 minutes
+    previousTimeout = "2min";
+  } else {
+    previousTimeout = previousTimeoutRead;
+  }
+
+  log(`Setting statement_timeout to ${newTimeout} minutes...`);
+
+  await prisma.$executeRawUnsafe(`SET statement_timeout = '${newTimeout}';`);
+
+  log(
+    `Updated statement_timeout. Current statement_timeout: ${JSON.stringify(
+      await prisma.$queryRaw(Prisma.sql`SHOW statement_timeout;`),
+    )}`,
+  );
+
+  return previousTimeout;
+}
+
+function log(message: string, ...args: any[]) {
+  console.log(new Date().toISOString(), " - ", message, ...args);
+}
+
+// Execute the script
+backfillCalculatedGenerationCost();