chore: release v2.55.0

feat(auth): password reset for email/password users via email-link auth (#2276 )
fix(security): add sentry to csp headers
2024-06-14 16:06:12 +02:00 · 2024-06-14 14:00:45 +00:00 · 2024-06-14 14:28:03 +02:00 · 2024-06-14 12:15:59 +00:00 · 2024-06-12 19:11:33 +02:00 · 2024-06-12 19:03:45 +02:00
276 changed files with 10419 additions and 4008 deletions
@@ -0,0 +1,14 @@
+{
+	"name": "langfuse dev",
+    "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+        "version": "latest",
+        "moby": true
+        }
+    },
+    "forwardPorts": [3000],
+
+    "onCreateCommand": "npm install -g pnpm@8.15.5",
+    "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
+}
@@ -34,10 +34,6 @@ S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 S3_REGION=

-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Set during docker build of application
 # Used to disable environment verification at build time
 # DOCKER_BUILD=1
@@ -47,5 +43,5 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"

 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
-# openssl rand -base64 32 used only here
+# openssl rand -hex 32 used only here
 ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 
@@ -13,11 +13,6 @@ NEXTAUTH_URL="http://localhost:3000"
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"

-
-# Prompt playground
-OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
-
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
@@ -43,6 +43,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_GOOGLE_CLIENT_ID=
 # AUTH_GOOGLE_CLIENT_SECRET=
 # AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GOOGLE_ALLOWED_DOMAINS=langfuse.com,google.com # optional allowlist of workspace domains that can sign in via Google
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
@@ -58,6 +59,15 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_CLIENT_ID=
+# AUTH_CUSTOM_CLIENT_SECRET=
+# AUTH_CUSTOM_ISSUER=
+# AUTH_CUSTOM_NAME=
+# AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false

 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -118,10 +128,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_TURNSTILE_SITE_KEY=
 # TURNSTILE_SECRET_KEY=

-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=

@@ -40,25 +40,34 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v3

-      - name: Build Web Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./web/Dockerfile
-          push: false
+      - name: Build and run both images from compose
+        run: |
+          docker compose -f docker-compose.build.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections

-      - name: Build Worker Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: false
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Ensure no unhealthy status
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi

  tests-web:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        node-version: [20]
+        postgres-version: [12, 15]
    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@master
@@ -89,6 +98,9 @@ jobs:
        run: |
          docker compose -f docker-compose.dev.yml up -d
          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}

      - name: Seed DB
        run: |
@@ -109,6 +121,7 @@ jobs:
    strategy:
      matrix:
        node-version: [20]
+        postgres-version: [12, 15]
    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@master
@@ -139,6 +152,7 @@ jobs:
        run: |
          docker compose -f docker-compose.dev.yml up -d
          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps

      - name: Seed DB
        run: |
@@ -176,6 +190,7 @@ jobs:
      - name: Run + migrate
        run: |
          docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
          sleep 5 # Wait for PostgreSQL to accept connections

      - name: Seed DB
@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}
@@ -0,0 +1,39 @@
+name: Snyk Container
+on: push
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build a Docker image
+        run: docker compose -f docker-compose.build.yml up -d
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-server
+          args: --file=web/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: web
+
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
+        continue-on-error: true
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: langfuse-worker
+          args: --file=worker/Dockerfile
+
+      - name: Upload result to GitHub Code Scanning
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: snyk.sarif
+          category: worker
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Commit will proceed
+    else
+        echo "Commit to $protected_branch branch has been canceled."
+        exit 1 # Commit will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the commit
+exit 0
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get the current branch
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Define the protected branch
+protected_branch="main"
+
+# Check if the current branch is the protected branch
+if [ "$current_branch" = "$protected_branch" ]; then
+    echo "🚨 You are about to push to the $protected_branch branch. Are you sure? (y/n)"
+    read -r answer < /dev/tty
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        exit 0 # Push will proceed
+    else
+        echo "Push to $protected_branch branch has been canceled."
+        exit 1 # Push will be blocked
+    fi
+fi
+
+# If not the protected branch, proceed with the push
+exit 0
@@ -151,6 +151,8 @@ Requirements
 - Node.js 20 as specified in the [.nvmrc](.nvmrc)
 - Docker to run the database locally

+**Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
+
 **Steps**

 1. Fork the repository and clone it locally
@@ -191,6 +193,16 @@ Requirements
    pnpm run dev
   ```

+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart

 - Available packages and their dependencies
@@ -358,7 +370,9 @@ The background color of the following component will be `hsl(var(--primary))` an

 By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.

-## Using secrets stored in 1Password
+## Maintainers
+
+### Using secrets stored in 1Password

 When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.

@@ -106,22 +106,26 @@ See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Lan

 ### Integrations

-| Integration                                                      | Supports      | Description                                                 |
-| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
+| Integration                                              | Supports                   | Description                                                                                                                                      |
+| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |

-External projects/packages that integrate with Langfuse:
+Packages that integrate with Langfuse:

-| Name                                                            | Description                                                                                                                                      |
-| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
-| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
+| Name                                                       | Description                                                                                                             |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
+| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
+| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
+| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |

 ## Questions and feedback

@@ -3,9 +3,9 @@ Join us in scaling Langfuse in Berlin, Germany. We are an open source company, w

 _Open Roles_

- Backend Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
- Product Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
- Developer Advocate, 60-100k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel

 _More Info_

@@ -20,6 +20,11 @@ services:
      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

  worker:
    build:
@@ -41,6 +46,11 @@ services:
      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

  redis:
    image: redis:7.2.4
@@ -1,5 +1,3 @@
-version: "3.5"
-
 services:
  redis:
    image: redis:7.2.4
@@ -10,7 +8,7 @@ services:
      - 6379:6379

  db:
-    image: postgres
+    image: postgres:${POSTGRES_VERSION:-latest}
    restart: always
    command: ["postgres", "-c", "log_statement=all"]
    environment:
@@ -2,7 +2,7 @@ version: "3.5"

 services:
  langfuse-server:
-    image: ghcr.io/langfuse/langfuse:latest
+    image: langfuse/langfuse:2
    depends_on:
      db:
        condition: service_healthy
@@ -33,7 +33,7 @@
    "@repo/eslint-config": "*",
    "@repo/typescript-config": "*",
    "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
@@ -2,14 +2,16 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
-import { prisma } from "@langfuse/shared/src/db";
+import { type SsoConfig, prisma } from "@langfuse/shared/src/db";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { env } from "../env";
+import { CustomSSOProvider } from "@langfuse/shared/src/server/auth";

 // Local cache for SSO configurations
 let cachedSsoConfigs: {
@@ -25,14 +27,36 @@ let cachedSsoConfigs: {
 async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
  if (!isEeAvailable) return [];
  const CACHE_TTL = 60 * 1000; // 1 minute
+  const DB_MAX_WAIT = 2000; // 2 seconds
+  const DB_TIMEOUT = 3000; // 3 seconds

  // Set/refresh the cache if it's empty or expired
  if (
    cachedSsoConfigs === null ||
    Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
  ) {
+    // findMany with custom timeout via $transaction
+    let dbConfigs: SsoConfig[] = [];
+    try {
+      dbConfigs = await prisma.$transaction(
+        async (prisma) => prisma.ssoConfig.findMany(),
+        {
+          maxWait: DB_MAX_WAIT,
+          timeout: DB_TIMEOUT,
+        }
+      );
+    } catch (e) {
+      // cache empty array to prevent repeated DB calls on error
+      cachedSsoConfigs = {
+        data: [],
+        timestamp: Date.now(),
+      };
+
+      // caught and logged in the caller
+      throw e;
+    }
+
    // transform into zod object
-    const dbConfigs = await prisma.ssoConfig.findMany();
    const parsedSsoConfigs = dbConfigs
      .map((v) => {
        try {
@@ -144,6 +168,18 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
      ...provider.authConfig,
      clientSecret: decrypt(provider.authConfig.clientSecret),
    });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "custom")
+    return CustomSSOProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
  else {
    // Type check to ensure we handle all providers
    // eslint-disable-next-line no-unused-vars
@@ -64,11 +64,38 @@ export const AzureAdProviderSchema = base.extend({
    .nullish(),
 });

+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const CustomProviderSchema = base.extend({
+  authProvider: z.literal("custom"),
+  authConfig: z
+    .object({
+      name: z.string(),
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;

 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
  GoogleProviderSchema,
@@ -76,6 +103,8 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
  Auth0ProviderSchema,
  OktaProviderSchema,
  AzureAdProviderSchema,
+  CognitoProviderSchema,
+  CustomProviderSchema,
 ]);

 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;
@@ -209,8 +209,8 @@ types:
      - ARCHIVED
  ScoreSource:
    enum:
+      - ANNOTATION
      - API
-      - REVIEW
      - EVAL

 errors:
@@ -7,28 +7,52 @@ service:
  base-path: /api/public
  endpoints:
    batch:
-      docs: Batched ingestion for Langfuse Tracing
+      docs: Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
      method: POST
      path: /ingestion
      request:
        name: IngestionRequest
        body:
          properties:
-            batch: list<IngestionEvent>
-      response: IngestionResponse # will be reportet as 200 response, but endpoint returns 207
+            batch:
+              type: list<IngestionEvent>
+              docs: "Batch of tracing events to be ingested. Discriminated by attribute `type`."
+            metadata:
+              type: optional<unknown>
+              docs: Optional. Metadata field used by the Langfuse SDKs for debugging.
+      response:
+        type: IngestionResponse
+        status-code: 207

 types:
  IngestionEvent:
    discriminant: "type"
    union:
-      trace-create: TraceEvent
-      score-create: ScoreEvent
-      event-create: CreateEventEvent
-      generation-create: CreateGenerationEvent
-      generation-update: UpdateGenerationEvent
-      span-create: CreateSpanEvent
-      span-update: UpdateSpanEvent
-      sdk-log: SDKLogEvent
+      trace-create:
+        type: TraceEvent
+        docs: Creates a new trace. Upserts on id for updates if trace with id exists.
+      score-create:
+        type: ScoreEvent
+        docs: Creates a new score. Upserts on id for updates if score with id exists.
+      span-create:
+        type: CreateSpanEvent
+        docs: Creates a new span.
+      span-update:
+        type: UpdateSpanEvent
+        docs: Updates span based on id.
+      generation-create:
+        type: CreateGenerationEvent
+        docs: Creates a new generation.
+      generation-update:
+        type: UpdateGenerationEvent
+        docs: Updates a generation based on id.
+      event-create:
+        type: CreateEventEvent
+        docs: Creates an event.
+
+      sdk-log:
+        type: SDKLogEvent
+        docs: Langfuse SDKs only, used for debugging purposes.

      # both are legacy
      observation-create:
@@ -162,9 +186,15 @@ types:

  BaseEvent:
    properties:
-      id: string
-      timestamp: string
-      metadata: unknown
+      id:
+        type: string
+        docs: UUID v4 that identifies the event
+      timestamp:
+        type: string
+        docs: "Datetime (ISO 8601) of event creation in client. Should be as close to actual event creation in client as possible, this timestamp will be used for ordering of events in future release. Resolution: milliseconds (required), microseconds (optimal)."
+      metadata:
+        type: optional<unknown>
+        docs: Optional. Metadata field used by the Langfuse SDKs for debugging.

  TraceEvent:
    extends: BaseEvent
@@ -48,12 +48,25 @@ types:
      date: date
      countTraces: integer
      countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
      usage: list<UsageByModel>
  UsageByModel:
    docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
    properties:
      model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD
@@ -70,14 +70,24 @@ types:
      name: string
      prompt: list<ChatMessage>
      config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.

  CreateTextPromptRequest:
    properties:
      name: string
      prompt: string
      config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.

  Prompt:
    union:
@@ -89,7 +99,12 @@ types:
      name: string
      version: integer
      config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.

  ChatMessage:
    properties:
@@ -31,7 +31,7 @@ service:
            type: optional<datetime>
            docs: Retrieve only scores newer than this datetime (ISO 8601).
          source:
-            type: commons.ScoreSource
+            type: optional<commons.ScoreSource>
            docs: Retrieve only scores from a specific source.
          operator:
            type: optional<string>
@@ -39,6 +39,9 @@ service:
          value:
            type: optional<double>
            docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
      response: Scores
    get-by-id:
      docs: Get a score
@@ -7,7 +7,7 @@ service:
  base-path: /api/public
  endpoints:
    get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
      method: GET
      path: /sessions/{sessionId}
      path-parameters:
@@ -30,6 +30,7 @@ service:
            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
          userId: optional<string>
          name: optional<string>
+          sessionId: optional<string>
          fromTimestamp:
            type: optional<datetime>
            docs: Retrieve only traces newer than this datetime (ISO 8601).
@@ -1,4 +1,4 @@
 {
  "organization": "finto",
-  "version": "0.21.0"
+  "version": "0.30.3"
 }
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "2.45.0",
+  "version": "2.55.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -17,26 +17,36 @@
    "db:seed": "turbo run db:seed",
    "db:seed:examples": "turbo run db:seed:examples",
    "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
    "build": "turbo run build",
    "start": "turbo run start",
    "dev": "turbo run dev",
    "lint": "turbo run lint",
    "test": "turbo run test",
    "models:migrate": "turbo run models:migrate",
-    "release": "dotenv -e ../.env -- release-it"
+    "release": "dotenv -e ../.env -- release-it",
+    "prepare": "husky"
  },
  "devDependencies": {
    "@release-it/bumper": "^6.0.1",
+    "braces": "3.0.3",
    "dotenv-cli": "^7.4.2",
+    "husky": "^9.0.11",
    "prettier": "^3.2.5",
-    "release-it": "^17.2.1",
+    "release-it": "^17.3.0",
    "turbo": "^1.13.3"
  },
  "release-it": {
    "git": {
      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
+      "tagName": "v${version}",
+      "commitArgs": [
+        "--no-verify"
+      ],
+      "pushArgs": [
+        "--no-verify"
+      ]
    },
    "plugins": {
      "@release-it/bumper": {
@@ -9,7 +9,7 @@
  ],
  "devDependencies": {
    "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
    "@vercel/style-guide": "^6.0.0",
    "eslint-config-next": "^14.2.3",
    "eslint-config-prettier": "^9.1.0",
@@ -52,6 +52,8 @@
    "bcryptjs": "^2.4.3",
    "kysely": "^0.27.3",
    "langchain": "^0.1.36",
+    "lodash": "^4.17.21",
+    "next-auth": "^4.24.7",
    "prisma-extension-kysely": "^2.1.0",
    "zod": "^3.22.4",
    "zod-to-json-schema": "^3.22.5"
@@ -59,17 +61,17 @@
  "devDependencies": {
    "@repo/eslint-config": "*",
    "@repo/typescript-config": "*",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
    "@types/node": "^20.11.29",
    "@types/pg": "^8.11.6",
    "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
    "eslint-plugin-prettier": "^5.1.3",
    "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.0.3",
+    "nodemon": "^3.1.3",
    "prettier": "^3.2.5",
    "prisma": "^5.13.0",
    "prisma-erd-generator": "^1.11.2",
@@ -25,22 +25,17 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
    API: "API",
-    REVIEW: "REVIEW",
    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const DatasetStatus = {
    ACTIVE: "ACTIVE",
    ARCHIVED: "ARCHIVED"
@@ -157,6 +152,7 @@ export type EvalTemplate = {
    version: number;
    prompt: string;
    model: string;
+    provider: string;
    model_params: unknown;
    vars: Generated<string[]>;
    output_schema: unknown;
@@ -204,8 +200,12 @@ export type LlmApiKeys = {
    created_at: Generated<Timestamp>;
    updated_at: Generated<Timestamp>;
    provider: string;
+    adapter: string;
    display_secret_key: string;
    secret_key: string;
+    base_url: string | null;
+    custom_models: Generated<string[]>;
+    with_default_models: Generated<boolean>;
    project_id: string;
 };
 export type MembershipInvitation = {
@@ -246,6 +246,7 @@ export type Observation = {
    status_message: string | null;
    version: string | null;
    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
    model: string | null;
    internal_model: string | null;
    modelParameters: unknown | null;
@@ -303,14 +304,6 @@ export type PosthogIntegration = {
    enabled: boolean;
    created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
    id: string;
    created_at: Generated<Timestamp>;
@@ -351,6 +344,24 @@ export type Score = {
    comment: string | null;
    trace_id: string;
    observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
    id: string;
@@ -446,10 +457,10 @@ export type DB = {
    observations: Observation;
    observations_view: ObservationView;
    posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
    project_memberships: ProjectMembership;
    projects: Project;
    prompts: Prompt;
+    score_configs: ScoreConfig;
    scores: Score;
    Session: Session;
    sso_configs: SsoConfig;
@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");
@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';
@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";
@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;
@@ -0,0 +1,4 @@
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "audit_logs_updated_at_idx" ON "audit_logs"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_created_at_idx" ON "dataset_items"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_updated_at_idx" ON "dataset_items"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_created_at_idx" ON "dataset_run_items"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_updated_at_idx" ON "dataset_run_items"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_created_at_idx" ON "dataset_runs"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_updated_at_idx" ON "dataset_runs"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_created_at_idx" ON "datasets"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_updated_at_idx" ON "datasets"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_updated_at_idx" ON "job_executions"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_updated_at_idx" ON "observations"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_created_at_idx" ON "prompts"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_updated_at_idx" ON "prompts"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_created_at_idx" ON "score_configs"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_updated_at_idx" ON "score_configs"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_created_at_idx" ON "scores"("created_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_updated_at_idx" ON "scores"("updated_at");
@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "trace_sessions_updated_at_idx" ON "trace_sessions"("updated_at");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';
@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";
@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";
@@ -0,0 +1,8 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+-- remove model if added by faulty migration, context: https://github.com/langfuse/langfuse/issues/2266
+DELETE FROM models WHERE id in ('clx30djsn0000w9mzebiv41we', 'clx30hkrx0000w9mz7lqi0ial');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';
@@ -0,0 +1,24 @@
+BEGIN;
+
+  ALTER TABLE "llm_api_keys" 
+    ADD COLUMN "base_url" TEXT,
+    ADD COLUMN "adapter" TEXT,
+    ADD COLUMN "custom_models" TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+    ADD COLUMN "with_default_models" BOOLEAN NOT NULL DEFAULT true;
+
+  UPDATE "llm_api_keys"
+    SET "adapter" = "provider";
+
+  ALTER TABLE "llm_api_keys" 
+    ALTER COLUMN "adapter" SET NOT NULL;
+
+  ALTER TABLE "eval_templates"
+    ADD COLUMN "provider" TEXT;
+  
+  UPDATE "eval_templates"
+    SET "provider" = 'openai';
+  
+  ALTER TABLE "eval_templates"
+    ALTER COLUMN "provider" SET NOT NULL;
+
+COMMIT;
@@ -0,0 +1,51 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	md5(random()::text || clock_timestamp()::text || s.project_id::text)::uuid AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND config_id IS NULL
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_project_id_prompt_id_idx" ON "observations"("project_id", "prompt_id");
@@ -114,6 +114,7 @@ model Project {
    LlmApiKeys         LlmApiKeys[]
    PosthogIntegration PosthogIntegration[]
    Score              Score[]
+    scoreConfig        ScoreConfig[]

    @@map("projects")
 }
@@ -143,9 +144,13 @@ model LlmApiKeys {
    createdAt DateTime @default(now()) @map("created_at")
    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")

-    provider         String
-    displaySecretKey String @map("display_secret_key")
-    secretKey        String @map("secret_key")
+    provider          String
+    adapter           String // This controls the interface that is used to connect with the LLM, e.g. 'openai' or 'anthropic'
+    displaySecretKey  String   @map("display_secret_key")
+    secretKey         String   @map("secret_key")
+    baseURL           String?  @map("base_url")
+    customModels      String[] @default([]) @map("custom_models")
+    withDefaultModels Boolean  @default(true) @map("with_default_models")

    projectId String  @map("project_id")
    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -205,6 +210,7 @@ model TraceSession {
    @@id([id, projectId])
    @@index([projectId])
    @@index([createdAt])
+    @@index([updatedAt])
    @@map("trace_sessions")
 }

@@ -231,9 +237,8 @@ model Trace {
    createdAt  DateTime      @default(now()) @map("created_at")
    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")

-    DatasetRunItems DatasetRunItems[]
-    DatasetItem     DatasetItem[]
-    JobExecution    JobExecution[]
+    DatasetItem  DatasetItem[]
+    JobExecution JobExecution[]

    @@index([projectId])
    @@index([sessionId])
@@ -292,24 +297,24 @@ model Observation {
    statusMessage       String?          @map("status_message")
    version             String?
    createdAt           DateTime         @default(now()) @map("created_at")
+    updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")

    // GENERATION ONLY
    model               String?
-    internalModel       String?           @map("internal_model")
+    internalModel       String?       @map("internal_model")
    modelParameters     Json?
    input               Json?
    output              Json?
-    promptTokens        Int               @default(0) @map("prompt_tokens")
-    completionTokens    Int               @default(0) @map("completion_tokens")
-    totalTokens         Int               @default(0) @map("total_tokens")
+    promptTokens        Int           @default(0) @map("prompt_tokens")
+    completionTokens    Int           @default(0) @map("completion_tokens")
+    totalTokens         Int           @default(0) @map("total_tokens")
    unit                String?
-    inputCost           Decimal?          @map("input_cost")
-    outputCost          Decimal?          @map("output_cost")
-    totalCost           Decimal?          @map("total_cost")
-    completionStartTime DateTime?         @map("completion_start_time")
-    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    inputCost           Decimal?      @map("input_cost")
+    outputCost          Decimal?      @map("output_cost")
+    totalCost           Decimal?      @map("total_cost")
+    completionStartTime DateTime?     @map("completion_start_time")
+    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
    derivedDatasetItems DatasetItem[]
-    datasetRunItems     DatasetRunItems[]

    promptId String? @map("prompt_id")
    prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@ -323,10 +328,12 @@ model Observation {
    @@index([type])
    @@index(startTime)
    @@index(createdAt)
+    @@index(updatedAt)
    @@index(projectId)
    @@index(parentObservationId)
    @@index(model)
    @@index(internalModel)
+    @@index([projectId, promptId])
    @@index(promptId)
    @@index([projectId, startTime, type])
    @@map("observations")
@@ -403,46 +410,64 @@ model Score {
    comment       String?
    traceId       String         @map("trace_id")
    observationId String?        @map("observation_id")
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    createdAt     DateTime       @default(now()) @map("created_at")
+    updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
    JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)

    @@unique([id, projectId]) // used for upserts via prisma
    @@index(timestamp)
    @@index([value])
    @@index([projectId])
+    @@index([projectId, name])
    @@index([authorUserId])
+    @@index([configId])
    @@index([traceId], type: Hash)
    @@index([observationId], type: Hash)
    @@index([source])
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("scores")
 }

 enum ScoreSource {
+    ANNOTATION
    API
-    REVIEW
    EVAL
 }

-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@index([createdAt])
+    @@index([updatedAt])
+    @@map("score_configs")
 }

-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+    BOOLEAN
 }

 model CronJobs {
@@ -468,6 +493,8 @@ model Dataset {

    @@unique([projectId, name])
    @@index([projectId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("datasets")
 }

@@ -487,8 +514,11 @@ model DatasetItem {
    updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
    datasetRunItems     DatasetRunItems[]

+    @@index([sourceTraceId], type: Hash)
    @@index([sourceObservationId], type: Hash)
    @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("dataset_items")
 }

@@ -510,26 +540,28 @@ model DatasetRuns {

    @@unique([datasetId, name])
    @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("dataset_runs")
 }

 model DatasetRunItems {
-    id            String       @id @default(cuid())
-    datasetRunId  String       @map("dataset_run_id")
-    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String       @map("dataset_item_id")
-    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime     @default(now()) @map("created_at")
-    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String      @map("trace_id")
+    observationId String?     @map("observation_id")
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")

    @@index([datasetRunId], type: Hash)
    @@index([datasetItemId], type: Hash)
    @@index([observationId], type: Hash)
    @@index([traceId])
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("dataset_run_items")
 }

@@ -572,6 +604,8 @@ model Prompt {
    @@index([projectId, name, version])
    @@index([projectId, id])
    @@index([projectId])
+    @@index([createdAt])
+    @@index([updatedAt])
    @@index([tags(ops: ArrayOps)], type: Gin)
    @@map("prompts")
 }
@@ -619,6 +653,7 @@ model AuditLog {

    @@index([projectId])
    @@index([createdAt])
+    @@index([updatedAt])
    @@map("audit_logs")
 }

@@ -634,6 +669,7 @@ model EvalTemplate {
    version          Int
    prompt           String
    model            String
+    provider         String
    modelParams      Json               @map("model_params")
    vars             String[]           @default([])
    outputSchema     Json               @map("output_schema")
@@ -712,6 +748,11 @@ model JobExecution {
    @@index([projectId, status])
    @@index([projectId, id])
    @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
+    @@index([updatedAt])
    @@map("job_executions")
 }

@@ -4,6 +4,7 @@ import {
  type Prisma,
  ObservationType,
  ScoreSource,
+  ScoreDataType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -12,6 +13,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";

 const LOAD_TRACE_VOLUME = 10_000;

@@ -38,6 +40,7 @@ async function main() {
      name: "Demo User",
      email: "demo@langfuse.com",
      password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
    },
  });

@@ -143,6 +146,11 @@ async function main() {
      });
    }

+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
    const promptIds = await generatePromptsForProject([project1, project2]);

    const envTags = [null, "development", "staging", "production"];
@@ -156,7 +164,8 @@ async function main() {
      colorTags,
      project1,
      project2,
-      promptIds
+      promptIds,
+      configIdsAndNames
    );

    console.log(
@@ -165,6 +174,25 @@ async function main() {

    await uploadObjects(traces, observations, scores, sessions, events);

+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+          adapter: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
    // add eval objects
    const evalTemplate = await prisma.evalTemplate.upsert({
      where: {
@@ -182,6 +210,7 @@ async function main() {
          "Please evaluate the toxicity of the following text {{input}} {{output}}",
        model: "gpt-3.5-turbo",
        vars: ["input", "output"],
+        provider: "openai",
        outputSchema: {
          score: "provide a score between 0 and 1",
          reasoning: "one sentence reasoning for the score",
@@ -434,13 +463,15 @@ function createObjects(
  colorTags: (string | null)[],
  project1: Project,
  project2: Project,
-  promptIds: Map<string, string[]>
+  promptIds: Map<string, string[]>,
+  configIdsAndNames: Map<string, { name: string; id: string }[]>
 ) {
  const traces: Prisma.TraceCreateManyInput[] = [];
  const observations: Prisma.ObservationCreateManyInput[] = [];
  const scores: Prisma.ScoreCreateManyInput[] = [];
  const sessions: Prisma.TraceSessionCreateManyInput[] = [];
  const events: Prisma.ObservationCreateManyInput[] = [];
+  const configs: Prisma.ScoreConfigCreateManyInput[] = [];

  for (let i = 0; i < traceVolume; i++) {
    // print progress to console with a progress bar that refreshes every 10 iterations
@@ -492,17 +523,28 @@ function createObjects(

    traces.push(trace);

+    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const randomIndex = Math.floor(Math.random() * 3);
+    const config =
+      configArray.length >= randomIndex - 1 && configArray[randomIndex];
+    const { name: annotationScoreName, id: configId } = config || {
+      name: "manual-score",
+      id: undefined,
+    };
+
    const traceScores = [
      ...(Math.random() > 0.5
        ? [
            {
              traceId: trace.id,
-              name: "manual-score",
+              name: annotationScoreName,
              value: Math.floor(Math.random() * 3) - 1,
              timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
              projectId,
              authorUserId: `user-${i}`,
+              dataType: ScoreDataType.NUMERIC,
+              ...(configId ? { configId } : {}),
            },
          ]
        : []),
@@ -515,6 +557,7 @@ function createObjects(
              timestamp: traceTs,
              source: ScoreSource.API,
              projectId,
+              dataType: ScoreDataType.NUMERIC,
            },
          ]
        : []),
@@ -753,6 +796,7 @@ function createObjects(
    traces,
    observations,
    scores,
+    configs,
    sessions: uniqueSessions,
    events,
  };
@@ -935,3 +979,80 @@ async function generatePrompts(project: Project) {
  }
  return promptIds;
 }
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
+    new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    })
+  );
+  return projectIdsToConfigs;
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: { name: string; id: string }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({ name: config.name, id: config.id });
+  }
+
+  return configNameAndId;
+}
@@ -0,0 +1,7 @@
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '_prisma_migrations') THEN
+  DELETE FROM _prisma_migrations
+  WHERE migration_name IN ('20240606090858_pricings_add_latest_gemini_models', '20240530212419_model_price_anthropic_via_google_vertex', '20240604133340_backfill_manual_scores');
+ END IF;
+END $$;
@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider, UIModelParams } from "../..";
+import { LLMAdapter } from "../..";

 export const langfuseObjects = [
  "trace",
@@ -77,38 +77,15 @@ export const availableEvalVariables = [
  },
 ];

-export const evalLLMModels: UIModelParams[] = [
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-3.5-turbo", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4-turbo-preview", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-  {
-    provider: { value: ModelProvider.OpenAI, enabled: true },
-    model: { value: "gpt-4o", enabled: true },
-    temperature: { value: 1, enabled: true },
-    maxTemperature: { value: 2, enabled: true },
-    max_tokens: { value: 256, enabled: true },
-    top_p: { value: 1, enabled: true },
-  },
-] as const;
-
-export const EvalModelNames = z.enum([
-  "gpt-3.5-turbo",
-  "gpt-4-turbo-preview",
-  "gpt-4o",
-]);
+export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
+  [LLMAdapter.OpenAI]: [
+    "gpt-4o",
+    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+  ] as const,
+  [LLMAdapter.Anthropic]: [] as const,
+  [LLMAdapter.Azure]: [] as const,
+};

 export const OutputSchema = z.object({
  reasoning: z.string(),
@@ -1,9 +1,9 @@
 import lodash from "lodash";
 import { z } from "zod";

-import { ModelUsageUnit } from "@langfuse/shared";
-import { NonEmptyString, jsonSchema } from "@/src/utils/zod";
-import { ObservationLevel } from "@langfuse/shared";
+import { NonEmptyString, jsonSchema } from "../../utils/zod";
+import { ModelUsageUnit } from "../../constants";
+import { ObservationLevel } from "@prisma/client";

 export const Usage = z.object({
  input: z.number().int().nullish(),
@@ -109,7 +109,7 @@ export const CreateGenerationBody = CreateSpanBody.extend({
      z.string(),
      z
        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
    )
    .nullish(),
  usage: usage,
@@ -131,7 +131,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
      z.string(),
      z
        .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
    )
    .nullish(),
  usage: usage,
@@ -195,7 +195,7 @@ export const LegacyGenerationsCreateSchema = z.object({
  modelParameters: z
    .record(
      z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
    )
    .nullish(),
  prompt: jsonSchema.nullish(),
@@ -219,7 +219,7 @@ export const LegacyGenerationPatchSchema = z.object({
  modelParameters: z
    .record(
      z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
    )
    .nullish(),
  prompt: jsonSchema.nullish(),
@@ -243,7 +243,7 @@ export const LegacyObservationBody = z.object({
  modelParameters: z
    .record(
      z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
    )
    .nullish(),
  input: jsonSchema.nullish(),
@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import {
  ColumnDefinition,
-  type TableNames as TableName,
+  type TableNames,
 } from "./interfaces/tableDefinition";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
  const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
  if (sql === Prisma.empty) {
@@ -43,7 +43,7 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
  filters: FilterState,
  tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
  const internalFilters = filters.map((filter) => {
    // Get column definition to map column to internal name, e.g. "t.id"
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(

 const castValueToPostgresTypes = (
  column: ColumnDefinition,
-  table: TableName
+  table: TableNames
 ) => {
  return column.name === "type" &&
    (table === "observations" ||
@@ -9,6 +9,9 @@ export * from "./filterToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
 export * from "./observationsTable";
+export * from "./features/ingestion/types";
+export * from "./utils/zod";
+export * from "./utils/json";
 export { env } from "./env";

 // llm api
@@ -42,6 +42,7 @@ export const tableNames = [
  "traces_parent_observation_scores",
  "sessions",
  "prompts",
+  "users",
 ] as const;

 export type TableNames = (typeof tableNames)[number];
@@ -1,6 +1,7 @@
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";

 export function generateSecretKey() {
  return `sk-lf-${randomUUID()}`;
@@ -48,3 +49,33 @@ export function createShaHash(privateKey: string, salt: string): string {

  return hash;
 }
+
+export interface CustomSSOUser extends Record<string, any> {
+  email: string;
+  id: string;
+  name: string;
+  verified: boolean;
+}
+
+export function CustomSSOProvider<P extends CustomSSOUser>(
+  options: OAuthUserConfig<P>
+): OAuthConfig<P> {
+  return {
+    id: "custom",
+    name: "CustomSSOProvider",
+    type: "oauth",
+    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
+    authorization: { params: { scope: "openid email profile" } },
+    checks: ["pkce", "state"],
+    idToken: true,
+    profile(profile) {
+      return {
+        id: profile.sub,
+        name: profile.name,
+        email: profile.email,
+        image: null,
+      };
+    },
+    options,
+  };
+}
@@ -17,7 +17,7 @@ import {
  ChatMessageRole,
  LLMFunctionCall,
  ModelParams,
-  ModelProvider,
+  LLMAdapter,
 } from "./types";
 import zodToJsonSchema from "zod-to-json-schema";
 import { JsonOutputFunctionsParser } from "langchain/output_parsers";
@@ -27,7 +27,9 @@ type LLMCompletionParams = {
  modelParams: ModelParams;
  functionCall?: LLMFunctionCall;
  callbacks?: BaseCallbackHandler[];
+  baseURL?: string;
  apiKey?: string;
+  maxRetries?: number;
 };

 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -57,7 +59,16 @@ export async function fetchLLMCompletion(
  params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
  // the apiKey must never be printed to the console
-  const { messages, modelParams, streaming, callbacks, apiKey } = params;
+  const {
+    messages,
+    modelParams,
+    streaming,
+    callbacks,
+    apiKey,
+    baseURL,
+    maxRetries,
+  } = params;
+
  const finalMessages = messages.map((message) => {
    if (message.role === ChatMessageRole.User)
      return new HumanMessage(message.content);
@@ -68,16 +79,18 @@ export async function fetchLLMCompletion(
  });

  let chatModel: ChatOpenAI | ChatAnthropic;
-  if (modelParams.provider === ModelProvider.Anthropic) {
+  if (modelParams.adapter === LLMAdapter.Anthropic) {
    chatModel = new ChatAnthropic({
      anthropicApiKey: apiKey,
+      anthropicApiUrl: baseURL,
      modelName: modelParams.model,
      temperature: modelParams.temperature,
      maxTokens: modelParams.max_tokens,
      topP: modelParams.top_p,
      callbacks,
+      clientOptions: { maxRetries },
    });
-  } else if (modelParams.provider === ModelProvider.OpenAI) {
+  } else if (modelParams.adapter === LLMAdapter.OpenAI) {
    chatModel = new ChatOpenAI({
      openAIApiKey: apiKey,
      modelName: modelParams.model,
@@ -85,15 +98,29 @@ export async function fetchLLMCompletion(
      maxTokens: modelParams.max_tokens,
      topP: modelParams.top_p,
      callbacks,
+      maxRetries,
+      configuration: {
+        baseURL,
+      },
+    });
+  } else if (modelParams.adapter === LLMAdapter.Azure) {
+    chatModel = new ChatOpenAI({
+      azureOpenAIApiKey: apiKey,
+      azureOpenAIBasePath: baseURL,
+      azureOpenAIApiDeploymentName: modelParams.model,
+      azureOpenAIApiVersion: "2024-02-01",
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
+      maxRetries,
    });
  } else {
    // eslint-disable-next-line no-unused-vars
-    const _exhaustiveCheck: never = modelParams;
+    const _exhaustiveCheck: never = modelParams.adapter;
    throw new Error("This model provider is not supported.");
  }

-  console.log("Making LLM call with params: ", modelParams);
-
  if (params.functionCall) {
    const functionCallingModel = chatModel.bind({
      functions: [
@@ -1,3 +1,4 @@
+import { LlmApiKeys } from "@prisma/client";
 import z from "zod";

 export type PromptVariable = { name: string; value: string; isUsed: boolean };
@@ -9,9 +10,10 @@ export type ChatMessage = {

 export type ChatMessageWithId = ChatMessage & { id: string };

-export enum ModelProvider {
+export enum LLMAdapter {
  Anthropic = "anthropic",
  OpenAI = "openai",
+  Azure = "azure",
 }

 export enum ChatMessageRole {
@@ -19,32 +21,30 @@ export enum ChatMessageRole {
  User = "user",
  Assistant = "assistant",
 }
-export type ModelParams = AnthropicModelParams | OpenAIModelParams;
+
+export type ModelParams = {
+  provider: string;
+  adapter: LLMAdapter;
+  model: string;
+} & ModelConfig;

 type RecordWithEnabledFlag<T> = {
  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
 export type UIModelParams = RecordWithEnabledFlag<
-  Required<AnthropicModelParams | OpenAIModelParams> & {
+  Required<ModelParams> & {
    maxTemperature: number;
  }
 >;

 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
-
 export const ZodModelConfig = z.object({
  max_tokens: z.coerce.number().optional(),
  temperature: z.coerce.number().optional(),
  top_p: z.coerce.number().optional(),
 });

-// OpenAI
-export type OpenAIModelParams = {
-  provider: ModelProvider.OpenAI;
-  model: OpenAIModel;
-} & ModelConfig;
-
 export const openAIModels = [
  "gpt-4o",
  "gpt-4o-2024-05-13",
@@ -64,12 +64,6 @@ export const openAIModels = [

 export type OpenAIModel = (typeof openAIModels)[number];

-// Anthropic
-export type AnthropicModelParams = {
-  provider: ModelProvider.Anthropic;
-  model: AnthropicModel;
-} & ModelConfig;
-
 export const anthropicModels = [
  "claude-3-opus-20240229",
  "claude-3-sonnet-20240229",
@@ -81,8 +75,9 @@ export const anthropicModels = [

 export type AnthropicModel = (typeof anthropicModels)[number];
 export const supportedModels = {
-  [ModelProvider.Anthropic]: anthropicModels,
-  [ModelProvider.OpenAI]: openAIModels,
+  [LLMAdapter.Anthropic]: anthropicModels,
+  [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.Azure]: [],
 } as const;

 export type LLMFunctionCall = {
@@ -90,3 +85,26 @@ export type LLMFunctionCall = {
  description: string;
  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
 };
+
+export const LLMApiKeySchema = z
+  .object({
+    id: z.string(),
+    projectId: z.string(),
+    createdAt: z.date(),
+    updatedAt: z.date(),
+    adapter: z.nativeEnum(LLMAdapter),
+    provider: z.string(),
+    displaySecretKey: z.string(),
+    secretKey: z.string(),
+    baseURL: z.string().nullable(),
+    customModels: z.array(z.string()),
+    withDefaultModels: z.boolean(),
+  })
+  // strict mode to prevent extra keys. Thorws error otherwise
+  // https://github.com/colinhacks/zod?tab=readme-ov-file#strict
+  .strict();
+
+export type LLMApiKey =
+  z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
+    ? z.infer<typeof LLMApiKeySchema>
+    : never;
@@ -30,4 +30,5 @@ export type TableName =
  | "sessions"
  | "scores"
  | "prompts"
-  | "dashboard";
+  | "dashboard"
+  | "users";
@@ -1,6 +1,6 @@
-import { type jsonSchema } from "@/src/utils/zod";
 import { type z } from "zod";
 import lodash from "lodash";
+import { jsonSchema } from "./zod";

 export const parseJson = (input: string) => {
  try {
@@ -37,7 +37,7 @@ export function deepParseJson(json: unknown): unknown {
        // Ensure we only iterate over the object's own properties
        if (Object.prototype.hasOwnProperty.call(json, key)) {
          (json as Record<string, unknown>)[key] = deepParseJson(
-            (json as Record<string, unknown>)[key],
+            (json as Record<string, unknown>)[key]
          );
        }
      }
@@ -50,7 +50,7 @@ export function deepParseJson(json: unknown): unknown {

 export const mergeJson = (
  json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
+  json2?: z.infer<typeof jsonSchema>
 ) => {
  if (json1 === undefined) {
    return json2;
@@ -26,7 +26,7 @@ const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
    nestedLiteralSchema,
    z.array(jsonSchemaNullable),
    z.record(jsonSchemaNullable),
-  ]),
+  ])
 );

 // Root schema that does not allow nulls at the root level
@@ -35,17 +35,17 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
    rootLiteralSchema,
    z.array(jsonSchemaNullable),
    z.record(jsonSchemaNullable),
-  ]),
+  ])
 );

 export const paginationZod = {
  page: z.preprocess(
    (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().default(1)
  ),
  limit: z.preprocess(
    (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().lte(100).default(50)
  ),
 };

@@ -9,7 +9,9 @@
    "start": {
      "dependsOn": ["^start"]
    },
-    "db:migrate": {},
+    "db:migrate": {
+      "cache": false
+    },
    "db:deploy": {},
    "db:reset": {},
    "db:push": {},
@@ -1,11 +1,11 @@
-FROM node:20-alpine AS alpine
+FROM node:20-alpine3.20 AS alpine

 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
+RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client

 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
@@ -78,6 +78,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/web/public ./web/public
 COPY --from=builder --chown=nextjs:nodejs /app/packages/shared/prisma ./packages/shared/prisma

 COPY --chown=nextjs:nodejs ./web/entrypoint.sh ./web/entrypoint.sh
+COPY --chown=nextjs:nodejs ./packages/shared/scripts/cleanup.sql ./packages/shared/scripts/cleanup.sql
 RUN chmod +x ./web/entrypoint.sh

 USER nextjs
@@ -1,5 +1,6 @@
 #!/bin/sh

+# Run cleanup script before running migrations
 # Check if DATABASE_URL is not set
 if [ -z "$DATABASE_URL" ]; then
    # Check if all required variables are provided
@@ -18,6 +19,8 @@ if [ -z "$DIRECT_URL" ]; then
    export DIRECT_URL=$DATABASE_URL
 fi

+prisma db execute --url "$DIRECT_URL" --file "./packages/shared/scripts/cleanup.sql"
+
 # Apply migrations
 prisma migrate deploy --schema=./packages/shared/prisma/schema.prisma
 status=$?
@@ -11,8 +11,8 @@ import { env } from "./src/env.mjs";
 * img-src https to allow loading images from SSO providers
 */
 const cspHeader = `
-  default-src 'self' https://ph.langfuse.com https://*.posthog.com wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  default-src 'self' https://ph.langfuse.com https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
+  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://*.sentry.io https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
  style-src 'self' 'unsafe-inline' https://*.crisp.chat;
  img-src 'self' https: blob: data:;
  font-src 'self' https://*.crisp.chat;
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "2.45.0",
+  "version": "2.55.0",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -25,8 +25,9 @@
    "@aws-sdk/lib-storage": "^3.568.0",
    "@aws-sdk/s3-request-presigner": "^3.554.0",
    "@codemirror/lang-json": "^6.0.1",
+    "@codemirror/lint": "^6.8.0",
    "@headlessui/react": "1.7.18",
-    "@headlessui/tailwindcss": "0.2.0",
+    "@headlessui/tailwindcss": "0.2.1",
    "@heroicons/react": "^2.1.3",
    "@hookform/resolvers": "^3.3.4",
    "@langchain/anthropic": "^0.1.21",
@@ -35,6 +36,7 @@
    "@langfuse/ee": "workspace:*",
    "@langfuse/shared": "workspace:*",
    "@marsidev/react-turnstile": "^0.5.4",
+    "@mui/x-tree-view": "^7.6.2",
    "@next-auth/prisma-adapter": "^1.0.7",
    "@radix-ui/react-accordion": "^1.1.2",
    "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -55,28 +57,28 @@
    "@radix-ui/react-switch": "^1.0.3",
    "@radix-ui/react-tabs": "^1.0.4",
    "@radix-ui/react-toggle": "^1.0.3",
+    "@radix-ui/react-toggle-group": "^1.0.4",
    "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.18",
-    "@react-email/render": "^0.0.14",
+    "@react-email/components": "^0.0.19",
+    "@react-email/render": "^0.0.15",
    "@remixicon/react": "^4.2.0",
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.113.0",
-    "@sentry/node": "^7.113.0",
-    "@sentry/profiling-node": "^7.113.0",
-    "@sentry/types": "^7.113.0",
+    "@sentry/nextjs": "^8.8.0",
+    "@sentry/node": "^8.8.0",
+    "@sentry/profiling-node": "^8.8.0",
+    "@sentry/types": "^8.8.0",
    "@t3-oss/env-nextjs": "^0.10.1",
    "@tailwindcss/container-queries": "^0.1.1",
    "@tanstack/react-query": "^4.36.1",
    "@tanstack/react-table": "^8.11.8",
-    "@tanstack/react-virtual": "^3.5.0",
    "@tremor/react": "3.16.2",
    "@trpc/client": "^10.45.0",
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
    "@uiw/codemirror-theme-github": "^4.21.25",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
    "@uiw/react-codemirror": "^4.21.25",
    "ai": "^3.0.23",
    "bcryptjs": "^2.4.3",
@@ -87,7 +89,7 @@
    "cors": "^2.8.5",
    "date-fns": "^3.3.1",
    "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.4",
+    "dompurify": "^3.1.5",
    "exponential-backoff": "^3.1.1",
    "js-tiktoken": "^1.0.12",
    "kysely": "^0.27.3",
@@ -106,7 +108,7 @@
    "react-day-picker": "^8.10.1",
    "react-dom": "18.2.0",
    "react-hook-form": "^7.51.5",
-    "react-icons": "^5.0.1",
+    "react-icons": "^5.2.1",
    "react-responsive": "^10.0.0",
    "react18-json-view": "^0.2.8-canary.6",
    "sonner": "^1.4.41",
@@ -115,6 +117,7 @@
    "tailwindcss-animate": "^1.0.7",
    "use-query-params": "^2.2.1",
    "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
    "zod": "^3.22.4"
  },
  "devDependencies": {
@@ -122,21 +125,21 @@
    "@mermaid-js/mermaid-cli": "^10.7.0",
    "@playwright/test": "^1.43.1",
    "@tailwindcss/forms": "^0.5.7",
-    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/jest-dom": "^6.4.6",
    "@testing-library/react": "^15.0.7",
    "@types/bcryptjs": "^2.4.6",
    "@types/cors": "^2.8.17",
    "@types/dompurify": "^3.0.5",
    "@types/eslint": "^8.56.7",
    "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.5",
    "@types/node": "20.10.5",
    "@types/nodemailer": "^6.4.15",
    "@types/react": "^18.2.79",
    "@types/react-dom": "^18.2.25",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
    "autoprefixer": "^10.4.19",
    "dotenv-cli": "^7.4.2",
    "eslint": "^8.56.0",
@@ -147,7 +150,7 @@
    "postcss": "^8.4.38",
    "prettier": "^3.2.5",
    "prettier-plugin-tailwindcss": "^0.5.14",
-    "tailwindcss": "^3.4.3",
+    "tailwindcss": "^3.4.4",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
    "tsx": "^4.7.2",
@@ -397,7 +397,10 @@ paths:
          description: ''
  /api/public/ingestion:
    post:
-      description: Batched ingestion for Langfuse Tracing
+      description: >-
+        Batched ingestion for Langfuse Tracing. If you want to use tracing via
+        the API, such as to build your own Langfuse client implementation, this
+        is the only API route you need to implement.
      operationId: ingestion_batch
      tags:
        - Ingestion
@@ -446,6 +449,14 @@ paths:
                  type: array
                  items:
                    $ref: '#/components/schemas/IngestionEvent'
+                  description: >-
+                    Batch of tracing events to be ingested. Discriminated by
+                    attribute `type`.
+                metadata:
+                  nullable: true
+                  description: >-
+                    Optional. Metadata field used by the Langfuse SDKs for
+                    debugging.
              required:
                - batch
  /api/public/metrics/daily:
@@ -997,9 +1008,10 @@ paths:
        - name: source
          in: query
          description: Retrieve only scores from a specific source.
-          required: true
+          required: false
          schema:
            $ref: '#/components/schemas/ScoreSource'
+            nullable: true
        - name: operator
          in: query
          description: Retrieve only scores with <operator> value.
@@ -1015,6 +1027,13 @@ paths:
            type: number
            format: double
            nullable: true
+        - name: scoreIds
+          in: query
+          description: Comma-separated list of score IDs to limit the results to.
+          required: false
+          schema:
+            type: string
+            nullable: true
      responses:
        '200':
          description: ''
@@ -1137,7 +1156,10 @@ paths:
      security: *ref_0
  /api/public/sessions/{sessionId}:
    get:
-      description: Get a session
+      description: >-
+        Get a session. Please note that `traces` on this endpoint are not
+        paginated, if you plan to fetch large sessions, consider `GET
+        /api/public/traces?sessionId=<sessionId>`
      operationId: sessions_get
      tags:
        - Sessions
@@ -1262,6 +1284,12 @@ paths:
          schema:
            type: string
            nullable: true
+        - name: sessionId
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
        - name: fromTimestamp
          in: query
          description: Retrieve only traces newer than this datetime (ISO 8601).
@@ -1825,8 +1853,8 @@ components:
      title: ScoreSource
      type: string
      enum:
+        - ANNOTATION
        - API
-        - REVIEW
        - EVAL
    CreateDatasetItemRequest:
      title: CreateDatasetItemRequest
@@ -1955,8 +1983,19 @@ components:
                type:
                  type: string
                  enum:
-                    - event-create
-            - $ref: '#/components/schemas/CreateEventEvent'
+                    - span-create
+            - $ref: '#/components/schemas/CreateSpanEvent'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - span-update
+            - $ref: '#/components/schemas/UpdateSpanEvent'
          required:
            - type
        - type: object
@@ -1988,19 +2027,8 @@ components:
                type:
                  type: string
                  enum:
-                    - span-create
-            - $ref: '#/components/schemas/CreateSpanEvent'
-          required:
-            - type
-        - type: object
-          allOf:
-            - type: object
-              properties:
-                type:
-                  type: string
-                  enum:
-                    - span-update
-            - $ref: '#/components/schemas/UpdateSpanEvent'
+                    - event-create
+            - $ref: '#/components/schemas/CreateEventEvent'
          required:
            - type
        - type: object
@@ -2323,13 +2351,20 @@ components:
      properties:
        id:
          type: string
+          description: UUID v4 that identifies the event
        timestamp:
          type: string
-        metadata: {}
+          description: >-
+            Datetime (ISO 8601) of event creation in client. Should be as close
+            to actual event creation in client as possible, this timestamp will
+            be used for ordering of events in future release. Resolution:
+            milliseconds (required), microseconds (optimal).
+        metadata:
+          nullable: true
+          description: Optional. Metadata field used by the Langfuse SDKs for debugging.
      required:
        - id
        - timestamp
-        - metadata
    TraceEvent:
      title: TraceEvent
      type: object
@@ -2499,6 +2534,7 @@ components:
        totalCost:
          type: number
          format: double
+          description: Total model cost in USD
        usage:
          type: array
          items:
@@ -2521,14 +2557,28 @@ components:
          nullable: true
        inputUsage:
          type: integer
+          description: Total number of generation input units (e.g. tokens)
        outputUsage:
          type: integer
+          description: Total number of generation output units (e.g. tokens)
        totalUsage:
          type: integer
+          description: Total number of generation total units (e.g. tokens)
+        countTraces:
+          type: integer
+        countObservations:
+          type: integer
+        totalCost:
+          type: number
+          format: double
+          description: Total model cost in USD
      required:
        - inputUsage
        - outputUsage
        - totalUsage
+        - countTraces
+        - countObservations
+        - totalCost
    Observations:
      title: Observations
      type: object
@@ -2654,6 +2704,13 @@ components:
          items:
            type: string
          nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
      required:
        - name
        - prompt
@@ -2672,6 +2729,13 @@ components:
          items:
            type: string
          nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
      required:
        - name
        - prompt
@@ -2713,11 +2777,20 @@ components:
          type: array
          items:
            type: string
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          description: >-
+            List of tags. Used to filter via UI and API. The same across
+            versions of a prompt.
      required:
        - name
        - version
        - config
        - labels
+        - tags
    ChatMessage:
      title: ChatMessage
      type: object
@@ -337,7 +337,7 @@
          "_type": "endpoint",
          "name": "Batch",
          "request": {
-            "description": "Batched ingestion for Langfuse Tracing",
+            "description": "Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.",
            "url": {
              "raw": "{{baseUrl}}/api/public/ingestion",
              "host": [
@@ -356,7 +356,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
+              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ],\n    \"metadata\": \"UNKNOWN\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -699,7 +699,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -755,7 +755,7 @@
          "request": {
            "description": "Get a list of scores",
            "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=&scoreIds=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -804,6 +804,11 @@
                  "key": "value",
                  "value": "",
                  "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "scoreIds",
+                  "value": "",
+                  "description": "Comma-separated list of score IDs to limit the results to."
                }
              ],
              "variable": []
@@ -890,7 +895,7 @@
          "_type": "endpoint",
          "name": "Get",
          "request": {
-            "description": "Get a session",
+            "description": "Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`",
            "url": {
              "raw": "{{baseUrl}}/api/public/sessions/:sessionId",
              "host": [
@@ -963,7 +968,7 @@
          "request": {
            "description": "Get list of traces.",
            "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&sessionId=&fromTimestamp=&orderBy=&tags=",
              "host": [
                "{{baseUrl}}"
              ],
@@ -993,6 +998,11 @@
                  "value": "",
                  "description": null
                },
+                {
+                  "key": "sessionId",
+                  "value": "",
+                  "description": null
+                },
                {
                  "key": "fromTimestamp",
                  "value": "",
@@ -4,7 +4,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
  Sentry.init({
    dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
    // Replay may only be enabled for the client-side
-    integrations: [new Sentry.Replay()],
+    integrations: [Sentry.replayIntegration()],

    // Set tracesSampleRate to 1.0 to capture 100%
    // of transactions for performance monitoring.
@@ -1,8 +1,6 @@
-import { prisma } from "@langfuse/shared/src/db";
 import * as Sentry from "@sentry/nextjs";
-import { ProfilingIntegration } from "@sentry/profiling-node";
-import type { SamplingContext, TransactionEvent } from "@sentry/types";
-import { jsonSchema } from "@/src/utils/zod";
+import { jsonSchema } from "@langfuse/shared";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";

 if (process.env.NEXT_PUBLIC_SENTRY_DSN)
  Sentry.init({
@@ -12,37 +10,34 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
    // Set tracesSampleRate to 1.0 to capture 100%
    // of transactions for performance monitoring.
    // We recommend adjusting this value in production
-    tracesSampler: (samplingContext: SamplingContext) => {
+    tracesSampler: (samplingContext) => {
      if (
        samplingContext.request &&
        samplingContext.request.url &&
        samplingContext.request.url.includes("api/trpc")
      ) {
-        return 0.3;
+        return 0.1;
      }
      if (
        samplingContext.request &&
        samplingContext.request.url &&
-        samplingContext.request.url.includes("api/auth") &&
-        samplingContext.transactionContext.status !== "ok" &&
-        samplingContext.transactionContext.status !== "unauthenticated"
+        samplingContext.request.url.includes("api/auth")
      ) {
-        return 1;
+        return 0.1;
      }
-      return 0.1;
+      return 0.01;
    },

-    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.1,
    integrations: [
      // Add profiling integration to list of integrations
-      new ProfilingIntegration(),
-      new Sentry.Integrations.Prisma({ client: prisma }),
-      Sentry.metrics.metricsAggregatorIntegration(),
+      Sentry.prismaIntegration(),
+      nodeProfilingIntegration(),
    ],

    // filter out passwords from the signup request body
    // transaction events are sentry transactions which include logs and spans.
-    beforeSendTransaction(transaction: TransactionEvent) {
+    beforeSendTransaction(transaction) {
      if (
        transaction.request &&
        typeof transaction.request.data === "string" &&
@@ -3,7 +3,7 @@

 import type { Session } from "next-auth";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { ModelProvider } from "@langfuse/shared";
+import { LLMAdapter } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
@@ -38,11 +38,20 @@ describe("llmApiKey.all RPC", () => {

  it("should create an llm api key", async () => {
    const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;

    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
    });

    const llmApiKeys = await prisma.llmApiKeys.findMany({
@@ -55,18 +64,31 @@ describe("llmApiKey.all RPC", () => {
    expect(llmApiKeys[0].projectId).toBe(projectId);
    expect(llmApiKeys[0].secretKey).not.toBeNull();
    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
-    expect(llmApiKeys[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
    // this has to be 3 dots and the last 4 characters of the secret
    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
  });

  it("should create and get an llm api key", async () => {
    const secret = "test-secret";
+    const provider = "openai";
+    const adapter = LLMAdapter.OpenAI;
+    const customModels = ["fancy-gpt-3.5-turbo"];
+    const baseURL = "https://custom.openai.com/v1";
+    const withDefaultModels = false;

    await caller.llmApiKey.create({
      projectId,
      secretKey: secret,
-      provider: ModelProvider.OpenAI,
+      provider,
+      adapter,
+      baseURL,
+      customModels,
+      withDefaultModels,
    });

    const dbLlmApiKeys = await prisma.llmApiKeys.findMany({
@@ -77,16 +99,23 @@ describe("llmApiKey.all RPC", () => {

    expect(dbLlmApiKeys.length).toBe(1);

-    const llmApiKeys = await caller.llmApiKey.all({
+    const { data: llmApiKeys } = await caller.llmApiKey.all({
      projectId,
    });

-    expect(llmApiKeys.data.length).toBe(1);
-    expect(llmApiKeys.data[0].provider).toBe(ModelProvider.OpenAI);
+    expect(llmApiKeys.length).toBe(1);
+    expect(llmApiKeys[0].projectId).toBe(projectId);
+    expect(llmApiKeys[0].secretKey).not.toBeNull();
+    expect(llmApiKeys[0].secretKey).not.toEqual(secret);
+    expect(llmApiKeys[0].provider).toBe(provider);
+    expect(llmApiKeys[0].adapter).toBe(adapter);
+    expect(llmApiKeys[0].baseURL).toBe(baseURL);
+    expect(llmApiKeys[0].customModels).toEqual(customModels);
+    expect(llmApiKeys[0].withDefaultModels).toBe(withDefaultModels);
    // this has to be 3 dots and the last 4 characters of the secret
-    expect(llmApiKeys.data[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
+    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);

    // response must not contain the secret key itself
-    expect(llmApiKeys.data[0]).not.toHaveProperty("secretKey");
+    expect(llmApiKeys[0]).not.toHaveProperty("secretKey");
  });
 });
@@ -68,12 +68,18 @@ describe("/api/public/metrics/daily API Endpoint", () => {
        inputUsage: 333,
        outputUsage: 0,
        totalUsage: 333,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
      },
      {
        model: "modelC",
        inputUsage: 666,
        outputUsage: 777,
        totalUsage: 1443,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 1024.22,
      },
    ]);

@@ -88,6 +94,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
        inputUsage: 100,
        outputUsage: 200,
        totalUsage: 300,
+        countObservations: 1,
+        countTraces: 1,
+        totalCost: 0,
      },
    ]);
  });
@@ -680,6 +680,71 @@ describe("/api/public/v2/prompts API Endpoint", () => {
      expect(fetchedPrompt.body.createdBy).toBe("API");
      expect(fetchedPrompt.body.config).toEqual({});
    });
+
+    it("should update tags across versions", async () => {
+      const promptName = "prompt-name" + nanoid();
+
+      const createPromptVersion = async (tags?: string[]) => {
+        await makeAPICall("POST", baseURI, {
+          name: promptName,
+          prompt: "This is a test prompt",
+          type: PromptType.Text,
+          ...(tags !== undefined && { tags: tags }),
+        });
+      };
+
+      const fetchPromptVersion = async (version: number) => {
+        const fetchedPrompt = await makeAPICall(
+          "GET",
+          `${baseURI}/${promptName}?version=${version}`,
+          undefined,
+        );
+        expect(fetchedPrompt.status).toBe(200);
+        if (!isPrompt(fetchedPrompt.body)) {
+          throw new Error("Expected body to be a prompt");
+        }
+        return fetchedPrompt.body;
+      };
+
+      // Create version 1 with ["tag"]
+      await createPromptVersion(["tag"]);
+      let fetchedPrompt1 = await fetchPromptVersion(1);
+      expect(fetchedPrompt1.tags).toEqual(["tag"]);
+      expect(fetchedPrompt1.version).toBe(1);
+
+      // Create version 2 with no tags provided (should use tags from version 1)
+      await createPromptVersion();
+      let fetchedPrompt2 = await fetchPromptVersion(2);
+      expect(fetchedPrompt2.tags).toEqual(["tag"]);
+      expect(fetchedPrompt2.version).toBe(2);
+
+      // Create version 3 with ["tag1", "tag2", "tag3"] (should update tags across versions)
+      await createPromptVersion(["tag1", "tag2", "tag3"]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      let fetchedPrompt3 = await fetchPromptVersion(3);
+      expect(fetchedPrompt1.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt3.version).toBe(3);
+
+      // remove tags
+      await createPromptVersion([]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      fetchedPrompt3 = await fetchPromptVersion(3);
+      let fetchedPrompt4 = await fetchPromptVersion(4);
+      expect(fetchedPrompt1.tags).toEqual([]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual([]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual([]);
+      expect(fetchedPrompt3.version).toBe(3);
+      expect(fetchedPrompt4.tags).toEqual([]);
+      expect(fetchedPrompt4.version).toBe(4);
+    });
  });

  describe("when fetching a prompt list", () => {
@@ -50,7 +50,7 @@ describe("Build valid SQL queries", () => {
        table: "traces_scores",
        values: ["project-id"],
        strings: [
-          ' FROM  traces t JOIN scores s ON t.id = s.trace_id AND t.project_id = s.project_id  WHERE  t."project_id" = ',
+          ` FROM  traces t JOIN scores s ON t.id = s.trace_id AND s.data_type != 'CATEGORICAL' AND t.project_id = s.project_id  WHERE  t."project_id" = `,
          ";",
        ],
      } as const,
@@ -489,16 +489,16 @@ describe("/api/public/scores API Endpoint", () => {
        totalPages: 1,
      });
      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
        {
          id: scoreId_2,
          name: scoreName,
          value: 50.5,
        },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
      ]);
    });
    it("test operator >=", async () => {
@@ -514,16 +514,16 @@ describe("/api/public/scores API Endpoint", () => {
        totalPages: 1,
      });
      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
        {
          id: scoreId_3,
          name: scoreName,
          value: 100.8,
        },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
      ]);
    });
    it("test operator !=", async () => {
@@ -539,16 +539,16 @@ describe("/api/public/scores API Endpoint", () => {
        totalPages: 1,
      });
      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
        {
          id: scoreId_3,
          name: scoreName,
          value: 100.8,
        },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
      ]);
    });
    it("test operator =", async () => {
@@ -591,5 +591,37 @@ describe("/api/public/scores API Endpoint", () => {
        message: "Invalid request data",
      });
    });
+
+    it("should filter scores by score IDs", async () => {
+      const getScore = await makeAPICall<{
+        data: [
+          {
+            id: string;
+            name: string;
+            value: number;
+          },
+        ];
+        meta: object;
+      }>("GET", `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`);
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
  });
 });
@@ -238,4 +238,62 @@ describe("/api/public/traces API Endpoint", () => {
    expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
    expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
  });
+
+  it("should filter traces by session ID", async () => {
+    const sessionId = "test-session-id";
+    const anotherSessionId = "another-session-id";
+
+    // Create traces with different session IDs
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-1",
+      name: "test-trace-1",
+      sessionId,
+      userId: "user-1",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-2",
+      name: "test-trace-2",
+      sessionId: anotherSessionId,
+      userId: "user-2",
+      projectId: "project-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "1.0.0",
+    });
+
+    // Filter by session ID
+    const tracesBySessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${sessionId}`,
+    );
+
+    expect(tracesBySessionId.status).toBe(200);
+    expect(tracesBySessionId.body.data).toHaveLength(1);
+    expect(tracesBySessionId.body.data[0].id).toBe("trace-1");
+
+    // Filter by another session ID
+    const tracesByAnotherSessionId = await makeAPICall<GetTracesAPIResponse>(
+      "GET",
+      `/api/public/traces?sessionId=${anotherSessionId}`,
+    );
+
+    expect(tracesByAnotherSessionId.status).toBe(200);
+    expect(tracesByAnotherSessionId.body.data).toHaveLength(1);
+    expect(tracesByAnotherSessionId.body.data[0].id).toBe("trace-2");
+
+    // Filter by non-existent session ID
+    const tracesByNonExistentSessionId =
+      await makeAPICall<GetTracesAPIResponse>(
+        "GET",
+        `/api/public/traces?sessionId=non-existent-session-id`,
+      );
+
+    expect(tracesByNonExistentSessionId.status).toBe(200);
+    expect(tracesByNonExistentSessionId.body.data).toHaveLength(0);
+  });
 });
--- a/Show More
+++ b/Show More