chore: release v2.85.0

fix

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx
+ignore-words-list = afterall,vertx,notIn
 

.devcontainer/devcontainer.json

@@ -1,14 +0,0 @@
-{
-  "name": "langfuse dev",
-  "image": "mcr.microsoft.com/devcontainers/javascript-node:20-bookworm",
-  "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {
-      "version": "latest",
-      "moby": true
-    }
-  },
-  "forwardPorts": [3000],
-
-  "onCreateCommand": "npm install -g pnpm@9.5.0",
-  "postCreateCommand": "cp .env.dev.example .env && pnpm run dx-f"
-}

.env.dev.example

@@ -6,6 +6,12 @@
 DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+
 # Next Auth
 # You can generate a new secret on the command line with:
 # openssl rand -base64 32
@@ -28,11 +34,25 @@ EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
 # S3 storage
-S3_ENDPOINT=
-S3_ACCESS_KEY_ID=
-S3_SECRET_ACCESS_KEY=
-S3_BUCKET_NAME=
-S3_REGION=
+S3_ENDPOINT=http://localhost:9090
+S3_ACCESS_KEY_ID=minio
+S3_SECRET_ACCESS_KEY=miniosecret
+S3_BUCKET_NAME=mybucket
+S3_REGION=us-east-1
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=false
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=mybucket
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 # Set during docker build of application
 # Used to disable environment verification at build time
@@ -44,4 +64,7 @@ REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 # openssl rand -hex 32 used only here
-ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.local.example

@@ -18,4 +18,7 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=myworkerpassword
+LANGFUSE_WORKER_PASSWORD=myworkerpassword
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -10,40 +10,54 @@ DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
 # optional, set to true to disable automated database migrations on Docker start
-# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED= 
+# LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED=
 
 # Next Auth
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
 NEXTAUTH_URL="http://localhost:3000"
 
-# You can generate a new secret on the command line with:
+# For each of these, you can generate a new secret on the command line with:
 # openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-NEXTAUTH_SECRET="secret"
-SALT="salt"
+NEXTAUTH_SECRET="secret" # https://next-auth.js.org/configuration/options#secret
+SALT="salt" # salt used to hash api keys
+
+# API level encryption for sensitive data
+# Must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32
+ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000"
 
 # Use CSP headers to enforce HTTPS, optional
-LANGFUSE_CSP_ENFORCE_HTTPS="true"
+# LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
+# Configure base path for self-hosting, optional
+# Note: You need to build the docker image with the base path set and cannot use the pre-built docker image if you set this.
+# NEXT_PUBLIC_BASE_PATH="/app"
 
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
 
-# Default project, optional
+# Opentelemetry, optional
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+OTEL_SERVICE_NAME="langfuse"
+
+# Default role for users who sign up, optional, can be org or org+project
+# LANGFUSE_DEFAULT_ORG_ID=
+# LANGFUSE_DEFAULT_ORG_ROLE=
 # LANGFUSE_DEFAULT_PROJECT_ID=
 # LANGFUSE_DEFAULT_PROJECT_ROLE=
 
+# Logging, optional
+# LANGFUSE_LOG_LEVEL=info
+# LANGFUSE_LOG_FORMAT=text
+
 # Enable experimental features, optional
 # LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
 
-# Disable automated database migrations on Docker start, optional
-# LANFUSE_WEB_MIGRATION_DISABLED=true
-
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
+# AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
 
 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -53,6 +67,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_CLIENT_ID=
+# AUTH_GITLAB_CLIENT_SECRET=
+# AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_ISSUER=
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
@@ -90,12 +108,53 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # S3_REGION=
 # BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
 
+# S3 storage for events, optional, used to persist all incoming events
+# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
+# LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
+# Optional prefix to be used within the bucket. Must end with `/` if set
+# LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+# The following four options are optional and fallback to the normal SDK credential provider chain if omitted
+# See https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html
+# LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=
+# LANGFUSE_S3_EVENT_UPLOAD_REGION=
+# LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=
+# LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=
+
 # Exports are streamed to S3 in pages to avoid memory issues
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
-# Disable logging to `events` table in db
-# ENABLE_EVENT_LOG=false
+# Automated provisioning of default resources
+# LANGFUSE_INIT_ORG_ID=org-id
+# LANGFUSE_INIT_ORG_NAME=org-name
+# LANGFUSE_INIT_PROJECT_ID=project-id
+# LANGFUSE_INIT_PROJECT_NAME=project-name
+# LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-1234567890
+# LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-1234567890
+# LANGFUSE_INIT_USER_EMAIL=user@example.com
+# LANGFUSE_INIT_USER_NAME=User Name
+# LANGFUSE_INIT_USER_PASSWORD=password
+
+
+
+### START Enterprise Edition Configuration
+
+# Allowlisted users that can create new organizations, by default all users can create organizations
+# LANGFUSE_ALLOWED_ORGANIZATION_CREATORS=user1@langfuse.com,user2@langfuse.com
+
+# UI Customization Options
+# LANGFUSE_UI_API_HOST=https://api.example.com
+# LANGFUSE_UI_DOCUMENTATION_HREF=https://docs.example.com
+# LANGFUSE_UI_SUPPORT_HREF=https://support.example.com
+# LANGFUSE_UI_FEEDBACK_HREF=https://feedback.example.com
+# LANGFUSE_UI_LOGO_LIGHT_MODE_HREF=https://static.langfuse.com/langfuse-dev/example-logo-light-mode.png
+# LANGFUSE_UI_LOGO_DARK_MODE_HREF=https://static.langfuse.com/langfuse-dev/example-logo-dark-mode.png
+# LANGFUSE_UI_DEFAULT_MODEL_ADAPTER=Anthropic # OpenAI, Anthropic, Azure
+# LANGFUSE_UI_DEFAULT_BASE_URL_OPENAI=https://api.openai.com/v1
+# LANGFUSE_UI_DEFAULT_BASE_URL_ANTHROPIC=https://api.anthropic.com
+# LANGFUSE_UI_DEFAULT_BASE_URL_AZURE_OPENAI=https://{instanceName}.openai.azure.com/openai/deployments
+
+### END Enterprise Edition Configuration
 
 
 
@@ -104,9 +163,6 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Disable the expensive analytics queries and related features
 # LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
 
-# Customize the default datetime offset filter used in for the UI tables
-# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
-
 ### END Envs to be deprecated in Langfuse v3.0
 
 
@@ -127,12 +183,12 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # NEXT_PUBLIC_POSTHOG_KEY=
 
 # Sentry
+# NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE
 # NEXT_PUBLIC_SENTRY_DSN=
 # NEXT_SENTRY_ORG=
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
-# Used to determine the Sentry sample rate
-# LANGFUSE_TRACING_SAMPLE_RATE=
+# SENTRY_CSP_REPORT_URI=
 
 
 # Cloudflare Turnstile
@@ -140,16 +196,56 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # TURNSTILE_SECRET_KEY=
 
 # Demo project that users can use to try the platform
+# NEXT_PUBLIC_DEMO_ORG_ID=
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
-# Encryption
-# ENCRYPTION_KEY=
-
 # Admin API
 # ADMIN_API_KEY=
 
+# Redis
+# REDIS_HOST=
+# REDIS_PORT=
+# REDIS_AUTH=
+# REDIS_CONNECTION_STRING=
+# REDIS_ENABLE_AUTO_PIPELINING=
+
+# Cache configuration
+# LANGFUSE_CACHE_API_KEY_ENABLED=
+# LANGFUSE_CACHE_API_KEY_TTL_SECONDS=
+# LANGFUSE_CACHE_PROMPT_ENABLED=
+# LANGFUSE_CACHE_PROMPT_TTL_SECONDS=
+
+# Rate limiting
+# LANGFUSE_RATE_LIMITS_ENABLED=
+
+# Stripe
+# STRIPE_SECRET_KEY=
+# STRIPE_WEBHOOK_SIGNING_SECRET=
 
 ### END Langfuse Cloud Config
+
+
+## START Langfuse V3 Ingestion 
+
+# Clickhouse
+# CLICKHOUSE_MIGRATION_URL=
+# CLICKHOUSE_URL=
+# CLICKHOUSE_USER=
+# CLICKHOUSE_PASSWORD=
+
+# Ingestion
+# LANGFUSE_INGESTION_BUFFER_TTL_SECONDS=
+# LANGFUSE_INGESTION_FLUSH_DELAY_MS= 
+# LANGFUSE_INGESTION_FLUSH_ATTEMPTS= 
+# LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE= 
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
+# LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
+# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
+# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+# QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
+
+## END Langfuse V3 Ingestion

.github/dependabot.yml

@@ -8,6 +8,7 @@ updates:
   - package-ecosystem: npm
     directory: "/" # Location of package manifests
     rebase-strategy: "disabled" # use dependabot-rebase-stale
+    open-pull-requests-limit: 10
     schedule:
       interval: "daily"
     versioning-strategy: "increase"
@@ -19,17 +20,29 @@ updates:
       - dependency-name: "@types/node"
       - dependency-name: "@trpc/*"
     groups:
-      sentry:
-        patterns:
-          - "@sentry/*"
       prisma:
         patterns:
           - "prisma"
-          - "@prisma/*"
+          - "@prisma/client"
       next:
         patterns:
           - "eslint-config-next"
           - "next"
-      patches:
-        update-types:
-          - "patch"
+      lodash:
+        patterns:
+          - "lodash"
+          - "@types/lodash"
+      express:
+        patterns:
+          - "express"
+          - "@types/express"
+      observability:
+        patterns:
+          - "dd-trace"
+          - "@opentelemetry/*"
+          - "@appsignal/opentelemetry-instrumentation-bullmq"
+          - "@prisma/instrumentation"
+          - "@sentry/*"
+      radix-ui:
+        patterns:
+          - "@radix-ui/*"

.github/workflows/_deploy_ecs_service.yml

@@ -0,0 +1,75 @@
+name: WorkflowCall - Deploy to ECS Service
+on:
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+        description: Deployment environment
+        required: true
+      service:
+        type: string
+        description: Name of the service to be deployed, e.g. web-ingestion, web, or worker.
+        required: true
+jobs:
+  ecs-deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - name: Get app name
+        uses: winterjung/split@v2
+        id: split
+        with:
+          msg: ${{ inputs.service }}
+          separator: "-"
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Authenticate with AWS
+        # GitHub/AWS recommend to use OIDC here: https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc
+        # Probably more painful to configure, but would remove all long-lived credentials.
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_REGION }}
+      - name: Login to AWS ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Build, tag, and push Docker image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ steps.split.outputs._0 }}
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build \
+            -t $REGISTRY/$REPOSITORY:$IMAGE_TAG \
+            -f ./${{ steps.split.outputs._0 }}/Dockerfile \
+            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }} \
+            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }} \
+            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }} \
+            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }} \
+            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }} \
+            --build-arg NEXT_PUBLIC_SENTRY_DSN=${{ vars.NEXT_PUBLIC_SENTRY_DSN }} \
+            --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
+            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
+            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
+            --build-arg NEXT_PUBLIC_CRISP_WEBSITE_ID=${{ vars.NEXT_PUBLIC_CRISP_WEBSITE_ID }} \
+            --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
+            --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
+            --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
+            .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+
+      - name: Render AWS ECS Task Definition
+        id: render-task-definition
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          container-name: ${{ inputs.service }}
+          image: ${{ steps.login-ecr.outputs.registry }}/${{ steps.split.outputs._0 }}:${{ github.sha }}
+          task-definition-family: ${{ inputs.environment }}-${{ inputs.service }}
+      - name: Update AWS ECS Service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ steps.render-task-definition.outputs.task-definition }}
+          service: ${{ inputs.environment }}-${{ inputs.service }}
+          cluster: ${{ inputs.environment }}-cluster
+          wait-for-service-stability: true

.github/workflows/ci.yml.template

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start containers
         run: docker compose -f "docker-compose.yml" up -d --build
@@ -31,7 +31,7 @@ jobs:
         uses: pnpm/action-setup@v2.2.4
 
       - name: Setup Node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main", "production", "v*"]
+  pull_request:
+    branches: ["main", "production", "v*"]
+  schedule:
+    - cron: "31 13 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: javascript-typescript
+            build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # If the analyze step fails for one of the languages you are analyzing with
+      # "We were unable to automatically build your code", modify the matrix above
+      # to set the build mode to "manual" for that language. Then modify this step
+      # to build your code.
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      - if: matrix.build-mode == 'manual'
+        shell: bash
+        run: |
+          echo 'If you are using a "manual" build mode for one or more of the' \
+            'languages you are analyzing, replace this with the commands to build' \
+            'your code, for example:'
+          echo '  make bootstrap'
+          echo '  make release'
+          exit 1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/deploy.yml

@@ -0,0 +1,105 @@
+on:
+  push:
+    branches:
+      - main
+      - production
+  workflow_dispatch:
+    inputs:
+      service:
+        description: 'Service to be deployed'
+        type: choice
+        options:
+          - all
+          - web
+          - web-ingestion
+          - worker
+        required: true
+      environment:
+        description: 'Environment to deploy to'
+        type: choice
+        options:
+          - staging
+          - prod-eu-temp
+          - prod-us
+        required: true
+
+concurrency:
+  # Support concurrent `push` and `workflow_dispatch`` actions
+  group: deploy-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+name: Deploy to ECS
+jobs:
+  affected-services:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.affected-services.outputs.result }}
+    steps:
+      - name: Get affected services
+        id: affected-services
+        uses: actions/github-script@v7
+        with:
+          script: |
+            if (context.eventName === "workflow_dispatch") {
+              if (context.payload.inputs.service === "all") {
+                return `["web", "web-ingestion", "worker"]`
+              }
+              return `["${context.payload.inputs.service}"]`
+            }
+            if (context.eventName === "push") {
+              return `["web", "web-ingestion", "worker"]`
+            }
+            return "[]"
+          result-encoding: string
+      - name: Print services to build
+        uses: actions/github-script@v7
+        env:
+          services: ${{ steps.affected-services.outputs.result }}
+        with:
+          result-encoding: string
+          script: |
+            console.log('Services', `${process.env.services}` ?? 'n/a');
+
+  affected-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      environments: ${{ steps.affected-environments.outputs.result }}
+    steps:
+      - name: Get affected environments
+        id: affected-environments
+        uses: actions/github-script@v7
+        with:
+          script: |
+            if (context.eventName === "workflow_dispatch") {
+              return `["${context.payload.inputs.environment}"]`
+            }
+            if (context.eventName === "push") {
+              if (context.ref === "refs/heads/main") {
+                return `["staging"]`
+              }
+              if (context.ref === "refs/heads/production") {
+                return `["prod-eu-temp", "prod-us"]`
+              }
+            }
+            return "[]"
+          result-encoding: string
+      - name: Print environments to build
+        uses: actions/github-script@v7
+        env:
+          environments: ${{ steps.affected-environments.outputs.result }}
+        with:
+          result-encoding: string
+          script: |
+            console.log('Environments', `${process.env.environments}` ?? 'n/a');
+
+  ecs-deploy:
+    uses: ./.github/workflows/_deploy_ecs_service.yml
+    needs: [affected-services, affected-environments]
+    secrets: inherit
+    strategy:
+      matrix:
+        service: ${{ fromJson(needs.affected-services.outputs.services) }}
+        environment: ${{ fromJson(needs.affected-environments.outputs.environments) }}
+    with:
+      service: ${{ matrix.service }}
+      environment: ${{ matrix.environment }}

.github/workflows/licencecheck.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/pipeline.yml

@@ -16,11 +16,11 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -35,24 +35,20 @@ jobs:
         run: pnpm run lint
 
   test-docker-build:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set NEXT_PUBLIC_BUILD_ID
+        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Build and run both images from compose
         run: |
           docker compose -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Check server health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
-
-      - name: Check worker health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
-
       - name: Ensure no unhealthy status
         run: |
           if docker-compose ps | grep "(unhealthy)"; then
@@ -62,7 +58,16 @@ jobs:
             echo "All services are healthy"
           fi
 
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
   tests-web:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
@@ -75,12 +80,12 @@ jobs:
         with:
           swap-size-gb: 10
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -93,7 +98,7 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          cp .env.dev.example web/.env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
 
       - name: Run + migrate
         run: |
@@ -106,18 +111,28 @@ jobs:
       - name: Seed DB
         run: |
           pnpm run db:migrate
-          pnpm run db:seed
 
       - name: Build
         run: pnpm run build
 
       - name: Start Langfuse
         run: (pnpm run start&)
+        env:
+          LANGFUSE_INIT_ORG_ID: "seed-org-id"
+          LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+          LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
+          LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
+          LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890"
+          LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
+          LANGFUSE_INIT_USER_NAME: "Demo User"
+          LANGFUSE_INIT_USER_PASSWORD: "password"
 
       - name: run tests
         run: pnpm --filter=web run test
 
   tests-worker:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
@@ -130,12 +145,13 @@ jobs:
         with:
           swap-size-gb: 10
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
+
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -145,10 +161,17 @@ jobs:
         run: |
           pnpm install
 
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+
       - name: Load default env
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
+          cp .env.dev.example worker/.env
 
       - name: Run + migrate
         run: |
@@ -156,25 +179,34 @@ jobs:
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
 
+      - name: Ensure no unhealthy status
+        run: |
+          if docker compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
+
       - name: Seed DB
         run: |
           pnpm run db:migrate
           pnpm run db:seed
+          pnpm run --filter=shared ch:up
 
       - name: Build
         run: pnpm --filter=worker... run build
 
       - name: run tests
         run: pnpm --filter=worker run test
-
   e2e-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -209,10 +241,62 @@ jobs:
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
 
+  e2e-server-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
+          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          docker compose ps
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed:examples
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Run server
+        run: (pnpm run start&)
+
+      - name: Run e2e tests
+        run: pnpm --filter=web run test:e2e:server
+
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
+    needs:
+      [
+        lint,
+        tests-web,
+        tests-worker,
+        e2e-tests,
+        test-docker-build,
+        e2e-server-tests,
+      ]
     if: always()
     steps:
       - name: Successful deploy
@@ -238,13 +322,16 @@ jobs:
         with:
           version: 9.5.0
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache-dependency-path: "pnpm-lock.yaml"
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set NEXT_PUBLIC_BUILD_ID
+        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Log in to the GitHub Container registry
         uses: docker/login-action@v2

.github/workflows/porter_stack_web_eu_temp.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-    - production
-name: Deploy to web
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Setup porter
-      uses: porter-dev/setup-porter@v0.1.0
-    - name: Deploy stack
-      timeout-minutes: 30
-      run: exec porter apply
-      env:
-        PORTER_CLUSTER: "4060"
-        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
-        PORTER_HOST: https://dashboard.porter.run
-        PORTER_PR_NUMBER: ${{ github.event.number }}
-        PORTER_PROJECT: "12565"
-        PORTER_REPO_NAME: ${{ github.event.repository.name }}
-        PORTER_STACK_NAME: web
-        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_web_staging.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-      - main
-name: Deploy to web (Staging)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set Github tag
-        id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Setup porter
-        uses: porter-dev/setup-porter@v0.1.0
-      - name: Deploy stack
-        timeout-minutes: 30
-        run: exec porter apply
-        env:
-          PORTER_CLUSTER: "4037"
-          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
-          PORTER_HOST: https://dashboard.getporter.dev
-          PORTER_PR_NUMBER: ${{ github.event.number }}
-          PORTER_PROJECT: "12565"
-          PORTER_REPO_NAME: ${{ github.event.repository.name }}
-          PORTER_STACK_NAME: web
-          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_web_us_temp.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-      - production
-name: Deploy to web (US temp)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set Github tag
-        id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Setup porter
-        uses: porter-dev/setup-porter@v0.1.0
-      - name: Deploy stack
-        timeout-minutes: 30
-        run: exec porter apply
-        env:
-          PORTER_CLUSTER: "4054"
-          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
-          PORTER_HOST: https://dashboard.porter.run
-          PORTER_PR_NUMBER: ${{ github.event.number }}
-          PORTER_PROJECT: "12565"
-          PORTER_REPO_NAME: ${{ github.event.repository.name }}
-          PORTER_STACK_NAME: web
-          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/porter_stack_worker_eu_temp.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-    - production
-name: Deploy to worker (EU temp)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Setup porter
-      uses: porter-dev/setup-porter@v0.1.0
-    - name: Deploy stack
-      timeout-minutes: 30
-      run: exec porter apply
-      env:
-        PORTER_CLUSTER: "4060"
-        PORTER_DEPLOYMENT_TARGET_ID: 3be075c0-b121-4123-aec1-5dc571626b0c
-        PORTER_HOST: https://dashboard.porter.run
-        PORTER_PR_NUMBER: ${{ github.event.number }}
-        PORTER_PROJECT: "12565"
-        PORTER_REPO_NAME: ${{ github.event.repository.name }}
-        PORTER_STACK_NAME: worker
-        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}

.github/workflows/porter_stack_worker_staging.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-      - main
-name: Deploy to worker (staging)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set Github tag
-        id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Setup porter
-        uses: porter-dev/setup-porter@v0.1.0
-      - name: Deploy stack
-        timeout-minutes: 30
-        run: exec porter apply
-        env:
-          PORTER_CLUSTER: "4037"
-          PORTER_DEPLOYMENT_TARGET_ID: c22d8fa2-f97c-4e1f-9cae-83cc6bfbad35
-          PORTER_HOST: https://dashboard.getporter.dev
-          PORTER_PR_NUMBER: ${{ github.event.number }}
-          PORTER_PROJECT: "12565"
-          PORTER_REPO_NAME: ${{ github.event.repository.name }}
-          PORTER_STACK_NAME: worker
-          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}

.github/workflows/porter_stack_worker_us.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-      - production
-name: Deploy to worker (US)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set Github tag
-        id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Setup porter
-        uses: porter-dev/setup-porter@v0.1.0
-      - name: Deploy stack
-        timeout-minutes: 30
-        run: exec porter apply
-        env:
-          PORTER_CLUSTER: "4038"
-          PORTER_DEPLOYMENT_TARGET_ID: 260cd04e-5dc8-4b22-a291-ad4453f981f7
-          PORTER_HOST: https://dashboard.getporter.dev
-          PORTER_PR_NUMBER: ${{ github.event.number }}
-          PORTER_PROJECT: "12565"
-          PORTER_REPO_NAME: ${{ github.event.repository.name }}
-          PORTER_STACK_NAME: worker
-          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4038 }}

.github/workflows/porter_stack_worker_us_temp.yml

@@ -1,29 +0,0 @@
-"on":
-  push:
-    branches:
-      - production
-name: Deploy to worker (US temp)
-jobs:
-  porter-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set Github tag
-        id: vars
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Setup porter
-        uses: porter-dev/setup-porter@v0.1.0
-      - name: Deploy stack
-        timeout-minutes: 30
-        run: exec porter apply
-        env:
-          PORTER_CLUSTER: "4054"
-          PORTER_DEPLOYMENT_TARGET_ID: cddc0f2a-bb19-401a-9819-29f59f76b101
-          PORTER_HOST: https://dashboard.porter.run
-          PORTER_PR_NUMBER: ${{ github.event.number }}
-          PORTER_PROJECT: "12565"
-          PORTER_REPO_NAME: ${{ github.event.repository.name }}
-          PORTER_STACK_NAME: worker
-          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
-          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}

.github/workflows/snyk.yml

@@ -1,5 +1,12 @@
 name: Snyk Container
-on: push
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - main
+
 jobs:
   snyk:
     runs-on: ubuntu-latest
@@ -18,7 +25,7 @@ jobs:
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: web
@@ -33,7 +40,7 @@ jobs:
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: worker

.vscode/launch.json

@@ -2,10 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Next.js: debug server-side",
+      "name": "Web and Worker: server side",
       "type": "node-terminal",
       "request": "launch",
-      "command": "npm run dev"
+      "command": "pnpm run dev"
     },
     {
       "name": "Next.js: debug client-side",

README.md

@@ -1,5 +1,8 @@
 ![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
 
+<div align="center"><h1>Langfuse: Open Source LLM Engineering Platform</h1></div>
+<div align="center"><h4>LLM Observability, Prompt Management, LLM Evaluations,<br/>Datasets, LLM Metrics, and Prompt Playground</h4></div>
+
 <div align="center">
    <div>
       <h3>
@@ -24,20 +27,20 @@
    </div>
    <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
    <br/>
-   <span>We're hiring. <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in Backend Engineering, Product Engineering, and Developer Relations.</span>
+   <span>We're hiring. <a href="https://langfuse.com/careers"><strong>Join us</strong></a> in Product Engineering and Developer Relations.</span>
    <br/>
    <br/>
    <div>
       <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
-      <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse" alt="langfuse npm package"></a>
-      <a href="https://pypi.python.org/pypi/langfuse"><img src="https://img.shields.io/pypi/v/langfuse.svg?style=flat-square&label=pypi+langfuse" alt="langfuse Python package on PyPi"></a>
+      <a href="https://pypi.python.org/pypi/langfuse"><img src="https://img.shields.io/pypi/dm/langfuse?style=flat-square&logo=python&logoColor=white&label=pypi%20langfuse&color=blue" alt="langfuse Python package on PyPi"></a>
+      <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/dm/langfuse?style=flat-square&logo=npm&logoColor=white&label=npm%20langfuse&color=blue" alt="langfuse npm package"></a>
    </div>
 </div>
 </br>
 
-## Overview
+## Langfuse Overview
 
 _Unmute video for voice-over_
 
@@ -45,18 +48,18 @@ https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6
 
 ### Develop
 
-- **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
+- **LLM Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
 - **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
 - **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
-- **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
-- **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
+- **LLM Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
+- **LLM Evaluations:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
+  - Run ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) and LLM-as-a-Judge within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
-  - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
+  - Manually score LLM outputs in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 
 ### Test
 
@@ -72,6 +75,8 @@ Managed deployment by the Langfuse team, generous free-tier (hobby plan), no cre
 
 **[» Langfuse Cloud](https://cloud.langfuse.com)**
 
+## Self-Hosting Open Source LLM Observability with Langfuse
+
 ### Localhost (docker)
 
 ```bash
@@ -87,10 +92,10 @@ docker compose up -d
 
 ### Self-host (docker)
 
-Langfuse is simple to self-host and keep updated. It currently requires only a single docker container.
+Langfuse is simple to self-host and keep updated. It currently requires only a single docker container and a postgres database.
 [→ Self Hosting Instructions](https://langfuse.com/docs/deployment/self-host)
 
-Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and others](https://langfuse.com/docs/deployment/self-host#platform-specific-information)
+Templated deployments: [Railway, GCP, AWS, Azure, Kubernetes and others](https://langfuse.com/docs/deployment/self-host#platform-specific-information)
 
 ## Get Started
 
@@ -98,50 +103,49 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe
 
 You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
-### Ingesting Data · Instrumenting Your Application
+### Ingesting Data · Instrumenting Your Application · LLM Observability with Langfuse
 
 Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
-You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.
-
 See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.
 
-### Integrations
+### LLM Observability Integrations
 
-| Integration                                              | Supports                   | Description                                                                                                                                      |
-| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
-| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
-| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
-| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
-| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
-| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
+| Integration                                                                  | Supports                   | Description                                                                                                                                      |
+| ---------------------------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](https://langfuse.com/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](https://langfuse.com/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](https://langfuse.com/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](https://langfuse.com/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [Vercel AI SDK](https://langfuse.com/docs/integrations/vercel-ai-sdk)        | JS/TS                      | TypeScript toolkit designed to help developers build AI-powered applications with React, Next.js, Vue, Svelte, Node.js.                          |
+| [API](https://langfuse.com/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
 
-Packages that integrate with Langfuse:
+Packages integrated with Langfuse:
 
-| Name                                                       | Description                                                                                                             |
-| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
-| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
-| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
-| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
-| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
-| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
-| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
+| Name                                                            | Description                                                                                                             |
+| --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](https://langfuse.com/docs/integrations/instructor) | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Dify](https://langfuse.com/docs/integrations/dify)             | Open source LLM app development platform with no-code builder.                                                          |
+| [Ollama](https://langfuse.com/docs/integrations/ollama)         | Easily run open source LLMs on your own machine.                                                                        |
+| [Mirascope](https://langfuse.com/docs/integrations/mirascope)   | Python toolkit for building LLM applications.                                                                           |
+| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
 
 ## Questions and feedback
 
 ### Ideas and roadmap
 
-- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
-- [Feature Requests](https://langfuse.com/idea)
 - [Roadmap](https://langfuse.com/roadmap)
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
+- [Feature Requests](https://langfuse.com/ideas)
 
 ### Support and feedback
 
 In order of preference the best way to communicate with us:
 
-- [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions) (preferred): Contribute [ideas](https://langfuse.com/ideas), [support requests](https://langfuse.com/gh-support) and [report bugs](https://langfuse.com/issues)
 - [Discord](https://langfuse.com/discord): community support
 - Privately: contact at langfuse dot com
 

careers

@@ -3,7 +3,6 @@ Join us in scaling Langfuse in Berlin, Germany. We are an open source company, w
 
 _Open Roles_
 
-- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
 - Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
 - Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
 

docker-compose.build.yml

@@ -16,11 +16,21 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
       - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
+      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
+      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
+      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
+      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
+      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
+      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
+      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]

docker-compose.dev.yml

@@ -1,4 +1,54 @@
 services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 5s
+
+  miniocreatebucket:
+    image: minio/mc
+    container_name: miniocreatebucket
+    entrypoint: ["/bin/sh", "-c"]
+    command: >
+      "mc alias set minio http://minio:9000 minio miniosecret && 
+       mc rm -r --force minio/mybucket || true &&
+       mc mb minio/mybucket && 
+       mc policy set download minio/mybucket"
+    depends_on:
+      minio:
+        condition: service_healthy
+
   redis:
     image: redis:7.2.4
     restart: always
@@ -7,7 +57,7 @@ services:
     ports:
       - 6379:6379
 
-  db:
+  postgres:
     image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     command: ["postgres", "-c", "log_statement=all"]
@@ -18,8 +68,14 @@ services:
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

docker-compose.yml

@@ -10,9 +10,19 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
+      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
+      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
+      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
+      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
+      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
+      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
+      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
 
   db:
     image: postgres

ee/package.json

@@ -25,7 +25,9 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "next": "^14.2.3",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "axios": "^1.7.7",
+    "next": "^14.2.15",
     "next-auth": "^4.24.7",
     "zod": "^3.23.8"
   },
@@ -38,7 +40,7 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"

ee/src/ee-license-check/index.ts

@@ -1,4 +1,5 @@
 import { env } from "../env";
 
 export const isEeAvailable: boolean =
-  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ||
+  env.LANGFUSE_EE_LICENSE_KEY !== undefined;

ee/src/env.ts

@@ -1,9 +1,12 @@
 import { z } from "zod";
-import { env as sharedEnv } from "@langfuse/shared";
+import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
-  ADMIN_API_KEY: z.string().optional(),
+  LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };
+export const env = {
+  ...sharedEnv,
+  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
+};

ee/src/index.ts

@@ -1 +0,0 @@
-export { isEeAvailable } from "./ee-license-check";

fern/apis/server/definition/commons.yml

@@ -5,16 +5,36 @@ types:
       id:
         type: string
         docs: The unique identifier of a trace
-      timestamp: datetime
-      name: optional<string>
-      input: optional<unknown>
-      output: optional<unknown>
-      sessionId: optional<string>
-      release: optional<string>
-      version: optional<string>
-      userId: optional<string>
-      metadata: optional<unknown>
-      tags: optional<list<string>>
+      timestamp:
+        type: datetime
+        docs: The timestamp when the trace was created
+      name:
+        type: optional<string>
+        docs: The name of the trace
+      input:
+        type: optional<unknown>
+        docs: The input data of the trace. Can be any JSON.
+      output:
+        type: optional<unknown>
+        docs: The output data of the trace. Can be any JSON.
+      sessionId:
+        type: optional<string>
+        docs: The session identifier associated with the trace
+      release:
+        type: optional<string>
+        docs: The release version of the application when the trace was created
+      version:
+        type: optional<string>
+        docs: The version of the trace
+      userId:
+        type: optional<string>
+        docs: The user identifier associated with the trace
+      metadata:
+        type: optional<unknown>
+        docs: The metadata associated with the trace. Can be any JSON.
+      tags:
+        type: optional<list<string>>
+        docs: The tags associated with the trace. Can be an array of strings or null.
       public:
         type: optional<boolean>
         docs: Public traces are accessible via url without login
@@ -65,37 +85,97 @@ types:
       traces: list<Trace>
   Observation:
     properties:
-      id: string
-      traceId: optional<string>
-      type: string
-      name: optional<string>
-      startTime: datetime
-      endTime: optional<datetime>
-      completionStartTime: optional<datetime>
-      model: optional<string>
-      modelParameters: optional<map<string, MapValue>>
-      input: optional<unknown>
-      version: optional<string>
-      metadata: optional<unknown>
-      output: optional<unknown>
-      usage: optional<Usage>
-      level: ObservationLevel
-      statusMessage: optional<string>
-      parentObservationId: optional<string>
-      promptId: optional<string>
+      id:
+        type: string
+        docs: The unique identifier of the observation
+      traceId:
+        type: optional<string>
+        docs: The trace ID associated with the observation
+      type:
+        type: string
+        docs: The type of the observation
+      name:
+        type: optional<string>
+        docs: The name of the observation
+      startTime:
+        type: datetime
+        docs: The start time of the observation
+      endTime:
+        type: optional<datetime>
+        docs: The end time of the observation.
+      completionStartTime:
+        type: optional<datetime>
+        docs: The completion start time of the observation
+      model:
+        type: optional<string>
+        docs: The model used for the observation
+      modelParameters:
+        type: optional<map<string, MapValue>>
+        docs: The parameters of the model used for the observation
+      input:
+        type: optional<unknown>
+        docs: The input data of the observation
+      version:
+        type: optional<string>
+        docs: The version of the observation
+      metadata:
+        type: optional<unknown>
+        docs: Additional metadata of the observation
+      output:
+        type: optional<unknown>
+        docs: The output data of the observation
+      usage:
+        type: optional<Usage>
+        docs: The usage data of the observation
+      level:
+        type: ObservationLevel
+        docs: The level of the observation
+      statusMessage:
+        type: optional<string>
+        docs: The status message of the observation
+      parentObservationId:
+        type: optional<string>
+        docs: The parent observation ID
+      promptId:
+        type: optional<string>
+        docs: The prompt ID associated with the observation
 
   ObservationsView:
     extends: Observation
     properties:
-      modelId: optional<string>
-      inputPrice: optional<double>
-      outputPrice: optional<double>
-      totalPrice: optional<double>
-      calculatedInputCost: optional<double>
-      calculatedOutputCost: optional<double>
-      calculatedTotalCost: optional<double>
-      latency: optional<double>
-      timeToFirstToken: optional<double>
+      promptName:
+        type: optional<string>
+        docs: The name of the prompt associated with the observation
+      promptVersion:
+        type: optional<integer>
+        docs: The version of the prompt associated with the observation
+      modelId:
+        type: optional<string>
+        docs: The unique identifier of the model
+      inputPrice:
+        type: optional<double>
+        docs: The price of the input in USD
+      outputPrice:
+        type: optional<double>
+        docs: The price of the output in USD.
+      totalPrice:
+        type: optional<double>
+        docs: The total price in USD.
+      calculatedInputCost:
+        type: optional<double>
+        docs: The calculated cost of the input in USD
+      calculatedOutputCost:
+        type: optional<double>
+        docs: The calculated cost of the output in USD
+      calculatedTotalCost:
+        type: optional<double>
+        docs: The calculated total cost in USD
+      latency:
+        type: optional<double>
+        docs: The latency in seconds.
+      timeToFirstToken:
+        type: optional<double>
+        docs: The time to the first token in seconds
 
   Usage:
     docs: Standard interface for usage and cost
@@ -237,14 +317,30 @@ types:
       updatedAt: datetime
   DatasetRun:
     properties:
-      id: string
-      name: string
-      description: optional<string>
-      metadata: optional<unknown>
-      datasetId: string
-      datasetName: string
-      createdAt: datetime
-      updatedAt: datetime
+      id:
+        type: string
+        docs: Unique identifier of the dataset run
+      name:
+        type: string
+        docs: Name of the dataset run
+      description:
+        type: optional<string>
+        docs: Description of the run
+      metadata:
+        type: optional<unknown>
+        docs: Metadata of the dataset run
+      datasetId:
+        type: string
+        docs: Id of the associated dataset
+      datasetName:
+        type: string
+        docs: Name of the associated dataset
+      createdAt:
+        type: datetime
+        docs: The date and time when the dataset run was created
+      updatedAt:
+        type: datetime
+        docs: The date and time when the dataset run was last updated
   DatasetRunWithItems:
     extends: DatasetRun
     properties:

fern/apis/server/definition/dataset-items.yml

@@ -49,7 +49,7 @@ types:
       sourceObservationId: optional<string>
       id:
         type: optional<string>
-        docs: Dataset items are upserted on their id. Id needs to be globally unique and cannot be reused across datasets.
+        docs: Dataset items are upserted on their id. Id needs to be unique (project-level) and cannot be reused across datasets.
       status:
         type: optional<commons.DatasetStatus>
         docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/metrics.yml

@@ -31,10 +31,10 @@ service:
             docs: Optional filter for metrics where traces include all of these tags
           fromTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
+            docs: Optional filter to only include traces and observations on or after a certain datetime (ISO 8601)
           toTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
+            docs: Optional filter to only include traces and observations before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:

fern/apis/server/definition/observations.yml

@@ -39,6 +39,9 @@ service:
           toStartTime:
             type: optional<datetime>
             docs: Retrieve only observations with a start_time before this datetime (ISO 8601).
+          version:
+            type: optional<string>
+            docs: Optional filter to only include observations with a certain version.
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -41,11 +41,17 @@ service:
           limit:
             type: optional<integer>
             docs: limit of items per page
+          fromUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated on or after a certain datetime (ISO 8601)
+          toUpdatedAt:
+            type: optional<datetime>
+            docs: Optional filter to only include prompt versions created/updated before a certain datetime (ISO 8601)
 
       response: PromptMetaListResponse
 
     create:
-      docs: Create a prompt
+      docs: Create a new version for the prompt with the given `name`
       method: POST
       path: /prompts
       request: CreatePromptRequest
@@ -63,6 +69,10 @@ types:
       versions: list<integer>
       labels: list<string>
       tags: list<string>
+      lastUpdatedAt: datetime
+      lastConfig:
+        type: unknown
+        docs: Config object of the most recent prompt version that matches the filters (if any are provided)
 
   CreatePromptRequest:
     union:

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     list:
-      docs: Get sessions.
+      docs: Get sessions
       method: GET
       path: /sessions
       request:

fern/apis/server/definition/trace.yml

@@ -16,7 +16,7 @@ service:
           docs: The unique langfuse identifier of a trace
       response: commons.TraceWithFullDetails
     list:
-      docs: Get list of traces.
+      docs: Get list of traces
       method: GET
       path: /traces
       request:
@@ -44,6 +44,12 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Only traces that include all of these tags will be returned.
+          version:
+            type: optional<string>
+            docs: Optional filter to only include traces with a certain version.
+          release:
+            type: optional<string>
+            docs: Optional filter to only include traces with a certain release.
       response: Traces
 
 types:

fern/apis/server/generators.yml

@@ -3,32 +3,22 @@ groups:
   local:
     generators:
       - name: fernapi/fern-openapi
-        version: 0.0.28
+        version: 0.0.31
         output:
           location: local-file-system
           path: ../../../web/public/generated/api
-        config:
-          namespaceExport: Langfuse
-          allowCustomFetcher: true
       - name: fernapi/fern-python-sdk
-        version: 1.1.0-rc2
+        version: 2.16.0
         output:
           location: local-file-system
           path: ../../../generated/python
         config:
           client_class_name: FernLangfuse
           improved_imports: false
+          inline_request_params: false
           pydantic_config:
             require_optional_fields: false
             use_str_enums: false
-      - name: fernapi/fern-typescript-node-sdk
-        version: 0.7.1
-        output:
-          location: local-file-system
-          path: ../../../generated/typescript-server
-        config:
-          namespaceExport: Langfuse
-          allowCustomFetcher: true
       - name: fernapi/fern-postman
         version: 0.0.45
         output:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.30.3"
+  "version": "0.43.7"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.65.0",
+  "version": "2.85.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -16,8 +16,9 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
@@ -32,7 +33,7 @@
     "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
     "husky": "^9.0.11",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "release-it": "^17.3.0",
     "turbo": "^1.13.4"
   },

packages/config-eslint/package.json

@@ -11,7 +11,7 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.3",
+    "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",

packages/shared/clickhouse/migrations/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces;

packages/shared/clickhouse/migrations/0001_traces.up.sql

@@ -0,0 +1,26 @@
+CREATE TABLE traces (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(String, String) CODEC(ZSTD(1)),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(1)),
+    `output` Nullable(String) CODEC(ZSTD(1)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/migrations/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations;

packages/shared/clickhouse/migrations/0002_observations.up.sql

@@ -0,0 +1,50 @@
+CREATE TABLE observations (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_input_usage_units` Nullable(Decimal64(12)),
+    `provided_output_usage_units` Nullable(Decimal64(12)),
+    `provided_total_usage_units` Nullable(Decimal64(12)),
+    `input_usage_units` Nullable(Decimal64(12)),
+    `output_usage_units` Nullable(Decimal64(12)),
+    `total_usage_units` Nullable(Decimal64(12)),
+    `unit` Nullable(String),
+    `provided_input_cost` Nullable(Decimal64(12)),
+    `provided_output_cost` Nullable(Decimal64(12)),
+    `provided_total_cost` Nullable(Decimal64(12)),
+    `input_cost` Nullable(Decimal64(12)),
+    `output_cost` Nullable(Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(start_time)
+ORDER BY (
+        project_id,
+        `type`,
+        trace_id,
+        toUnixTimestamp(start_time),
+        id
+    );

packages/shared/clickhouse/migrations/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores;

packages/shared/clickhouse/migrations/0003_scores.up.sql

@@ -0,0 +1,25 @@
+CREATE TABLE scores (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+ORDER BY (
+        project_id,
+        trace_id,
+        toUnixTimestamp(timestamp),
+        id
+    );

packages/shared/clickhouse/scripts/down.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the down command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down

packages/shared/clickhouse/scripts/drop.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+# Execute the drop command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/up.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Load environment variables
+source ../../.env
+
+# Check if golang-migrate is installed
+if ! command -v migrate &> /dev/null
+then
+    echo "Error: golang-migrate is not installed or not in PATH."
+    echo "Please install golang-migrate via 'brew install golang-migrate' to run this script."
+    echo "Visit https://github.com/golang-migrate/migrate for more installation instructions."
+    exit 1
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+else
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+fi
+
+# Execute the up command
+migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up

packages/shared/package.json

@@ -21,6 +21,10 @@
       "import": "./dist/src/server/index.js",
       "require": "./dist/src/server/index.js"
     },
+    "./src/server/auth/apiKeys": {
+      "import": "./dist/src/server/auth/apiKeys.js",
+      "require": "./dist/src/server/auth/apiKeys.js"
+    },
     "./encryption": {
       "import": "./dist/src/encryption/index.js",
       "require": "./dist/src/encryption/index.js"
@@ -38,58 +42,75 @@
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
-    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load"
+    "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
+    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
+    "ch:up": "bash clickhouse/scripts/up.sh",
+    "ch:down": "bash clickhouse/scripts/down.sh",
+    "ch:drop": "bash clickhouse/scripts/drop.sh",
+    "ch:reset": "pnpm run ch:down && pnpm run ch:up"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.568.0",
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@aws-sdk/client-s3": "^3.627.0",
+    "@aws-sdk/lib-storage": "^3.667.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
-    "@langchain/anthropic": "^0.1.21",
-    "@langchain/core": "^0.1.61",
-    "@langchain/openai": "^0.0.33",
-    "@prisma/client": "^5.13.0",
+    "@clickhouse/client": "^1.4.0",
+    "@langchain/anthropic": "^0.3.1",
+    "@langchain/aws": "^0.1.0",
+    "@langchain/core": "^0.3.9",
+    "@langchain/openai": "^0.3.0",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "@prisma/client": "^5.20.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
+    "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
-    "kysely": "^0.27.3",
-    "langchain": "^0.2.6",
+    "bullmq": "^5.12.10",
+    "dd-trace": "^5.23.1",
+    "decimal.js": "^10.4.3",
+    "exponential-backoff": "^3.1.1",
+    "ioredis": "^5.4.1",
+    "kysely": "^0.27.4",
+    "langchain": "^0.3.2",
     "lodash": "^4.17.21",
     "next-auth": "^4.24.7",
-    "nodemailer": "^6.9.13",
+    "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
+    "uuid": "^9.0.1",
+    "winston": "^3.14.2",
     "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.22.5"
+    "zod-to-json-schema": "^3.23.2"
   },
   "devDependencies": {
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.10",
     "@types/node": "^20.11.29",
-    "@types/nodemailer": "^6.4.15",
-    "@types/pg": "^8.11.6",
+    "@types/nodemailer": "^6.4.16",
+    "@types/pg": "^8.11.10",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.1.3",
-    "prettier": "^3.2.5",
-    "prisma": "^5.13.0",
+    "kysely-codegen": "^0.16.8",
+    "nodemon": "^3.1.7",
+    "prettier": "^3.3.3",
+    "prisma": "^5.20.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.3"
+    "vitest": "^2.1.2"
   },
   "peerDependencies": {
-    "@types/react": "^18.2.79",
-    "react": "^18.0.0"
+    "@types/react": "~18.2.79",
+    "react": "~18.2.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,14 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const ProjectRole = {
+export const Role = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
-    VIEWER: "VIEWER"
+    VIEWER: "VIEWER",
+    NONE: "NONE"
 } as const;
-export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
+export type Role = (typeof Role)[keyof typeof Role];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -36,11 +37,28 @@ export const ScoreDataType = {
     BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
+export const AnnotationQueueStatus = {
+    PENDING: "PENDING",
+    COMPLETED: "COMPLETED"
+} as const;
+export type AnnotationQueueStatus = (typeof AnnotationQueueStatus)[keyof typeof AnnotationQueueStatus];
+export const AnnotationQueueObjectType = {
+    TRACE: "TRACE",
+    OBSERVATION: "OBSERVATION"
+} as const;
+export type AnnotationQueueObjectType = (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const CommentObjectType = {
+    TRACE: "TRACE",
+    OBSERVATION: "OBSERVATION",
+    SESSION: "SESSION",
+    PROMPT: "PROMPT"
+} as const;
+export type CommentObjectType = (typeof CommentObjectType)[keyof typeof CommentObjectType];
 export const JobType = {
     EVAL: "EVAL"
 } as const;
@@ -72,6 +90,31 @@ export type Account = {
     scope: string | null;
     id_token: string | null;
     session_state: string | null;
+    refresh_token_expires_in: number | null;
+    created_at: number | null;
+};
+export type AnnotationQueue = {
+    id: string;
+    name: string;
+    description: string | null;
+    score_config_ids: Generated<string[]>;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type AnnotationQueueItem = {
+    id: string;
+    queue_id: string;
+    object_id: string;
+    object_type: AnnotationQueueObjectType;
+    status: Generated<AnnotationQueueStatus>;
+    locked_at: Timestamp | null;
+    locked_by_user_id: string | null;
+    annotator_user_id: string | null;
+    completed_at: Timestamp | null;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type ApiKey = {
     id: string;
@@ -90,8 +133,10 @@ export type AuditLog = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     user_id: string;
-    project_id: string;
-    user_project_role: ProjectRole;
+    org_id: string;
+    user_org_role: string;
+    project_id: string | null;
+    user_project_role: string | null;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -113,6 +158,16 @@ export type BatchExport = {
     url: string | null;
     log: string | null;
 };
+export type Comment = {
+    id: string;
+    project_id: string;
+    object_type: CommentObjectType;
+    object_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    content: string;
+    author_user_id: string | null;
+};
 export type CronJobs = {
     name: string;
     last_run: Timestamp | null;
@@ -121,15 +176,16 @@ export type CronJobs = {
 };
 export type Dataset = {
     id: string;
+    project_id: string;
     name: string;
     description: string | null;
     metadata: unknown | null;
-    project_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
 export type DatasetItem = {
     id: string;
+    project_id: string;
     status: Generated<DatasetStatus>;
     input: unknown | null;
     expected_output: unknown | null;
@@ -142,6 +198,7 @@ export type DatasetItem = {
 };
 export type DatasetRunItems = {
     id: string;
+    project_id: string;
     dataset_run_id: string;
     dataset_item_id: string;
     trace_id: string;
@@ -151,6 +208,7 @@ export type DatasetRunItems = {
 };
 export type DatasetRuns = {
     id: string;
+    project_id: string;
     name: string;
     description: string | null;
     metadata: unknown | null;
@@ -221,14 +279,17 @@ export type LlmApiKeys = {
     base_url: string | null;
     custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
+    config: unknown | null;
     project_id: string;
 };
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: ProjectRole;
-    project_id: string;
-    sender_id: string | null;
+    org_id: string;
+    org_role: Role;
+    project_id: string | null;
+    project_role: Role | null;
+    invited_by_user_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -306,6 +367,8 @@ export type ObservationView = {
     unit: string | null;
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
+    prompt_name: string | null;
+    prompt_version: number | null;
     model_id: string | null;
     input_price: string | null;
     output_price: string | null;
@@ -316,6 +379,21 @@ export type ObservationView = {
     latency: number | null;
     time_to_first_token: number | null;
 };
+export type Organization = {
+    id: string;
+    name: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    cloud_config: unknown | null;
+};
+export type OrganizationMembership = {
+    id: string;
+    org_id: string;
+    user_id: string;
+    role: Role;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
     project_id: string;
     encrypted_posthog_api_key: string;
@@ -326,15 +404,16 @@ export type PosthogIntegration = {
 };
 export type Project = {
     id: string;
+    org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     name: string;
-    cloud_config: unknown | null;
 };
 export type ProjectMembership = {
+    org_membership_id: string;
     project_id: string;
     user_id: string;
-    role: ProjectRole;
+    role: Role;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -366,6 +445,7 @@ export type Score = {
     observation_id: string | null;
     config_id: string | null;
     string_value: string | null;
+    queue_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     data_type: Generated<ScoreDataType>;
@@ -462,9 +542,12 @@ export type VerificationToken = {
 };
 export type DB = {
     Account: Account;
+    annotation_queue_items: AnnotationQueueItem;
+    annotation_queues: AnnotationQueue;
     api_keys: ApiKey;
     audit_logs: AuditLog;
     batch_exports: BatchExport;
+    comments: Comment;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;
@@ -479,6 +562,8 @@ export type DB = {
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
+    organization_memberships: OrganizationMembership;
+    organizations: Organization;
     posthog_integrations: PosthogIntegration;
     project_memberships: ProjectMembership;
     projects: Project;

packages/shared/prisma/migrations/20240718004923_datasets_tables_add_projectid_composite_key/migration.sql

@@ -0,0 +1,68 @@
+-- Add project_id to datasets, dataset_items, dataset_runs, dataset_run_items
+ALTER TABLE "dataset_items" 
+ADD COLUMN "project_id" TEXT;
+
+ALTER TABLE "dataset_run_items" 
+ADD COLUMN "project_id" TEXT;
+
+ALTER TABLE "dataset_runs" 
+ADD COLUMN "project_id" TEXT;
+
+-- Backfill project_id for dataset_items
+UPDATE dataset_items
+SET project_id = datasets.project_id
+FROM datasets
+WHERE dataset_items.dataset_id = datasets.id;
+
+-- Backfill project_id for dataset_runs
+UPDATE dataset_runs
+SET project_id = datasets.project_id
+FROM datasets
+WHERE dataset_runs.dataset_id = datasets.id;
+
+-- Backfill project_id for dataset_run_items
+UPDATE dataset_run_items
+SET project_id = dataset_runs.project_id
+FROM dataset_runs
+WHERE dataset_run_items.dataset_run_id = dataset_runs.id;
+
+-- Drop the old foreign keys
+ALTER TABLE "dataset_run_items" 
+DROP CONSTRAINT "dataset_run_items_dataset_item_id_fkey",
+DROP CONSTRAINT "dataset_run_items_dataset_run_id_fkey";
+
+ALTER TABLE "dataset_items" 
+DROP CONSTRAINT "dataset_items_dataset_id_fkey";
+
+ALTER TABLE "dataset_runs" 
+DROP CONSTRAINT "dataset_runs_dataset_id_fkey";
+
+-- Now alter the columns to NOT NULL and update primary keys
+ALTER TABLE "datasets" 
+DROP CONSTRAINT "datasets_pkey",
+ADD CONSTRAINT "datasets_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_items" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_items_pkey",
+ADD CONSTRAINT "dataset_items_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_runs" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_runs_pkey",
+ADD CONSTRAINT "dataset_runs_pkey" PRIMARY KEY ("id", "project_id");
+
+ALTER TABLE "dataset_run_items" 
+ALTER COLUMN "project_id" SET NOT NULL,
+DROP CONSTRAINT "dataset_run_items_pkey",
+ADD CONSTRAINT "dataset_run_items_pkey" PRIMARY KEY ("id", "project_id");
+
+
+-- Add new foreign keys
+ALTER TABLE "dataset_items" ADD CONSTRAINT "dataset_items_dataset_id_project_id_fkey" FOREIGN KEY ("dataset_id", "project_id") REFERENCES "datasets"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_runs" ADD CONSTRAINT "dataset_runs_dataset_id_project_id_fkey" FOREIGN KEY ("dataset_id", "project_id") REFERENCES "datasets"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_dataset_run_id_project_id_fkey" FOREIGN KEY ("dataset_run_id", "project_id") REFERENCES "dataset_runs"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_dataset_item_id_project_id_fkey" FOREIGN KEY ("dataset_item_id", "project_id") REFERENCES "dataset_items"("id", "project_id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240718011733_dataset_runs_add_unique_dataset_id_project_id_name copy/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "dataset_runs_dataset_id_project_id_name_key" ON "dataset_runs"("dataset_id", "project_id", "name");

packages/shared/prisma/migrations/20240718011734_dataset_runs_drop_unique_dataset_id_name/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY "dataset_runs_dataset_id_name_key";

packages/shared/prisma/migrations/20240718011735_observation_view_add_prompt_name_and_version/migration.sql

@@ -0,0 +1,100 @@
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240705154048_observation_view_add_created_at_updated_at and update view must have same columns
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    p.name as prompt_name,         -- added in this change
+    p.version as prompt_version,   -- added in this change
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+LEFT JOIN LATERAL (
+    SELECT
+        prompts.*
+    FROM
+        prompts
+    WHERE prompts.id = o.prompt_id
+    AND prompts.project_id = o.project_id
+    LIMIT 1
+) p ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240807111358_models_add_openai_gpt_4o_2024_08_06/migration.sql

@@ -0,0 +1,18 @@
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o-2024-08-06
+  ('clzjr85f70000ymmzg7hqffra', NULL, 'gpt-4o-2024-08-06', '(?i)^(gpt-4o-2024-08-06)$', NULL, 0.0000025, 0.000010, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }')
+  

packages/shared/prisma/migrations/20240807111359_add_organizations_main_migration/migration.sql

@@ -0,0 +1,164 @@
+-- Table ORGANIZATIONS
+-- Create empty table ORGANIZATIONS
+CREATE TABLE "organizations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "cloud_config" JSONB,
+
+    CONSTRAINT "organizations_pkey" PRIMARY KEY ("id")
+);
+-- Backfill table ORGANIZATIONS: create a new organization for each project, move cloudConfig, and set the org_id on the project to project_id prefixed with 'o'
+INSERT INTO "organizations" ("id", "name", "cloud_config", "created_at")
+SELECT
+  CONCAT('o', "id") as "id", -- This mapping is used in other migration steps as well, keep it consistent
+  "name",
+  "cloud_config",
+  "created_at"
+FROM "projects";
+-- Drop column cloud_config from projects as it's now on organization level
+ALTER TABLE "projects" DROP COLUMN "cloud_config";
+
+
+-- PROJECT.ORG_ID
+-- Add org_id to projects
+ALTER TABLE "projects" ADD COLUMN "org_id" TEXT;
+-- Backfill: org_id on projects, set it to project_id prefixed with 'o'
+UPDATE "projects"
+SET "org_id" = CONCAT('o', "id");
+-- Set not null after backfill
+ALTER TABLE "projects" ALTER COLUMN "org_id" SET NOT NULL;
+-- AddForeignKey
+ALTER TABLE "projects" ADD CONSTRAINT "projects_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "projects_org_id_idx" ON "projects"("org_id");
+
+
+-- ORGANIZATION MEMBERSHIPS
+-- Create UserRole ENUM
+CREATE TYPE "Role" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER', 'NONE');
+-- Create empty table
+CREATE TABLE "organization_memberships" (
+    "id" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "role" "Role" NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_memberships_pkey" PRIMARY KEY ("id")
+);
+-- Add indexes
+CREATE INDEX "organization_memberships_user_id_idx" ON "organization_memberships"("user_id");
+CREATE INDEX "organization_memberships_org_id_idx" ON "organization_memberships"("org_id");
+CREATE UNIQUE INDEX "organization_memberships_org_id_user_id_key" ON "organization_memberships"("org_id", "user_id");
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- AddForeignKey
+ALTER TABLE "organization_memberships" ADD CONSTRAINT "organization_memberships_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+
+
+-- Migrate project memberships to organization memberships
+INSERT INTO "organization_memberships" ("id", "org_id", "user_id", "role", "created_at", "updated_at")
+SELECT
+  md5(random()::text || clock_timestamp()::text || project_id::text || user_id::text)::uuid AS "id",
+  CONCAT('o', "project_id") as "org_id",
+  "user_id",
+  "role"::text::"Role" as "role",
+  "created_at",
+  "updated_at"
+FROM "project_memberships";
+
+
+
+-- Delete all project memberships after migration to organization memberships
+DELETE FROM "project_memberships";
+
+-- Add org_membership_id to project_memberships, ok to be not null as it's a new column on a now empty table
+ALTER TABLE "project_memberships" ADD COLUMN     "org_membership_id" TEXT NOT NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "project_memberships" DROP COLUMN "role",
+ADD COLUMN     "role" "Role" NOT NULL;
+-- AddForeignKey
+ALTER TABLE "project_memberships" ADD CONSTRAINT "project_memberships_org_membership_id_fkey" FOREIGN KEY ("org_membership_id") REFERENCES "organization_memberships"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- CreateIndex
+CREATE INDEX "project_memberships_project_id_idx" ON "project_memberships"("project_id");
+-- CreateIndex
+CREATE INDEX "project_memberships_org_membership_id_idx" ON "project_memberships"("org_membership_id");
+
+
+
+-- AUDIT LOGS
+-- DropForeignKey, just index on these, should remain after project/org/user deletions
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_user_id_fkey";
+ALTER TABLE "audit_logs" DROP CONSTRAINT "audit_logs_project_id_fkey";
+-- Drop not null on project level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" DROP NOT NULL;
+-- Add org-level cols
+ALTER TABLE "audit_logs" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "audit_logs" ADD COLUMN "user_org_role" TEXT;
+-- Backfill org_id
+UPDATE "audit_logs"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "audit_logs"."project_id" = "projects"."id";
+-- Backfill user_org_role with value from user_project_role
+UPDATE "audit_logs"
+SET "user_org_role" = "user_project_role"::text::"Role";
+-- Drop and recreate user_project_role column as text column going forward, empty for historical data as it's all on org level now
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role" TEXT; -- nullable
+-- Add not null on org level cols
+ALTER TABLE "audit_logs" ALTER COLUMN "org_id" SET NOT NULL;
+ALTER TABLE "audit_logs" ALTER COLUMN "user_org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "audit_logs_user_id_idx" ON "audit_logs"("user_id");
+CREATE INDEX "audit_logs_org_id_idx" ON "audit_logs"("org_id");
+
+
+-- MEMBERSHIP INVITATIONS
+-- Rename sender_id to invited_by_user_id
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_sender_id_fkey";
+ALTER TABLE "membership_invitations" RENAME COLUMN "sender_id" TO "invited_by_user_id";
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_invited_by_user_id_fkey" FOREIGN KEY ("invited_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+-- DropForeignKey
+ALTER TABLE "membership_invitations" DROP CONSTRAINT "membership_invitations_project_id_fkey";
+-- AlterTable
+ALTER TABLE "membership_invitations" RENAME COLUMN "role" TO "project_role";
+ALTER TABLE "membership_invitations" ADD COLUMN "org_id" TEXT;
+ALTER TABLE "membership_invitations" ADD COLUMN "org_role" "Role";
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_id" DROP NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "project_role" DROP NOT NULL;
+-- Backfill org id
+UPDATE "membership_invitations"
+SET "org_id" = "projects"."org_id"
+FROM "projects"
+WHERE "membership_invitations"."project_id" = "projects"."id";
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+-- Backfill org role with value from project role
+UPDATE "membership_invitations"
+SET "org_role" = "project_role"::text::"Role";
+-- Set project-level cols to null, as it's now org level for all existing invitations and role enum will change below
+UPDATE "membership_invitations"
+SET "project_role" = NULL, "project_id" = NULL;
+-- Switch to new UserRole enum
+ALTER TABLE "membership_invitations" DROP COLUMN "project_role",
+ADD COLUMN     "project_role" "Role";
+
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- Add not null on org level cols after backfill
+ALTER TABLE "membership_invitations"
+ALTER COLUMN "org_id" SET NOT NULL,
+ALTER COLUMN "org_role" SET NOT NULL;
+-- CreateIndex
+CREATE INDEX "membership_invitations_org_id_idx" ON "membership_invitations"("org_id");
+
+
+-- Drop ProjectRole enum as it is replaced by Role
+DROP TYPE "ProjectRole";

packages/shared/prisma/migrations/20240814223824_model_fix_text_embedding_3_large/migration.sql

@@ -0,0 +1,7 @@
+
+-- Fix model name
+-- https://github.com/langfuse/langfuse/issues/2688
+
+UPDATE models
+SET model_name = 'text-embedding-3-large'
+WHERE id = 'clruwn76700020al7gp8e4g4l'

packages/shared/prisma/migrations/20240814233029_dataset_items_drop_fkey_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "dataset_items" DROP CONSTRAINT "dataset_items_source_trace_id_fkey";

packages/shared/prisma/migrations/20240815171916_add_comments/migration.sql

@@ -0,0 +1,22 @@
+-- CreateEnum
+CREATE TYPE "CommentObjectType" AS ENUM ('TRACE', 'OBSERVATION', 'SESSION', 'PROMPT');
+
+-- CreateTable
+CREATE TABLE "comments" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object_type" "CommentObjectType" NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "content" TEXT NOT NULL,
+    "author_user_id" TEXT,
+
+    CONSTRAINT "comments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "comments_project_id_object_type_object_id_idx" ON "comments"("project_id", "object_type", "object_id");
+
+-- AddForeignKey
+ALTER TABLE "comments" ADD CONSTRAINT "comments_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240913095558_models_add_openai_o1_2024-09-12/migration.sql

@@ -0,0 +1,30 @@
+-- For OpenAI reasoning models, manually tokenizing output string does not account for reasoning tokens, thus would lead to incorrect cost
+-- So no manual tokenizer config is provided
+-- Reference: https://platform.openai.com/docs/guides/reasoning/how-reasoning-works
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- o1-preview
+  ('cm10ivcdp0000gix7lelmbw80', NULL, 'o1-preview', '(?i)^(o1-preview)$', NULL, 0.000015, 0.00006, NULL, 'TOKENS', NULL, NULL), 
+
+  -- o1-preview-2024-09-12
+  ('cm10ivo130000n8x7qopcjjcg', NULL, 'o1-preview-2024-09-12', '(?i)^(o1-preview-2024-09-12)$', NULL, 0.000015, 0.00006, NULL, 'TOKENS', NULL, NULL), 
+
+  -- o1-mini
+  ('cm10ivwo40000r1x7gg3syjq0', NULL, 'o1-mini', '(?i)^(o1-mini)$', NULL, 0.000003, 0.000012, NULL, 'TOKENS', NULL, NULL), 
+
+  -- o1-mini-2024-09-12
+  ('cm10iw6p20000wgx7it1hlb22', NULL, 'o1-mini-2024-09-12', '(?i)^(o1-mini-2024-09-12)$', NULL, 0.000003, 0.000012, NULL, 'TOKENS', NULL, NULL) 
+

packages/shared/prisma/migrations/20240913185822_account_add_refresh_token_expires_in/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "refresh_token_expires_in" INTEGER;

packages/shared/prisma/migrations/20240917183001_remove_covered_indexes_01/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "traces_project_id_idx"; -- covered by traces_project_id_timestamp_idx

packages/shared/prisma/migrations/20240917183002_remove_covered_indexes_02/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "datasets_project_id_idx"; -- covered by datasets_project_id_name_key (unique)

packages/shared/prisma/migrations/20240917183003_remove_covered_indexes_03/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "eval_templates_project_id_idx"; -- covered by eval_templates_project_id_id_idx

packages/shared/prisma/migrations/20240917183004_remove_covered_indexes_04/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "job_configurations_project_id_idx"; -- covered by job_configurations_project_id_id_idx

packages/shared/prisma/migrations/20240917183005_remove_covered_indexes_05/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "job_executions_project_id_idx"; -- covered by job_executions_project_id_id_idx

packages/shared/prisma/migrations/20240917183006_remove_covered_indexes_06/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "llm_api_keys_project_id_provider_idx"; -- covered by llm_api_keys_project_id_provider_key (unique)

packages/shared/prisma/migrations/20240917183007_remove_covered_indexes_07/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "models_project_id_model_name_idx"; -- covered by models_project_id_model_name_start_date_unit_key (unique)

packages/shared/prisma/migrations/20240917183008_remove_covered_indexes_08/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "models_project_id_model_name_start_date_unit_idx"; -- covered by models_project_id_model_name_start_date_unit_key (unique)

packages/shared/prisma/migrations/20240917183009_remove_covered_indexes_09/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "observations_project_id_idx"; -- covered by observations_project_id_start_time_type_idx

packages/shared/prisma/migrations/20240917183010_remove_covered_indexes_10/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "observations_trace_id_idx"; -- covered by observations_trace_id_project_id_start_time_idx

packages/shared/prisma/migrations/20240917183011_remove_covered_indexes_11/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "observations_trace_id_project_id_idx"; -- covered by observations_trace_id_project_id_start_time_idx

packages/shared/prisma/migrations/20240917183012_remove_covered_indexes_12/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "organization_memberships_org_id_idx"; -- covered by organization_memberships_org_id_user_id_key (unique)

packages/shared/prisma/migrations/20240917183013_remove_covered_indexes_13/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "posthog_integrations_project_id_idx"; -- covered by posthog_integrations_pkey (unique)

packages/shared/prisma/migrations/20240917183014_remove_covered_indexes_14/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "prompts_project_id_idx"; -- covered by prompts_project_id_name_version_key (unique)

packages/shared/prisma/migrations/20240917183015_remove_covered_indexes_15/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "prompts_project_id_name_version_idx"; -- covered by prompts_project_id_name_version_key (unique)

packages/shared/prisma/migrations/20240917183016_remove_covered_indexes_16/migration.sql

@@ -0,0 +1,3 @@
+-- DropIndex
+-- Removes indexes that are covered by other indexes on the same table, e.g. (project_id) if (project_id, timestamp) exists.
+DROP INDEX CONCURRENTLY IF EXISTS "scores_project_id_idx"; -- covered by scores_project_id_name_idx

packages/shared/prisma/migrations/20241009042557_auth_add_created_at_for_gitlab/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "created_at" INTEGER;

packages/shared/prisma/migrations/20241009110720_scores_add_nullable_queue_id_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "queue_id" TEXT;

packages/shared/prisma/migrations/20241009113245_add_annotation_queue/migration.sql

@@ -0,0 +1,75 @@
+-- CreateEnum
+CREATE TYPE "AnnotationQueueStatus" AS ENUM ('PENDING', 'COMPLETED');
+
+-- CreateEnum
+CREATE TYPE "AnnotationQueueObjectType" AS ENUM ('TRACE', 'OBSERVATION');
+
+-- CreateTable
+CREATE TABLE "annotation_queues" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT,
+    "score_config_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queues_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "annotation_queue_items" (
+    "id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "object_type" "AnnotationQueueObjectType" NOT NULL,
+    "status" "AnnotationQueueStatus" NOT NULL DEFAULT 'PENDING',
+    "locked_at" TIMESTAMP(3),
+    "locked_by_user_id" TEXT,
+    "annotator_user_id" TEXT,
+    "completed_at" TIMESTAMP(3),
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_items_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_id_project_id_idx" ON "annotation_queues"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_project_id_created_at_idx" ON "annotation_queues"("project_id", "created_at");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queues_project_id_name_key" ON "annotation_queues"("project_id", "name");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_id_project_id_idx" ON "annotation_queue_items"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_project_id_queue_id_status_idx" ON "annotation_queue_items"("project_id", "queue_id", "status");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_object_id_object_type_project_id_que_idx" ON "annotation_queue_items"("object_id", "object_type", "project_id", "queue_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_annotator_user_id_idx" ON "annotation_queue_items"("annotator_user_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_created_at_idx" ON "annotation_queue_items"("created_at");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queues" ADD CONSTRAINT "annotation_queues_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_locked_by_user_id_fkey" FOREIGN KEY ("locked_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_annotator_user_id_fkey" FOREIGN KEY ("annotator_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241010120245_llm_keys_add_config/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "llm_api_keys" ADD COLUMN "config" JSONB;

packages/shared/prisma/schema.prisma

@@ -34,21 +34,23 @@ generator kysely {
 
 // Necessary for Next auth
 model Account {
-    id                String  @id @default(cuid())
-    userId            String  @map("user_id")
-    type              String
-    provider          String
-    providerAccountId String
-    refresh_token     String? // @db.Text
-    access_token      String? // @db.Text
-    expires_at        Int?
-    expires_in        Int?
-    ext_expires_in    Int?
-    token_type        String?
-    scope             String?
-    id_token          String? // @db.Text
-    session_state     String?
-    user              User    @relation(fields: [userId], references: [id], onDelete: Cascade)
+    id                       String  @id @default(cuid())
+    userId                   String  @map("user_id")
+    type                     String
+    provider                 String
+    providerAccountId        String
+    refresh_token            String? // @db.Text
+    access_token             String? // @db.Text
+    expires_at               Int?
+    expires_in               Int?
+    ext_expires_in           Int?
+    token_type               String?
+    scope                    String?
+    id_token                 String? // @db.Text
+    session_state            String?
+    refresh_token_expires_in Int?
+    created_at               Int? // GitLab
+    user                     User    @relation(fields: [userId], references: [id], onDelete: Cascade)
 
     @@unique([provider, providerAccountId])
     @@index([userId])
@@ -63,21 +65,23 @@ model Session {
 }
 
 model User {
-    id                 String                 @id @default(cuid())
-    name               String?
-    email              String?                @unique
-    emailVerified      DateTime?              @map("email_verified")
-    password           String?
-    image              String?
-    admin              Boolean                @default(false)
-    accounts           Account[]
-    sessions           Session[]
-    projectMemberships ProjectMembership[]
-    invitations        MembershipInvitation[]
-    createdAt          DateTime               @default(now()) @map("created_at")
-    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags       String[]               @default([]) @map("feature_flags")
-    AuditLog           AuditLog[]
+    id                      String                   @id @default(cuid())
+    name                    String?
+    email                   String?                  @unique
+    emailVerified           DateTime?                @map("email_verified")
+    password                String?
+    image                   String?
+    admin                   Boolean                  @default(false)
+    accounts                Account[]
+    sessions                Session[]
+    organizationMemberships OrganizationMembership[]
+    projectMemberships      ProjectMembership[]
+    invitations             MembershipInvitation[]
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    featureFlags            String[]                 @default([]) @map("feature_flags")
+    annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
+    annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
 
     @@map("users")
 }
@@ -91,32 +95,49 @@ model VerificationToken {
     @@map("verification_tokens")
 }
 
-model Project {
-    id                 String                 @id @default(cuid())
-    createdAt          DateTime               @default(now()) @map("created_at")
-    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
-    name               String
-    cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    projectMembers     ProjectMembership[]
-    traces             Trace[]
-    observations       Observation[]
-    apiKeys            ApiKey[]
-    dataset            Dataset[]
-    RawEvents          Events[]
-    invitations        MembershipInvitation[]
-    sessions           TraceSession[]
-    Prompt             Prompt[]
-    Model              Model[]
-    AuditLog           AuditLog[]
-    EvalTemplate       EvalTemplate[]
-    JobConfiguration   JobConfiguration[]
-    JobExecution       JobExecution[]
-    LlmApiKeys         LlmApiKeys[]
-    PosthogIntegration PosthogIntegration[]
-    Score              Score[]
-    scoreConfig        ScoreConfig[]
-    BatchExport        BatchExport[]
+model Organization {
+    id                      String                   @id @default(cuid())
+    name                    String
+    createdAt               DateTime                 @default(now()) @map("created_at")
+    updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
+    cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+    organizationMemberships OrganizationMembership[]
+    projects                Project[]
+    MembershipInvitation    MembershipInvitation[]
 
+    @@map("organizations")
+}
+
+model Project {
+    id                  String                 @id @default(cuid())
+    orgId               String                 @map("org_id")
+    createdAt           DateTime               @default(now()) @map("created_at")
+    updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+    name                String
+    projectMembers      ProjectMembership[]
+    organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+    traces              Trace[]
+    observations        Observation[]
+    apiKeys             ApiKey[]
+    dataset             Dataset[]
+    RawEvents           Events[]
+    invitations         MembershipInvitation[]
+    sessions            TraceSession[]
+    Prompt              Prompt[]
+    Model               Model[]
+    EvalTemplate        EvalTemplate[]
+    JobConfiguration    JobConfiguration[]
+    JobExecution        JobExecution[]
+    LlmApiKeys          LlmApiKeys[]
+    PosthogIntegration  PosthogIntegration[]
+    Score               Score[]
+    scoreConfig         ScoreConfig[]
+    BatchExport         BatchExport[]
+    comment             Comment[]
+    annotationQueue     AnnotationQueue[]
+    annotationQueueItem AnnotationQueueItem[]
+
+    @@index([orgId])
     @@map("projects")
 }
 
@@ -152,50 +173,76 @@ model LlmApiKeys {
     baseURL           String?  @map("base_url")
     customModels      String[] @default([]) @map("custom_models")
     withDefaultModels Boolean  @default(true) @map("with_default_models")
+    config            Json?
 
     projectId String  @map("project_id")
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     @@unique([projectId, provider])
-    @@index([projectId, provider])
     @@map("llm_api_keys")
 }
 
+model OrganizationMembership {
+    id                 String              @id @default(cuid())
+    orgId              String              @map("org_id")
+    organization       Organization        @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    userId             String              @map("user_id")
+    user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role               Role                @map("role")
+    createdAt          DateTime            @default(now()) @map("created_at")
+    updatedAt          DateTime            @default(now()) @updatedAt @map("updated_at")
+    ProjectMemberships ProjectMembership[]
+
+    @@unique([orgId, userId])
+    @@index([userId])
+    @@map("organization_memberships")
+}
+
+// Set a project-specific role for a user in an organization
 model ProjectMembership {
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String      @map("user_id")
-    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      ProjectRole
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    orgMembershipId        String                 @map("org_membership_id")
+    organizationMembership OrganizationMembership @relation(fields: [orgMembershipId], references: [id], onDelete: Cascade)
+    projectId              String                 @map("project_id")
+    project                Project                @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId                 String                 @map("user_id")
+    user                   User                   @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role                   Role
+    createdAt              DateTime               @default(now()) @map("created_at")
+    updatedAt              DateTime               @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
+    @@index([projectId])
+    @@index([orgMembershipId])
     @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String      @id @unique @default(cuid())
-    email     String
-    role      ProjectRole
-    projectId String      @map("project_id")
-    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?     @map("sender_id")
-    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime    @default(now()) @map("created_at")
-    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
+    id              String       @id @unique @default(cuid())
+    email           String
+    orgId           String       @map("org_id")
+    organization    Organization @relation(fields: [orgId], references: [id], onDelete: Cascade)
+    orgRole         Role         @map("org_role")
+    projectId       String?      @map("project_id")
+    project         Project?     @relation(fields: [projectId], references: [id], onDelete: SetNull)
+    projectRole     Role?        @map("project_role")
+    invitedByUserId String?      @map("invited_by_user_id")
+    invitedByUser   User?        @relation(fields: [invitedByUserId], references: [id], onDelete: SetNull)
+    createdAt       DateTime     @default(now()) @map("created_at")
+    updatedAt       DateTime     @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
+    @@index([orgId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum ProjectRole {
+enum Role {
     OWNER
     ADMIN
     MEMBER
     VIEWER
+    NONE
 }
 
 model TraceSession {
@@ -238,10 +285,8 @@ model Trace {
     createdAt  DateTime      @default(now()) @map("created_at")
     updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    DatasetItem  DatasetItem[]
     JobExecution JobExecution[]
 
-    @@index([projectId])
     @@index([projectId, timestamp])
     @@index([sessionId])
     @@index([name])
@@ -323,9 +368,8 @@ model Observation {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
 
-    completionStartTime DateTime?     @map("completion_start_time")
-    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    derivedDatasetItems DatasetItem[]
+    completionStartTime DateTime? @map("completion_start_time")
+    project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     promptId String? @map("prompt_id") // no fk constraint, prompt can be deleted
 
@@ -333,12 +377,9 @@ model Observation {
     @@index([projectId, internalModel, startTime, unit])
     @@index([traceId, projectId, type, startTime])
     @@index([traceId, projectId, startTime])
-    @@index([traceId, projectId])
-    @@index([traceId])
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
-    @@index(projectId)
     @@index(model)
     @@index(internalModel)
     @@index([projectId, promptId])
@@ -376,7 +417,10 @@ view ObservationView {
     unit                String?
     completionStartTime DateTime? @map("completion_start_time")
 
-    promptId String? @map("prompt_id")
+    // prompts
+    promptId      String? @map("prompt_id")
+    promptName    String? @map("prompt_name")
+    promptVersion Int?    @map("prompt_version")
 
     // model fields
     modelId     String?  @map("model_id")
@@ -421,6 +465,7 @@ model Score {
     observationId String?        @map("observation_id")
     configId      String?        @map("config_id")
     stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+    queueId       String?        @map("queue_id")
     createdAt     DateTime       @default(now()) @map("created_at")
     updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
     dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
@@ -430,7 +475,6 @@ model Score {
     @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
-    @@index([projectId])
     @@index([projectId, name])
     @@index([authorUserId])
     @@index([configId])
@@ -478,6 +522,59 @@ enum ScoreDataType {
     BOOLEAN
 }
 
+model AnnotationQueue {
+    id                  String                @id @default(cuid())
+    name                String
+    description         String?
+    scoreConfigIds      String[]              @default([]) @map("score_config_ids")
+    projectId           String                @map("project_id")
+    project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    createdAt           DateTime              @default(now()) @map("created_at")
+    updatedAt           DateTime              @default(now()) @updatedAt @map("updated_at")
+    annotationQueueItem AnnotationQueueItem[]
+
+    @@unique([projectId, name])
+    @@index([id, projectId])
+    @@index([projectId, createdAt])
+    @@map("annotation_queues")
+}
+
+model AnnotationQueueItem {
+    id              String                    @id @default(cuid())
+    queueId         String                    @map("queue_id")
+    queue           AnnotationQueue           @relation(fields: [queueId], references: [id], onDelete: Cascade)
+    objectId        String                    @map("object_id")
+    objectType      AnnotationQueueObjectType @map("object_type")
+    status          AnnotationQueueStatus     @default(PENDING)
+    lockedAt        DateTime?                 @map("locked_at")
+    lockedByUserId  String?                   @map("locked_by_user_id")
+    lockedByUser    User?                     @relation("LockedByUser", fields: [lockedByUserId], references: [id], onDelete: SetNull)
+    annotatorUserId String?                   @map("annotator_user_id")
+    annotatorUser   User?                     @relation("AnnotatorUser", fields: [annotatorUserId], references: [id], onDelete: SetNull)
+    completedAt     DateTime?                 @map("completed_at")
+    projectId       String                    @map("project_id")
+    project         Project                   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    createdAt       DateTime                  @default(now()) @map("created_at")
+    updatedAt       DateTime                  @default(now()) @updatedAt @map("updated_at")
+
+    @@index([id, projectId])
+    @@index([projectId, queueId, status])
+    @@index([objectId, objectType, projectId, queueId])
+    @@index([annotatorUserId])
+    @@index([createdAt])
+    @@map("annotation_queue_items")
+}
+
+enum AnnotationQueueStatus {
+    PENDING
+    COMPLETED
+}
+
+enum AnnotationQueueObjectType {
+    TRACE
+    OBSERVATION
+}
+
 model CronJobs {
     name         String    @id
     lastRun      DateTime? @map("last_run")
@@ -488,40 +585,40 @@ model CronJobs {
 }
 
 model Dataset {
-    id           String        @id @default(cuid())
+    id           String        @default(cuid())
+    projectId    String        @map("project_id")
     name         String
     description  String?
     metadata     Json?
-    projectId    String        @map("project_id")
     project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     createdAt    DateTime      @default(now()) @map("created_at")
     updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
     datasetItems DatasetItem[]
     datasetRuns  DatasetRuns[]
 
+    @@id([id, projectId])
     @@unique([projectId, name])
-    @@index([projectId], type: Hash)
     @@index([createdAt])
     @@index([updatedAt])
     @@map("datasets")
 }
 
 model DatasetItem {
-    id                  String            @id @default(cuid())
+    id                  String            @default(cuid())
+    projectId           String            @map("project_id")
     status              DatasetStatus     @default(ACTIVE)
     input               Json?
     expectedOutput      Json?             @map("expected_output")
     metadata            Json?
     sourceTraceId       String?           @map("source_trace_id")
-    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
     sourceObservationId String?           @map("source_observation_id")
-    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
     datasetId           String            @map("dataset_id")
-    dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    dataset             Dataset           @relation(fields: [datasetId, projectId], references: [id, projectId], onDelete: Cascade)
     createdAt           DateTime          @default(now()) @map("created_at")
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@id([id, projectId])
     @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
@@ -536,17 +633,19 @@ enum DatasetStatus {
 }
 
 model DatasetRuns {
-    id              String            @id @default(cuid())
+    id              String            @default(cuid())
+    projectId       String            @map("project_id")
     name            String
     description     String?
     metadata        Json?
     datasetId       String            @map("dataset_id")
-    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    dataset         Dataset           @relation(fields: [datasetId, projectId], references: [id, projectId], onDelete: Cascade)
     createdAt       DateTime          @default(now()) @map("created_at")
     updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems DatasetRunItems[]
 
-    @@unique([datasetId, name])
+    @@id([id, projectId])
+    @@unique([datasetId, projectId, name])
     @@index([datasetId], type: Hash)
     @@index([createdAt])
     @@index([updatedAt])
@@ -554,16 +653,18 @@ model DatasetRuns {
 }
 
 model DatasetRunItems {
-    id            String      @id @default(cuid())
+    id            String      @default(cuid())
+    projectId     String      @map("project_id")
     datasetRunId  String      @map("dataset_run_id")
-    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId, projectId], references: [id, projectId], onDelete: Cascade)
     datasetItemId String      @map("dataset_item_id")
-    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    datasetItem   DatasetItem @relation(fields: [datasetItemId, projectId], references: [id, projectId], onDelete: Cascade)
     traceId       String      @map("trace_id")
     observationId String?     @map("observation_id")
     createdAt     DateTime    @default(now()) @map("created_at")
     updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
+    @@id([id, projectId])
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)
     @@index([observationId], type: Hash)
@@ -588,6 +689,28 @@ model Events {
     @@map("events")
 }
 
+model Comment {
+    id           String            @id @default(cuid())
+    projectId    String            @map("project_id")
+    project      Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    objectType   CommentObjectType @map("object_type")
+    objectId     String            @map("object_id")
+    createdAt    DateTime          @default(now()) @map("created_at")
+    updatedAt    DateTime          @default(now()) @updatedAt @map("updated_at")
+    content      String
+    authorUserId String?           @map("author_user_id") // no fk constraint, user can be deleted
+
+    @@index([projectId, objectType, objectId])
+    @@map("comments")
+}
+
+enum CommentObjectType {
+    TRACE
+    OBSERVATION
+    SESSION
+    PROMPT
+}
+
 model Prompt {
     id        String   @id @default(cuid())
     createdAt DateTime @default(now()) @map("created_at")
@@ -608,9 +731,7 @@ model Prompt {
     labels   String[] @default([])
 
     @@unique([projectId, name, version])
-    @@index([projectId, name, version])
     @@index([projectId, id])
-    @@index([projectId])
     @@index([createdAt])
     @@index([updatedAt])
     @@index([tags(ops: ArrayOps)], type: Gin)
@@ -637,28 +758,29 @@ model Model {
     tokenizerConfig Json?     @map("tokenizer_config")
 
     @@unique([projectId, modelName, startDate, unit])
-    @@index([projectId, modelName, startDate, unit])
-    @@index([projectId, modelName])
     @@index(modelName)
     @@map("models")
 }
 
+// No FK constraints to preserve audit logs
 model AuditLog {
-    id              String      @id @default(cuid())
-    createdAt       DateTime    @default(now()) @map("created_at")
-    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
-    userId          String      @map("user_id")
-    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String      @map("project_id")
-    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole ProjectRole @map("user_project_role")
-    resourceType    String      @map("resource_type")
-    resourceId      String      @map("resource_id")
+    id              String   @id @default(cuid())
+    createdAt       DateTime @default(now()) @map("created_at")
+    updatedAt       DateTime @default(now()) @updatedAt @map("updated_at")
+    userId          String   @map("user_id")
+    orgId           String   @map("org_id")
+    userOrgRole     String   @map("user_org_role")
+    projectId       String?  @map("project_id")
+    userProjectRole String?  @map("user_project_role")
+    resourceType    String   @map("resource_type")
+    resourceId      String   @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
 
     @@index([projectId])
+    @@index([userId])
+    @@index([orgId])
     @@index([createdAt])
     @@index([updatedAt])
     @@map("audit_logs")
@@ -684,7 +806,6 @@ model EvalTemplate {
 
     @@unique([projectId, name, version])
     @@index([projectId, id])
-    @@index([projectId])
     @@map("eval_templates")
 }
 
@@ -719,7 +840,6 @@ model JobConfiguration {
     JobExecution    JobExecution[]
 
     @@index([projectId, id])
-    @@index([projectId])
     @@map("job_configurations")
 }
 
@@ -754,7 +874,6 @@ model JobExecution {
 
     @@index([projectId, status])
     @@index([projectId, id])
-    @@index([projectId])
     @@index([jobConfigurationId])
     @@index([jobOutputScoreId])
     @@index([jobInputTraceId])
@@ -787,7 +906,6 @@ model PosthogIntegration {
     enabled                Boolean
     createdAt              DateTime  @default(now()) @map("created_at")
 
-    @@index([projectId])
     @@map("posthog_integrations")
 }
 

packages/shared/prisma/seed.ts

@@ -5,6 +5,7 @@ import {
   ObservationType,
   ScoreSource,
   ScoreDataType,
+  AnnotationQueueObjectType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -12,8 +13,9 @@ import { parseArgs } from "node:util";
 import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
-import { getDisplaySecretKey, hashSecretKey } from "../src/server";
+import { getDisplaySecretKey, hashSecretKey, logger } from "../src/server";
 import { encrypt } from "../src/encryption";
+import { redis } from "../src/server/redis/redis";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -33,39 +35,114 @@ async function main() {
     options,
   }).values.environment;
 
+  const seedOrgId = "seed-org-id";
+  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  const seedUserId1 = "user-1"; // Owner of org
+  const seedUserId2 = "user-2"; // Member of org, admin of project
+
   const user = await prisma.user.upsert({
-    where: { id: "user-1" },
+    where: { id: seedUserId1 },
     update: {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
     },
     create: {
-      id: "user-1",
+      id: seedUserId1,
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
       image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
+  const user2 = await prisma.user.upsert({
+    where: { id: seedUserId2 },
+    update: {
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+    create: {
+      id: seedUserId2,
+      name: "Demo User 2",
+      email: "member@langfuse.com",
+      password: await hash("password", 12),
+    },
+  });
 
-  const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+  await prisma.organization.upsert({
+    where: { id: seedOrgId },
+    update: {
+      name: "Seed Org",
+    },
+    create: {
+      id: seedOrgId,
+      name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
+    },
+  });
 
   const project1 = await prisma.project.upsert({
     where: { id: seedProjectId },
     update: {
       name: "llm-app",
+      orgId: seedOrgId,
     },
     create: {
       id: seedProjectId,
       name: "llm-app",
-      projectMembers: {
-        create: {
-          role: "OWNER",
-          userId: user.id,
-        },
+      orgId: seedOrgId,
+    },
+  });
+
+  const orgMembership = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user.id,
+        orgId: seedOrgId,
       },
     },
+    create: {
+      userId: user.id,
+      orgId: seedOrgId,
+      role: "OWNER",
+    },
+    update: {},
+  });
+
+  const orgMembership2 = await prisma.organizationMembership.upsert({
+    where: {
+      orgId_userId: {
+        userId: user2.id,
+        orgId: seedOrgId,
+      },
+    },
+    create: {
+      userId: user2.id,
+      orgId: seedOrgId,
+      role: "MEMBER",
+    },
+    update: {},
+  });
+
+  const projectMembership = await prisma.projectMembership.upsert({
+    where: {
+      projectId_userId: {
+        projectId: project1.id,
+        userId: user2.id,
+      },
+    },
+    create: {
+      userId: user2.id,
+      projectId: project1.id,
+      role: "ADMIN",
+      orgMembershipId: orgMembership2.id,
+    },
+    update: {
+      orgMembershipId: orgMembership2.id,
+    },
   });
 
   await prisma.prompt.upsert({
@@ -113,18 +190,39 @@ async function main() {
 
   // Do not run the following for local docker compose setup
   if (environment === "examples" || environment === "load") {
-    const project2 = await prisma.project.upsert({
-      where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+    const seedOrgIdOrg2 = "demo-org-id";
+    const project2Id = "239ad00f-562f-411d-af14-831c75ddd875";
+    const org2 = await prisma.organization.upsert({
+      where: { id: seedOrgIdOrg2 },
+      update: {
+        name: "Langfuse Demo",
+      },
       create: {
-        id: "239ad00f-562f-411d-af14-831c75ddd875",
+        id: seedOrgIdOrg2,
+        name: "Langfuse Demo",
+      },
+    });
+    const project2 = await prisma.project.upsert({
+      where: { id: project2Id },
+      create: {
+        id: project2Id,
         name: "demo-app",
-        projectMembers: {
-          create: {
-            role: "OWNER",
-            userId: user.id,
-          },
+        orgId: org2.id,
+      },
+      update: { orgId: seedOrgIdOrg2 },
+    });
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: {
+          userId: user.id,
+          orgId: seedOrgIdOrg2,
         },
       },
+      create: {
+        userId: user.id,
+        orgId: seedOrgIdOrg2,
+        role: "VIEWER",
+      },
       update: {},
     });
 
@@ -156,6 +254,11 @@ async function main() {
       project2,
     ]);
 
+    const queueIds = await generateQueuesForProject(
+      [project1, project2],
+      configIdsAndNames
+    );
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -163,21 +266,38 @@ async function main() {
 
     const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;
 
-    const { traces, observations, scores, sessions, events } = createObjects(
+    const {
+      traces,
+      observations,
+      scores,
+      sessions,
+      events,
+      comments,
+      queueItems,
+    } = createObjects(
       traceVolume,
       envTags,
       colorTags,
       project1,
       project2,
       promptIds,
+      queueIds,
       configIdsAndNames
     );
 
-    console.log(
+    logger.info(
       `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
     );
 
-    await uploadObjects(traces, observations, scores, sessions, events);
+    await uploadObjects(
+      traces,
+      observations,
+      scores,
+      sessions,
+      events,
+      comments,
+      queueItems
+    );
 
     // If openai key is in environment, add it to the projects LLM API keys
     const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
@@ -193,7 +313,7 @@ async function main() {
         },
       });
     } else {
-      console.warn(
+      logger.warn(
         "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
       );
     }
@@ -286,6 +406,7 @@ async function main() {
             : undefined;
         const datasetItem = await prisma.datasetItem.create({
           data: {
+            projectId: project2.id,
             datasetId: dataset.id,
             sourceTraceId: sourceObservation?.traceId,
             sourceObservationId:
@@ -312,6 +433,7 @@ async function main() {
       for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.create({
           data: {
+            projectId: project2.id,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
@@ -336,6 +458,7 @@ async function main() {
 
           await prisma.datasetRunItems.create({
             data: {
+              projectId: project2.id,
               datasetItemId,
               traceId: observation.traceId as string,
               observationId: Math.random() > 0.5 ? observation.id : undefined,
@@ -351,10 +474,14 @@ async function main() {
 main()
   .then(async () => {
     await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
   })
   .catch(async (e) => {
-    console.error(e);
+    logger.error(e);
     await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
     process.exit(1);
   });
 
@@ -363,7 +490,9 @@ async function uploadObjects(
   observations: Prisma.ObservationCreateManyInput[],
   scores: Prisma.ScoreCreateManyInput[],
   sessions: Prisma.TraceSessionCreateManyInput[],
-  events: Prisma.ObservationCreateManyInput[]
+  events: Prisma.ObservationCreateManyInput[],
+  comments: Prisma.CommentCreateManyInput[],
+  queueItems: Prisma.AnnotationQueueItemCreateManyInput[]
 ) {
   let promises: Prisma.PrismaPromise<unknown>[] = [];
 
@@ -383,7 +512,7 @@ async function uploadObjects(
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
-      console.log(
+      logger.info(
         `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
@@ -400,7 +529,7 @@ async function uploadObjects(
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
-      console.log(
+      logger.info(
         `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
@@ -417,7 +546,7 @@ async function uploadObjects(
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
-      console.log(
+      logger.info(
         `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
@@ -434,7 +563,7 @@ async function uploadObjects(
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
-      console.log(
+      logger.info(
         `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
@@ -450,11 +579,43 @@ async function uploadObjects(
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
-      console.log(
+      logger.info(
         `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`
       );
     await promises[i];
   }
+
+  promises = [];
+  chunk(comments, chunkSize).forEach((chunk) => {
+    promises.push(
+      prisma.comment.createMany({
+        data: chunk,
+      })
+    );
+  });
+  for (let i = 0; i < promises.length; i++) {
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      logger.info(
+        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`
+      );
+    await promises[i];
+  }
+
+  promises = [];
+  chunk(queueItems, chunkSize).forEach((chunk) => {
+    promises.push(
+      prisma.annotationQueueItem.createMany({
+        data: chunk,
+      })
+    );
+  });
+  for (let i = 0; i < promises.length; i++) {
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      logger.info(
+        `Seeding of Annotation Queue Items ${((i + 1) / promises.length) * 100}% complete`
+      );
+    await promises[i];
+  }
 }
 
 function createObjects(
@@ -464,6 +625,7 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
+  queueIds: Map<string, string[]>,
   configParams: Map<
     string,
     {
@@ -480,6 +642,8 @@ function createObjects(
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
   const configs: Prisma.ScoreConfigCreateManyInput[] = [];
+  const comments: Prisma.CommentCreateManyInput[] = [];
+  const queueItems: Prisma.AnnotationQueueItemCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -510,6 +674,7 @@ function createObjects(
     const trace = {
       id: `trace-${v4()}`,
       timestamp: traceTs,
+      createdAt: traceTs,
       projectId: projectId,
       name: ["generate-outreach", "label-inbound", "draft-response"][
         i % 3
@@ -561,6 +726,21 @@ function createObjects(
       }),
     };
 
+    const queueItem = [
+      ...(Math.random() > 0.9 && queueIds.get(projectId)?.[0]
+        ? [
+            {
+              queueId: queueIds.get(projectId)?.[0] as string,
+              objectId: trace.id,
+              objectType: AnnotationQueueObjectType.TRACE,
+              projectId,
+            },
+          ]
+        : []),
+    ];
+
+    queueItems.push(...queueItem);
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
@@ -568,6 +748,7 @@ function createObjects(
               traceId: trace.id,
               name: annotationScoreName,
               timestamp: traceTs,
+              createdAt: traceTs,
               source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
@@ -584,6 +765,7 @@ function createObjects(
               name: "sentiment",
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
+              createdAt: traceTs,
               source: ScoreSource.API,
               projectId,
               dataType: ScoreDataType.NUMERIC,
@@ -596,6 +778,7 @@ function createObjects(
               traceId: trace.id,
               name: "Completeness",
               timestamp: traceTs,
+              createdAt: traceTs,
               source: ScoreSource.API,
               projectId,
               dataType: ScoreDataType.CATEGORICAL,
@@ -606,6 +789,15 @@ function createObjects(
         : []),
     ];
 
+    if (Math.random() > 0.9)
+      comments.push({
+        projectId: trace.projectId,
+        objectId: trace.id,
+        objectType: "TRACE",
+        content: "Trace comment content",
+        ...(Math.random() > 0.5 ? { authorUserId: `user-${i}` } : {}),
+      });
+
     scores.push(...traceScores);
 
     const existingSpanIds: string[] = [];
@@ -624,6 +816,7 @@ function createObjects(
         type: ObservationType.SPAN,
         id: `span-${v4()}`,
         startTime: spanTsStart,
+        createdAt: spanTsStart,
         endTime: spanTsEnd,
         name: `span-${i}-${j}`,
         metadata: {
@@ -691,45 +884,21 @@ function createObjects(
             )
           ];
 
+        const { input, output } = getGenerationInputOutput();
+
         const generation = {
           type: ObservationType.GENERATION,
           id: `generation-${v4()}`,
           startTime: generationTsStart,
+          createdAt: generationTsStart,
           endTime: generationTsEnd,
           completionStartTime:
             Math.random() > 0.5 ? generationTsCompletionStart : undefined,
           name: `generation-${i}-${j}-${k}`,
           projectId: trace.projectId,
           promptId: promptId,
-          input:
-            Math.random() > 0.5
-              ? [
-                  {
-                    role: "system",
-                    content: "Be a helpful assistant",
-                  },
-                  {
-                    role: "user",
-                    content: "How can i create a *React* component?",
-                  },
-                ]
-              : {
-                  input: "How can i create a React component?",
-                  retrievedDocuments: [
-                    {
-                      title: "How to create a React component",
-                      url: "https://www.google.com",
-                      description: "A guide to creating React components",
-                    },
-                    {
-                      title: "React component creation",
-                      url: "https://www.google.com",
-                      description: "A guide to creating React components",
-                    },
-                  ],
-                },
-          output:
-            "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.\n\n**Image**\n\n![Languse Example Image](https://static.langfuse.com/langfuse-dev/langfuse-example-image.jpeg)\n\n1.  **Functional Component**:\n\nA functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:\n\n```javascript\nimport React from 'react';\nfunction Greeting(props) {\n  return <h1>Hello, {props.name}</h1>;\n}\nexport default Greeting;\n```\n\nTo use this component in another file, you can do:\n\n```javascript\nimport Greeting from './Greeting';\nfunction App() {\n  return (\n    <div>\n      <Greeting name=\"John\" />\n    </div>\n  );\n}\nexport default App;\n```\n\n2.  **Class Component**:\n\nYou can also define components as classes in React. These have some additional features compared to functional components:\n\n```javascript\nimport React, { Component } from 'react';\nclass Greeting extends Component {\n  render() {\n    return <h1>Hello, {this.props.name}</h1>;\n  }\n}\nexport default Greeting;\n```\n\nAnd here's how to use this component:\n\n```javascript\nimport Greeting from './Greeting';\nclass App extends Component {\n  render() {\n    return (\n      <div>\n        <Greeting name=\"John\" />\n      </div>\n    );\n  }\n}\nexport default App;\n```\n\nWith the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.\n\nRemember to import React at the top of your file whenever you're creating a component, because JSX transpiles to `React.createElement` calls under the hood.",
+          input,
+          output,
           model: model,
           internalModel: model,
           modelParameters: {
@@ -765,6 +934,8 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
+            createdAt: traceTs,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -774,6 +945,16 @@ function createObjects(
             traceId: trace.id,
             source: ScoreSource.API,
             projectId: trace.projectId,
+            timestamp: generationTsEnd,
+            createdAt: traceTs,
+          });
+
+        if (Math.random() > 0.8)
+          comments.push({
+            projectId: trace.projectId,
+            objectId: generation.id,
+            objectType: "OBSERVATION",
+            content: "Observation comment content",
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -789,6 +970,7 @@ function createObjects(
             type: ObservationType.EVENT,
             id: `event-${v4()}`,
             startTime: eventTs,
+            createdAt: eventTs,
             name: `event-${i}-${j}-${k}-${l}`,
             metadata: {
               user: `user-${i}@langfuse.com`,
@@ -811,8 +993,10 @@ function createObjects(
     observations,
     scores,
     configs,
+    queueItems,
     sessions: uniqueSessions,
     events,
+    comments,
   };
 }
 
@@ -1087,3 +1271,125 @@ async function generateConfigs(project: Project) {
 
   return configNameAndId;
 }
+
+function getGenerationInputOutput(): {
+  input: Prisma.InputJsonValue;
+  output: Prisma.InputJsonValue;
+} {
+  if (Math.random() > 0.9) {
+    const input = [
+      {
+        role: "user",
+        content: [
+          { text: "What’s depicted in this image?", type: "text" },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
+            },
+          },
+          { text: "Describe the scene in detail.", type: "text" },
+        ],
+      },
+    ];
+
+    const output =
+      "The image depicts a serene landscape featuring a wooden pathway or boardwalk that winds through a lush green field. The field is filled with tall grass and surrounded by trees and shrubs. Above, the sky is bright with scattered clouds, suggesting a clear and pleasant day. The scene conveys a sense of tranquility and natural beauty.";
+
+    return { input, output };
+  }
+
+  const input =
+    Math.random() > 0.5
+      ? [
+          {
+            role: "system",
+            content: "Be a helpful assistant",
+          },
+          {
+            role: "user",
+            content: "How can i create a *React* component?",
+          },
+        ]
+      : {
+          input: "How can i create a React component?",
+          retrievedDocuments: [
+            {
+              title: "How to create a React component",
+              url: "https://www.google.com",
+              description: "A guide to creating React components",
+            },
+            {
+              title: "React component creation",
+              url: "https://www.google.com",
+              description: "A guide to creating React components",
+            },
+          ],
+        };
+
+  const output =
+    "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.\n\n**Image**\n\n![Languse Example Image](https://static.langfuse.com/langfuse-dev/langfuse-example-image.jpeg)\n\n1.  **Functional Component**:\n\nA functional component is just a plain JavaScript function that accepts props as an argument, and returns a React element. Here's how you can create one:\n\n```javascript\nimport React from 'react';\nfunction Greeting(props) {\n  return <h1>Hello, {props.name}</h1>;\n}\nexport default Greeting;\n```\n\nTo use this component in another file, you can do:\n\n```javascript\nimport Greeting from './Greeting';\nfunction App() {\n  return (\n    <div>\n      <Greeting name=\"John\" />\n    </div>\n  );\n}\nexport default App;\n```\n\n2.  **Class Component**:\n\nYou can also define components as classes in React. These have some additional features compared to functional components:\n\n```javascript\nimport React, { Component } from 'react';\nclass Greeting extends Component {\n  render() {\n    return <h1>Hello, {this.props.name}</h1>;\n  }\n}\nexport default Greeting;\n```\n\nAnd here's how to use this component:\n\n```javascript\nimport Greeting from './Greeting';\nclass App extends Component {\n  render() {\n    return (\n      <div>\n        <Greeting name=\"John\" />\n      </div>\n    );\n  }\n}\nexport default App;\n```\n\nWith the advent of hooks in React, functional components can do everything that class components can do and hence, the community has been favoring functional components over class components.\n\nRemember to import React at the top of your file whenever you're creating a component, because JSX transpiles to `React.createElement` calls under the hood.";
+
+  return { input, output };
+}
+
+async function generateQueuesForProject(
+  projects: Project[],
+  configIdsAndNames: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >
+) {
+  const projectIdsToQueues: Map<string, string[]> = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const queueIds = await generateQueues(
+        project,
+        configIdsAndNames.get(project.id) ?? []
+      );
+      projectIdsToQueues.set(project.id, queueIds);
+    })
+  );
+  return projectIdsToQueues;
+}
+
+async function generateQueues(
+  project: Project,
+  configIdsAndNames: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[]
+) {
+  const queue = {
+    id: `queue-${v4()}`,
+    name: "Default",
+    description: "Default queue",
+    scoreConfigIds: configIdsAndNames.map((config) => config.id),
+    projectId: project.id,
+  };
+
+  await prisma.annotationQueue.upsert({
+    where: {
+      projectId_name: {
+        projectId: queue.projectId,
+        name: queue.name,
+      },
+    },
+    create: {
+      ...queue,
+    },
+    update: {
+      id: queue.id,
+    },
+  });
+
+  return [queue.id];
+}

packages/shared/src/db.ts

@@ -1,7 +1,7 @@
 // This file exports the prisma db connection, the Prisma Object, and the Typescript types.
 // This is not imported in the index.ts file of this package, as we must not import this into FE code.
 
-import { PrismaClient } from "@prisma/client";
+import { Prisma, PrismaClient } from "@prisma/client";
 import { env } from "process";
 import kyselyExtension from "prisma-extension-kysely";
 import {
@@ -11,21 +11,83 @@ import {
   PostgresQueryCompiler,
 } from "kysely";
 import { DB } from ".";
+import { logger } from "./server";
 
-// Instantiated according to the Prisma documentation
-// https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
+export class PrismaClientSingleton {
+  private static instance: PrismaClient;
 
-const prismaClientSingleton = () => {
-  return new PrismaClient({
-    log:
-      env.NODE_ENV === "development"
-        ? ["query", "error", "warn"]
-        : ["error", "warn"],
-  });
-};
+  public static getInstance(): PrismaClient {
+    if (PrismaClientSingleton.instance) {
+      return PrismaClientSingleton.instance;
+    }
 
-const kyselySingleton = (prismaClient: PrismaClient) => {
-  return prismaClient.$extends(
+    const client = new PrismaClient<
+      Prisma.PrismaClientOptions,
+      "warn" | "error" | "query"
+    >({
+      log: [
+        { emit: "event", level: "query" },
+        { emit: "event", level: "error" },
+        { emit: "event", level: "warn" },
+      ],
+    });
+
+    if (env.NODE_ENV === "development") {
+      client.$on("query", (event) => {
+        logger.info(`prisma:query ${event.query}, ${event.duration}ms`);
+      });
+    }
+
+    client.$on("warn", (event) => {
+      logger.warn(`prisma:warn ${event.message}`);
+    });
+
+    client.$on("error", (event) => {
+      logger.error(`prisma:error ${event.message}`);
+    });
+
+    PrismaClientSingleton.instance = client;
+
+    return PrismaClientSingleton.instance;
+  }
+}
+
+export class KyselySingleton {
+  private static instance: { $kysely: Kysely<DB> };
+
+  public static getInstance() {
+    if (KyselySingleton.instance) {
+      return KyselySingleton.instance;
+    }
+
+    KyselySingleton.instance = PrismaClientSingleton.getInstance().$extends(
+      kyselyExtension({
+        kysely: (driver) =>
+          new Kysely<DB>({
+            dialect: {
+              // This is where the magic happens!
+              createDriver: () => driver,
+              // Don't forget to customize these to match your database!
+              createAdapter: () => new PostgresAdapter(),
+              createIntrospector: (db) => new PostgresIntrospector(db),
+              createQueryCompiler: () => new PostgresQueryCompiler(),
+            },
+          }),
+      })
+    );
+
+    return KyselySingleton.instance;
+  }
+}
+
+declare const globalThis: {
+  prismaGlobal: PrismaClient | undefined;
+  kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
+} & typeof global;
+
+if (process.env.NODE_ENV === "development") {
+  globalThis.prismaGlobal ??= new PrismaClient(); // regular instantiation
+  globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
     kyselyExtension({
       kysely: (driver) =>
         new Kysely<DB>({
@@ -41,15 +103,10 @@ const kyselySingleton = (prismaClient: PrismaClient) => {
     })
   );
 };
-declare global {
-  // eslint-disable-next-line no-var
-  var prisma: undefined | ReturnType<typeof prismaClientSingleton>;
-  var kyselyPrisma: undefined | ReturnType<typeof kyselySingleton>;
-}
 
-export const prisma = globalThis.prisma ?? prismaClientSingleton();
-export const kyselyPrisma = globalThis.kyselyPrisma ?? kyselySingleton(prisma);
+export const prisma =
+  globalThis.prismaGlobal ?? PrismaClientSingleton.getInstance();
+export const kyselyPrisma =
+  globalThis.kyselyPrismaGlobal ?? KyselySingleton.getInstance();
 
 export * from "@prisma/client";
-
-if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;

packages/shared/src/encryption/index.ts

@@ -1,7 +1,7 @@
 import crypto from "crypto";
 import { env } from "../env";
 
-const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const ENCRYPTION_KEY: string | undefined = env.ENCRYPTION_KEY; // Must be 256 bits (32 bytes, 64 hex characters)
 const IV_LENGTH: number = 16; // For AES, this is always 16
 
 // Alternatively: openssl rand -hex 32
@@ -13,17 +13,16 @@ export function keyGen() {
  * Encrypts the given plain text using AES-256-GCM algorithm.
  *
  * @param {string} plainText - The text to encrypt.
- * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
  * @returns {string} The encrypted data in hex format, including IV and authentication tag.
  */
-export function encrypt(
-  plainText: string,
-  keyHex: string = ENCRYPTION_KEY
-): string {
+export function encrypt(plainText: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
   const cipher = crypto.createCipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   let encrypted = cipher.update(plainText, "utf8", "hex");
@@ -34,7 +33,10 @@ export function encrypt(
   return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
 }
 
-export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+export function decrypt(text: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
   const [ivHex, encryptedHex, authTagHex] = text.split(":");
   if (!ivHex || !encryptedHex || !authTagHex) {
     throw new Error("Invalid or corrupted cipher format");
@@ -46,7 +48,7 @@ export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
 
   const decipher = crypto.createDecipheriv(
     "aes-256-gcm",
-    Buffer.from(keyHex, "hex"),
+    Buffer.from(ENCRYPTION_KEY, "hex"),
     iv
   );
   decipher.setAuthTag(authTag);

packages/shared/src/env.ts

@@ -1,6 +1,23 @@
 import { z } from "zod";
+import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
+  NODE_ENV: z
+    .enum(["development", "test", "production"])
+    .default("development"),
+  REDIS_HOST: z.string().nullish(),
+  REDIS_PORT: z.coerce
+    .number({
+      description:
+        ".env files convert numbers to strings, therefoore we have to enforce them to be numbers",
+    })
+    .positive()
+    .max(65536, `options.port should be >= 0 and < 65536`)
+    .default(6379)
+    .nullable(),
+  REDIS_AUTH: z.string().nullish(),
+  REDIS_CONNECTION_STRING: z.string().nullish(),
+  REDIS_ENABLE_AUTO_PIPELINING: z.enum(["true", "false"]).default("true"),
   ENCRYPTION_KEY: z
     .string()
     .length(
@@ -8,6 +25,25 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
     )
     .optional(),
+  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
+  CLICKHOUSE_URL: z.string().url().optional(),
+  CLICKHOUSE_USER: z.string().optional(),
+  CLICKHOUSE_PASSWORD: z.string().optional(),
+  LANGFUSE_INGESTION_FLUSH_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(10000),
+  LANGFUSE_INGESTION_FLUSH_ATTEMPTS: z.coerce.number().positive().default(3),
+  LANGFUSE_INGESTION_BUFFER_TTL_SECONDS: z.coerce
+    .number()
+    .positive()
+    .default(60 * 10),
+  SALT: z.string().optional(), // used by components imported by web package
+  LANGFUSE_LOG_LEVEL: z
+    .enum(["trace", "debug", "info", "warn", "error", "fatal"])
+    .optional(),
+  LANGFUSE_LOG_FORMAT: z.enum(["text", "json"]).default("text"),
 });
 
-export const env = EnvSchema.parse(process.env);
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/annotation/types.ts

@@ -14,7 +14,7 @@ const NumericData = z.object({
 });
 
 const CategoricalData = z.object({
-  value: z.number().optional().nullish(),
+  value: z.number().nullish(),
   stringValue: z.string(),
   dataType: z.literal("CATEGORICAL"),
 });
@@ -31,7 +31,8 @@ const CreateAnnotationScoreBase = z.object({
   traceId: z.string(),
   configId: z.string().optional(),
   observationId: z.string().optional(),
-  comment: z.string().optional().nullish(),
+  comment: z.string().nullish(),
+  queueId: z.string().nullish(),
 });
 
 const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
@@ -57,3 +58,15 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
   UpdateAnnotationScoreBase.merge(CategoricalData),
   UpdateAnnotationScoreBase.merge(BooleanData),
 ]);
+
+// annotation queues
+
+export const CreateQueueData = z.object({
+  name: z.string().min(1).max(35),
+  description: z.string().max(1000).optional(),
+  scoreConfigIds: z.array(z.string()).min(1, {
+    message: "At least 1 score config must be selected",
+  }),
+});
+
+export type CreateQueue = z.infer<typeof CreateQueueData>;

packages/shared/src/features/batchExport/types.ts

@@ -19,6 +19,8 @@ export enum BatchExportFileFormat {
 
 export enum BatchExportTableName {
   Sessions = "sessions",
+  Traces = "traces",
+  Generations = "generations",
 }
 
 export const exportOptions: Record<

packages/shared/src/features/comments/types.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+const COMMENT_OBJECT_TYPES = [
+  "TRACE",
+  "OBSERVATION",
+  "SESSION",
+  "PROMPT",
+] as const;
+
+export const CreateCommentData = z.object({
+  projectId: z.string(),
+  content: z.string().trim().min(1).max(500),
+  objectId: z.string(),
+  objectType: z.enum(COMMENT_OBJECT_TYPES),
+});
+
+export const DeleteCommentData = z.object({
+  projectId: z.string(),
+  commentId: z.string(),
+  objectId: z.string(),
+  objectType: z.enum(COMMENT_OBJECT_TYPES),
+});

packages/shared/src/features/entitlements/plans.ts

@@ -0,0 +1,17 @@
+export const planLabels = {
+  oss: "OSS",
+  "cloud:hobby": "Hobby",
+  "cloud:pro": "Pro",
+  "cloud:team": "Team",
+  "self-hosted:enterprise": "Enterprise",
+} as const;
+
+export type Plan = keyof typeof planLabels;
+
+export const plans = Object.keys(planLabels) as Plan[];
+
+// This function is kept here to ensure consistency when updating plan names in the future.
+export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+
+export const isPlan = (value: string): value is Plan =>
+  plans.includes(value as Plan);

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,4 @@
 import z from "zod";
-import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,17 +76,6 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
-  [LLMAdapter.OpenAI]: [
-    "gpt-4o",
-    "gpt-4o-mini",
-    "gpt-4-turbo-preview",
-    "gpt-3.5-turbo",
-  ] as const,
-  [LLMAdapter.Anthropic]: [] as const,
-  [LLMAdapter.Azure]: [] as const,
-};
-
 export const OutputSchema = z.object({
   reasoning: z.string(),
   score: z.string(),

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -1,12 +1,13 @@
-import { isPresent } from "@/src/utils/typeChecks";
+import { z } from "zod";
+
+import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
+
+import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
   paginationZod,
-  type ScoreConfig as ScoreConfigDbType,
-} from "@langfuse/shared";
-import { z } from "zod";
-import * as Sentry from "@sentry/node";
+} from "../../utils/zod";
 
 /**
  * Types to use across codebase
@@ -16,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx,
+  ctx: z.RefinementCtx
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -53,8 +54,8 @@ export const Category = z.object({
 const Categories = z.array(Category);
 
 const NumericScoreConfig = z.object({
-  maxValue: z.number().optional().nullish(),
-  minValue: z.number().optional().nullish(),
+  maxValue: z.number().nullish(),
+  minValue: z.number().nullish(),
   dataType: z.literal("NUMERIC"),
   categories: z.undefined().nullish(),
 });
@@ -93,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value,
+          category.value === expectedCategories[index].value
       );
     }),
 });
@@ -102,7 +103,7 @@ const ScoreConfigBase = z.object({
   id: z.string(),
   name: z.string().min(1).max(35),
   isArchived: z.boolean(),
-  description: z.string().optional().nullish(),
+  description: z.string().nullish(),
   createdAt: z.coerce.date(),
   updatedAt: z.coerce.date(),
   projectId: z.string(),
@@ -122,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      }),
+      })
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -143,19 +144,20 @@ const ValidatedScoreConfigSchema = z
 
 /**
  * Use this function when pulling a list of score configs from the database before using in the application to ensure type safety.
- * All score configs are expected to pass the validation. If a score fails validation, it will be logged to Sentry.
+ * All score configs are expected to pass the validation. If a score fails validation, it will be logged to Otel.
  * @param scoreConfigs
  * @returns list of validated score configs
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
+  onParseError?: (error: z.ZodError) => void
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
     if (result.success) {
       acc.push(result.data);
     } else {
-      Sentry.captureException(result.error);
+      onParseError?.(result.error);
     }
     return acc;
   }, [] as ValidatedScoreConfig[]);
@@ -168,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType,
+  scoreConfig: ScoreConfigDbType
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -203,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      }),
+      })
     ),
   ])
   .superRefine((data, ctx) => {

packages/shared/src/features/scores/scoreTypes.ts

@@ -1,19 +1,19 @@
-import * as Sentry from "@sentry/node";
-import {
-  paginationZod,
-  paginationMetaResponseZod,
-  NonEmptyString,
-  type Score,
-  stringDateTime,
-} from "@langfuse/shared";
 import { z } from "zod";
-import { isPresent } from "@/src/utils/typeChecks";
-import { Category as ConfigCategory } from "./score-configs";
+
+import { Score } from "@prisma/client";
+
+import { isPresent, stringDateTime } from "../../utils/typeChecks";
+import {
+  NonEmptyString,
+  paginationMetaResponseZod,
+  paginationZod,
+} from "../../utils/zod";
+import { Category as ConfigCategory } from "./scoreConfigTypes";
 
 /**
  * Types to use across codebase
  */
-export type APIScore = z.infer<typeof APIScore>;
+export type APIScore = z.infer<typeof APIScoreSchema>;
 
 /**
  * Helpers
@@ -31,7 +31,7 @@ const NumericData = z.object({
 });
 
 const CategoricalData = z.object({
-  value: z.number().optional().nullish(),
+  value: z.number().nullish(),
   stringValue: z.string(),
   dataType: z.literal("CATEGORICAL"),
 });
@@ -69,7 +69,7 @@ const BaseScoreBody = z.object({
  * Objects
  */
 
-export const APIScore = z.discriminatedUnion("dataType", [
+export const APIScoreSchema = z.discriminatedUnion("dataType", [
   ScoreBase.merge(NumericData),
   ScoreBase.merge(CategoricalData),
   ScoreBase.merge(BooleanData),
@@ -155,18 +155,21 @@ export const ScorePropsAgainstConfig = z.union([
 
 /**
  * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
- * All scores are expected to pass the validation. If a score fails validation, it will be logged to Sentry.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
  * @param scores
  * @returns list of validated scores
  */
-export const filterAndValidateDbScoreList = (scores: Score[]): APIScore[] =>
+export const filterAndValidateDbScoreList = (
+  scores: Score[],
+  onParseError?: (error: z.ZodError) => void,
+): APIScore[] =>
   scores.reduce((acc, ts) => {
-    const result = APIScore.safeParse(ts);
+    const result = APIScoreSchema.safeParse(ts);
     if (result.success) {
       acc.push(result.data);
     } else {
       console.error("Score parsing error: ", result.error);
-      Sentry.captureException(result.error);
+      onParseError?.(result.error);
     }
     return acc;
   }, [] as APIScore[]);
@@ -179,7 +182,7 @@ export const filterAndValidateDbScoreList = (scores: Score[]): APIScore[] =>
  * @throws error if score fails validation
  */
 export const validateDbScore = (score: Score): APIScore =>
-  APIScore.parse(score);
+  APIScoreSchema.parse(score);
 
 /**
  * Endpoints
@@ -189,63 +192,39 @@ export const validateDbScore = (score: Score): APIScore =>
 /**
  * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
  */
-export const PostScoresBody = z
-  .discriminatedUnion("dataType", [
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number(),
-        dataType: z.literal("NUMERIC"),
-        configId: z.string().nullish(),
+export const PostScoresBody = z.discriminatedUnion("dataType", [
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number(),
+      dataType: z.literal("NUMERIC"),
+      configId: z.string().nullish(),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.string(),
+      dataType: z.literal("CATEGORICAL"),
+      configId: z.string().nullish(),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.number().refine((value) => value === 0 || value === 1, {
+        message:
+          "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
       }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.string(),
-        dataType: z.literal("CATEGORICAL"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number().refine((val) => val === 0 || val === 1, {
-          message: "Value must be either 0 or 1",
-        }),
-        dataType: z.literal("BOOLEAN"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.union([z.string(), z.number()]),
-        dataType: z.undefined(),
-        configId: z.string().nullish(),
-      }),
-    ),
-  ])
-  .superRefine((data, ctx) => {
-    if (data.dataType) {
-      if (typeof data.value === "number") {
-        if (data.dataType === "CATEGORICAL") {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `Value must be a string for data type ${data.dataType}`,
-          });
-        }
-      } else if (typeof data.value === "string") {
-        if (data.dataType === "NUMERIC") {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `Value must be a number for data type ${data.dataType}`,
-          });
-        } else if (data.dataType === "BOOLEAN") {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `Value must number equal to either 0 or 1 for data type ${data.dataType}`,
-          });
-        }
-      }
-    }
-  });
+      dataType: z.literal("BOOLEAN"),
+      configId: z.string().nullish(),
+    }),
+  ),
+  BaseScoreBody.merge(
+    z.object({
+      value: z.union([z.string(), z.number()]),
+      dataType: z.undefined(),
+      configId: z.string().nullish(),
+    }),
+  ),
+]);
 
 export const PostScoresResponse = z.object({ id: z.string() });
 
@@ -272,7 +251,7 @@ export const GetScoresQuery = z.object({
 
 // LegacyGetScoreResponseDataV1 is only used for response of GET /scores list endpoint
 const LegacyGetScoreResponseDataV1 = z.intersection(
-  APIScore,
+  APIScoreSchema,
   z.object({
     trace: z.object({
       userId: z.string().nullish(),
@@ -286,6 +265,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
+  onParseError?: (error: z.ZodError) => void,
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
     (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
@@ -294,7 +274,7 @@ export const legacyFilterAndValidateV1GetScoreList = (
         acc.push(result.data);
       } else {
         console.error("Score parsing error: ", result.error);
-        Sentry.captureException(result.error);
+        onParseError?.(result.error);
       }
       return acc;
     },
@@ -306,7 +286,7 @@ export const GetScoreQuery = z.object({
   scoreId: z.string(),
 });
 
-export const GetScoreResponse = APIScore;
+export const GetScoreResponse = APIScoreSchema;
 
 // DELETE /scores/{scoreId}
 export const DeleteScoreQuery = z.object({

packages/shared/src/index.ts

@@ -1,23 +1,24 @@
 export * from "./constants";
-export * from "./queries";
-export * from "./queues";
 export * from "./interfaces/filters";
 export * from "./interfaces/orderBy";
+export * from "./interfaces/cloudConfigSchema";
+export * from "./interfaces/parseDbOrg";
+export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
-export * from "./filterToPrisma";
-export * from "./orderByToPrisma";
 export * from "./tracesTable";
-export * from "./server/auth";
+export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
-export * from "./features/ingestion/types";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/objects";
+export * from "./utils/typeChecks";
+export * from "./features/entitlements/plans";
+export * from "./interfaces/rate-limits";
 export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";
-export * from "./server/llm/fetchLLMCompletion";
 
 // evals
 export * from "./features/evals/types";
@@ -26,9 +27,18 @@ export * from "./features/batchExport/types";
 // annotation
 export * from "./features/annotation/types";
 
+// scores
+export * from "./features/scores/scoreConfigTypes";
+export * from "./features/scores/scoreTypes";
+
+// comments
+export * from "./features/comments/types";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
 
 // errors
 export * from "./errors/index";
+
+export * from "./utils/environment";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+import { CloudConfigRateLimit } from "./rate-limits";
+
+export const CloudConfigSchema = z.object({
+  plan: z.enum(["Hobby", "Pro", "Team", "Enterprise"]).optional(),
+  monthlyObservationLimit: z.number().int().positive().optional(),
+  // used for table and dashboard queries
+  defaultLookBackDays: z.number().int().positive().optional(),
+  // need to update stripe webhook if you change this, it fetches from db via these fields
+  stripe: z
+    .object({
+      customerId: z.string().optional(),
+      activeSubscriptionId: z.string().optional(),
+      activeProductId: z.string().optional(),
+    })
+    .optional(),
+
+  // custom rate limits for an organization
+  rateLimitOverrides: CloudConfigRateLimit.optional(),
+});
+
+export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+export const BedrockConfigSchema = z.object({ region: z.string() });
+export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
+
+export const BedrockCredentialSchema = z
+  .object({
+    accessKeyId: z.string(),
+    secretAccessKey: z.string(),
+  })
+  .optional();
+export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;

packages/shared/src/interfaces/parseDbOrg.ts

@@ -0,0 +1,19 @@
+import { type Organization } from "@prisma/client";
+import { CloudConfigSchema } from "./cloudConfigSchema";
+
+type parsedOrg = Omit<Organization, "cloudConfig"> & {
+  cloudConfig: CloudConfigSchema | null;
+};
+
+export function parseDbOrg(dbOrg: Organization): parsedOrg {
+  const { cloudConfig, ...org } = dbOrg;
+
+  const parsedCloudConfig = CloudConfigSchema.safeParse(cloudConfig);
+
+  const parsedOrg = {
+    ...org,
+    cloudConfig: parsedCloudConfig.success ? parsedCloudConfig.data : null,
+  };
+
+  return parsedOrg;
+}

packages/shared/src/interfaces/rate-limits.ts

@@ -0,0 +1,31 @@
+import z from "zod";
+import { ApiAccessScope } from "../server";
+
+export const RateLimitResource = z.enum([
+  "ingestion",
+  "public-api",
+  "public-api-metrics",
+  "prompts",
+  "legacy-ingestion",
+]);
+
+// result of the rate limit check.
+export type RateLimitResult = {
+  resource: z.infer<typeof RateLimitResource>;
+  points: number;
+  scope: ApiAccessScope;
+
+  // from rate-limiter-flexible
+  remainingPoints: number;
+  msBeforeNext: number;
+  consumedPoints: number;
+  isFirstInDuration: boolean;
+};
+
+export const RateLimitConfig = z.object({
+  points: z.number().nullish(),
+  durationInSec: z.number().nullish(),
+  resource: RateLimitResource,
+});
+
+export const CloudConfigRateLimit = z.array(RateLimitConfig);