chore: release v3.2.0

chore: return previous annotation score in delete method

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx
+ignore-words-list = afterall,vertx,notIn
 

.env.dev-azure.example

@@ -0,0 +1,84 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_BATCH_EXPORT_REGION=auto
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.dev.example

@@ -11,6 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -33,12 +34,37 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=
-S3_ACCESS_KEY_ID=
-S3_SECRET_ACCESS_KEY=
-S3_BUCKET_NAME=
-S3_REGION=
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 # Set during docker build of application
 # Used to disable environment verification at build time
@@ -48,7 +74,6 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 # openssl rand -hex 32 used only here
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 

.env.dev.legacy.example

@@ -0,0 +1,86 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# # S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=true
+LANGFUSE_RETURN_FROM_CLICKHOUSE=false
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false

.env.local.example

@@ -1,24 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
-# Next Auth
-NEXTAUTH_SECRET="secret"
-NEXTAUTH_URL="http://localhost:3000"
-
-# feature flag to enable experimental features locally
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
-
-# Redis
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-LANGFUSE_WORKER_PASSWORD=myworkerpassword
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -36,6 +36,10 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # PORT=3000
 # HOSTNAME=localhost
 
+# Opentelemetry, optional
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+OTEL_SERVICE_NAME="langfuse"
+
 # Default role for users who sign up, optional, can be org or org+project
 # LANGFUSE_DEFAULT_ORG_ID=
 # LANGFUSE_DEFAULT_ORG_ROLE=
@@ -63,6 +67,14 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_CLIENT_ID=
+# AUTH_GITLAB_CLIENT_SECRET=
+# AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_ISSUER=
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
@@ -79,6 +91,10 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
@@ -92,16 +108,16 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # Defines the connection url for smtp server. 
 # SMTP_CONNECTION_URL=
 
-# S3 storage, optional, used for exports from the UI
-# S3_ENDPOINT=
-# S3_ACCESS_KEY_ID=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION=
-# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
+# S3 Batch Exports
+# LANGFUSE_S3_BATCH_EXPORT_ENABLED=
+# LANGFUSE_S3_BATCH_EXPORT_BUCKET=
+# LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=
+# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=
+# LANGFUSE_S3_BATCH_EXPORT_REGION=
+# LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=
+# LANGFUSE_S3_BATCH_EXPORT_PREFIX=
 
 # S3 storage for events, optional, used to persist all incoming events
-# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
 # LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
 # Optional prefix to be used within the bucket. Must end with `/` if set
 # LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
@@ -229,15 +245,11 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # CLICKHOUSE_PASSWORD=
 
 # Ingestion
-# LANGFUSE_INGESTION_BUFFER_TTL_SECONDS=
-# LANGFUSE_INGESTION_FLUSH_DELAY_MS= 
-# LANGFUSE_INGESTION_FLUSH_ATTEMPTS= 
-# LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY=
-# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE= 
+# LANGFUSE_INGESTION_QUEUE_DELAY_MS=
+# LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
 # LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
 ## END Langfuse V3 Ingestion

.github/workflows/_deploy_ecs_service.yml

@@ -58,7 +58,6 @@ jobs:
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-
       - name: Render AWS ECS Task Definition
         id: render-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1

.github/workflows/ci.yml.template

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start containers
         run: docker compose -f "docker-compose.yml" up -d --build
@@ -31,7 +31,7 @@ jobs:
         uses: pnpm/action-setup@v2.2.4
 
       - name: Setup Node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main", "production", "v*"]
+  pull_request:
+    branches: ["main", "production", "v*"]
+  schedule:
+    - cron: "31 13 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: javascript-typescript
+            build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # If the analyze step fails for one of the languages you are analyzing with
+      # "We were unable to automatically build your code", modify the matrix above
+      # to set the build mode to "manual" for that language. Then modify this step
+      # to build your code.
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      - if: matrix.build-mode == 'manual'
+        shell: bash
+        run: |
+          echo 'If you are using a "manual" build mode for one or more of the' \
+            'languages you are analyzing, replace this with the commands to build' \
+            'your code, for example:'
+          echo '  make bootstrap'
+          echo '  make release'
+          exit 1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/codespell.yml

@@ -6,6 +6,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/deploy.yml

@@ -19,7 +19,7 @@ on:
         type: choice
         options:
           - staging
-          - prod-eu-temp
+          - prod-eu
           - prod-us
         required: true
 
@@ -78,7 +78,7 @@ jobs:
                 return `["staging"]`
               }
               if (context.ref === "refs/heads/production") {
-                return `["prod-eu-temp", "prod-us"]`
+                return `["prod-eu", "prod-us"]`
               }
             }
             return "[]"

.github/workflows/licencecheck.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/pipeline.yml

@@ -10,17 +10,21 @@ on:
   merge_group:
   pull_request:
     branches:
-      - "*"
+      - "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
@@ -35,27 +39,23 @@ jobs:
         run: pnpm run lint
 
   test-docker-build:
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-
+        uses: actions/checkout@v4
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Set NEXT_PUBLIC_BUILD_ID
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-
       - name: Build and run both images from compose
         run: |
           docker compose -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
-
-      - name: Check server health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
-
-      - name: Check worker health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
-
       - name: Ensure no unhealthy status
         run: |
           if docker-compose ps | grep "(unhealthy)"; then
@@ -64,11 +64,17 @@ jobs:
           else
             echo "All services are healthy"
           fi
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
 
-  tests-web:
+  tests-web-sync:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
@@ -78,27 +84,35 @@ jobs:
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 10
-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
-
       - name: install dependencies
         run: |
           pnpm install
-
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
-
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -106,14 +120,12 @@ jobs:
           docker compose ps
         env:
           POSTGRES_VERSION: ${{ matrix.postgres-version }}
-
       - name: Seed DB
         run: |
           pnpm run db:migrate
-
+          pnpm --filter=shared ch:up
       - name: Build
         run: pnpm run build
-
       - name: Start Langfuse
         run: (pnpm run start&)
         env:
@@ -126,58 +138,129 @@ jobs:
           LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
           LANGFUSE_INIT_USER_NAME: "Demo User"
           LANGFUSE_INIT_USER_PASSWORD: "password"
+      - name: run test-sync
+        run: pnpm --filter=web run test-sync
+      - name: run test-client
+        run: pnpm --filter=web run test-client
 
+  tests-web-async:
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
+    strategy:
+      matrix:
+        node-version: [20]
+        postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+      - uses: actions/checkout@v4
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm install
+      - name: Load default env
+        run: |
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm --filter=shared ch:up
+      - name: Build
+        run: pnpm run build
+      - name: Start Langfuse
+        run: (pnpm run start&)
+        env:
+          LANGFUSE_INIT_ORG_ID: "seed-org-id"
+          LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+          LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
+          LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
+          LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890"
+          LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
+          LANGFUSE_INIT_USER_NAME: "Demo User"
+          LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run tests
         run: pnpm --filter=web run test
 
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 10
-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
-
       - name: install dependencies
         run: |
           pnpm install
-
       - name: Install golang-migrate for Clickhouse migrations
         run: |
           curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
-
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
-
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
-
       - name: Ensure no unhealthy status
         run: |
           if docker compose ps | grep "(unhealthy)"; then
@@ -186,101 +269,116 @@ jobs:
           else
             echo "All services are healthy"
           fi
-
       - name: Seed DB
         run: |
           pnpm run db:migrate
           pnpm run db:seed
           pnpm run --filter=shared ch:up
-
       - name: Build
         run: pnpm --filter=worker... run build
-
       - name: run tests
         run: pnpm --filter=worker run test
+
   e2e-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
-
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: install dependencies
         run: |
           pnpm install
-
       - name: Load default env
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
-
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
-
       - name: Seed DB
         run: |
           pnpm run db:migrate
           pnpm run db:seed
-
       - name: Build
         run: pnpm run build
-
       - name: Install playwright
         run: pnpm --filter=web exec playwright install --with-deps
-
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
 
   e2e-server-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
-
       - name: install dependencies
         run: |
           pnpm install
-
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
-
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
-
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed:examples
-
       - name: Build
         run: pnpm run build
-
       - name: Run server
         run: (pnpm run start&)
-
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e:server
 
@@ -290,11 +388,12 @@ jobs:
     needs:
       [
         lint,
-        tests-web,
+        tests-web-sync,
         tests-worker,
         e2e-tests,
         test-docker-build,
         e2e-server-tests,
+        tests-web-async,
       ]
     if: always()
     steps:
@@ -321,38 +420,31 @@ jobs:
         with:
           version: 9.5.0
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache-dependency-path: "pnpm-lock.yaml"
-
       - name: Checkout
-        uses: actions/checkout@v3
-
+        uses: actions/checkout@v4
       - name: Set NEXT_PUBLIC_BUILD_ID
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-
       - name: Log in to the GitHub Container registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
           driver-opts: network=host
-
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
@@ -360,6 +452,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -367,7 +461,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
@@ -379,7 +473,6 @@ jobs:
           platforms: |
             linux/amd64
             ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
-
       - name: Extract metadata (tags, labels) for Docker
         id: meta-worker
         uses: docker/metadata-action@v4
@@ -387,6 +480,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse-worker # GitHub
             langfuse/langfuse-worker # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -394,7 +489,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.github/workflows/snyk.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   snyk:
@@ -15,13 +20,13 @@ jobs:
       - name: Build a Docker image
         run: docker compose -f docker-compose.build.yml up -d
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
         continue-on-error: true
         uses: snyk/actions/docker@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-server
+          image: langfuse-langfuse-web
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
@@ -36,7 +41,7 @@ jobs:
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-worker
+          image: langfuse-langfuse-worker
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning

.gitignore

@@ -35,9 +35,11 @@ yarn-error.log*
 
 # local env files
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
-.env
-.env.local
-.env*.local
+.env*
+!.env.dev.example
+!.env.dev-azure.example
+!.env.prod.example
+!.env.dev.legacy.example
 
 # vercel
 .vercel

.vscode/settings.json

@@ -20,6 +20,7 @@
   "prettier.documentSelectors": [
     "**/*.{cjs,mjs,ts,tsx,astro,md,mdx,json,yaml,yml}"
   ],
+  "prettier.trailingComma": "all",
   "mdx.experimentalLanguageServer": true,
   "typescript.preferences.importModuleSpecifier": "non-relative",
   "docwriter.style": "JSDoc",

CONTRIBUTING.md

@@ -432,6 +432,20 @@ Example:
 op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
 ```
 
+### Editing default models and prices
+
+You can update the default AI models and prices by adding or updating an entry in `worker/src/constants/default-model-prices.json`.
+
+Please note that
+
+- prices are in USD
+- the list is ordered by ID, so make sure to keep this order
+- the `updated_at` field must be updated with the current date in ISO 8601 format. Otherwise, the change will be ignored.
+
+### Transition period until V3 release
+
+Until the V3 release, both the JSON record must be updated **and** a migration must be created to continue supporting self-hosted users. Note that the migration must updated both the `models` as well as the `prices` table accordingly.
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -2,8 +2,8 @@ Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
-- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- All content that resides under the "ee/", "web/src/ee/", and/or "worker/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Langfuse Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -181,3 +181,50 @@ This helps us to:
 None of the data is shared with third parties and does not include any sensitive information. We want to be super transparent about this and you can find the exact data we collect [here](/web/src/features/telemetry/index.ts).
 
 You can opt-out by setting `TELEMETRY_ENABLED=false`.
+
+### Star History
+
+<a href="https://star-history.com/#langfuse/langfuse&Date">
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date&theme=dark" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
+   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
+ </picture>
+</a>
+
+### Open Source Projects Using Langfuse
+
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents/python.md)):
+
+| Repository                                                                                                                                                                                                                                                          | Stars |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 53464 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 49387 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37087 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 36690 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26871 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22348 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 14508 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7319 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5352 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2084 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  1936 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1140 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1044 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   945 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   770 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   675 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   583 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   505 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   383 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   379 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   357 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   300 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   278 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   237 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   164 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   133 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   122 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/134601687?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langfuse](https://github.com/langfuse) / [langfuse-python](https://github.com/langfuse/langfuse-python)                            |   108 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/42655961?s=40&v=4" width="20" height="20" alt=""> &nbsp; [zozoheir](https://github.com/zozoheir) / [tinyllm](https://github.com/zozoheir/tinyllm)                                             |    95 |

docker-compose.build.yml

@@ -1,27 +1,53 @@
-version: "3.5"
-
 services:
-  server:
+  langfuse-web:
     build:
       dockerfile: ./web/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: &langfuse-depends-on
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
     ports:
       - "3000:3000"
-    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
-      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
+    environment: &langfuse-web-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      NEXTAUTH_SECRET: mysecret
+      SALT: mysalt
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: http://localhost:3000
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -29,27 +55,17 @@ services:
       timeout: 10s
       retries: 3
 
-  worker:
+  langfuse-worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: *langfuse-depends-on
     ports:
       - "3030:3030"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - PORT=${PORT:-3030}
-      - REDIS_HOST=${REDIS_HOST:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
-      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
+      <<: *langfuse-web-env
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
@@ -57,26 +73,86 @@ services:
       timeout: 10s
       retries: 3
 
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
-    image: redis:7.2.4
+    image: redis:7
+    restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
-    restart: always
     ports:
       - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
     driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
+    driver: local
+

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.dev.yml

@@ -17,6 +17,27 @@ services:
     depends_on:
       - postgres
 
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
     image: redis:7.2.4
     restart: always
@@ -28,6 +49,11 @@ services:
   postgres:
     image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     command: ["postgres", "-c", "log_statement=all"]
     environment:
       - POSTGRES_USER=postgres
@@ -45,3 +71,5 @@ volumes:
     driver: local
   langfuse_clickhouse_logs:
     driver: local
+  langfuse_minio_data:
+    driver: local

docker-compose.yml

@@ -1,22 +1,131 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:3
+    restart: always
+    depends_on: &langfuse-depends-on
+      postgres:
         condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:3
+    restart: always
+    depends_on: *langfuse-depends-on
     ports:
       - "3000:3000"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
 
-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    restart: always
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -24,14 +133,20 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

ee/package.json

@@ -25,9 +25,10 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "next": "^14.2.6",
-    "next-auth": "^4.24.7",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "axios": "^1.7.7",
+    "next": "^14.2.15",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -39,9 +40,14 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"
+  },
+  "pnpm": {
+    "overrides": {
+      "jsonpath-plus": "10.0.7"
+    }
   }
 }

ee/prettierrc.json

@@ -1,4 +0,0 @@
-{
-  "trailingComma": "es5",
-  "printWidth": 120
-}

ee/src/env.ts

@@ -1,9 +1,9 @@
 import { z } from "zod";
-import { env as sharedEnv } from "@langfuse/shared";
+import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

fern/apis/server/definition/comments.yml

@@ -0,0 +1,73 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    create:
+      docs: Create a comment. Comments may be attached to different object types (trace, observation, session, prompt).
+      method: POST
+      path: /comments
+      request: CreateCommentRequest
+      response: CreateCommentResponse
+    get:
+      docs: Get all comments
+      method: GET
+      path: /comments
+      request:
+        name: GetCommentsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit
+          objectType:
+            type: optional<string>
+            docs: Filter comments by object type (trace, observation, session, prompt).
+          objectId:
+            type: optional<string>
+            docs: Filter comments by object id. If objectType is not provided, an error will be thrown.
+          authorUserId:
+            type: optional<string>
+            docs: Filter comments by author user id.
+      response: GetCommentsResponse
+    get-by-id:
+      docs: Get a comment by id
+      method: GET
+      path: /comments/{commentId}
+      path-parameters:
+        commentId:
+          type: string
+          docs: The unique langfuse identifier of a comment
+      response: commons.Comment
+types:
+  CreateCommentRequest:
+    properties:
+      projectId:
+        type: string
+        docs: The id of the project to attach the comment to.
+      objectType:
+        type: string
+        docs: The type of the object to attach the comment to (trace, observation, session, prompt).
+      objectId:
+        type: string
+        docs: The id of the object to attach the comment to. If this does not reference a valid existing object, an error will be thrown.
+      content:
+        type: string
+        docs: The content of the comment. May include markdown. Currently limited to 3000 characters.
+      authorUserId:
+        type: optional<string>
+        docs: The id of the user who created the comment.
+  CreateCommentResponse:
+    properties:
+      id:
+        type: string
+        docs: The id of the created object in Langfuse
+  GetCommentsResponse:
+    properties:
+      data: list<commons.Comment>
+      meta: pagination.MetaResponse

fern/apis/server/definition/commons.yml

@@ -126,7 +126,7 @@ types:
         docs: The output data of the observation
       usage:
         type: optional<Usage>
-        docs: The usage data of the observation
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The usage data of the observation
       level:
         type: ObservationLevel
         docs: The level of the observation
@@ -139,6 +139,12 @@ types:
       promptId:
         type: optional<string>
         docs: The prompt ID associated with the observation
+      usageDetails:
+        type: optional<map<string, integer>>
+        docs: The usage details of the observation. Key is the name of the usage metric, value is the number of units consumed. The total key is the sum of all (non-total) usage metrics or the total value ingested.
+      costDetails:
+        type: optional<map<string, double>>
+        docs: The cost details of the observation. Key is the name of the cost metric, value is the cost in USD. The total key is the sum of all (non-total) cost metrics or the total value ingested.
 
   ObservationsView:
     extends: Observation
@@ -163,13 +169,13 @@ types:
         docs: The total price in USD.
       calculatedInputCost:
         type: optional<double>
-        docs: The calculated cost of the input in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the input in USD
       calculatedOutputCost:
         type: optional<double>
-        docs: The calculated cost of the output in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the output in USD
       calculatedTotalCost:
         type: optional<double>
-        docs: The calculated total cost in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated total cost in USD
       latency:
         type: optional<double>
         docs: The latency in seconds.
@@ -178,7 +184,7 @@ types:
         docs: The time to the first token in seconds
 
   Usage:
-    docs: Standard interface for usage and cost
+    docs: (Deprecated. Use usageDetails and costDetails instead.) Standard interface for usage and cost
     properties:
       input:
         docs: Number of input units (e.g. tokens)
@@ -240,6 +246,9 @@ types:
       configId:
         type: optional<string>
         docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+      queueId:
+        type: optional<string>
+        docs: Reference an annotation queue on a score. Populated if the score was initially created in an annotation queue.
   NumericScore:
     extends: BaseScore
     properties:
@@ -283,6 +292,18 @@ types:
       - double
       - string
     docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores
+
+  Comment:
+    properties:
+      id: string
+      projectId: string
+      createdAt: datetime
+      updatedAt: datetime
+      objectType: CommentObjectType
+      objectId: string
+      content: string
+      authorUserId: optional<string>
+
   Dataset:
     properties:
       id: string
@@ -402,6 +423,12 @@ types:
       - optional<integer>
       - optional<boolean>
       - optional<list<string>>
+  CommentObjectType:
+    enum:
+      - TRACE
+      - OBSERVATION
+      - SESSION
+      - PROMPT
   DatasetStatus:
     enum:
       - ACTIVE

fern/apis/server/definition/ingestion.yml

@@ -126,6 +126,8 @@ types:
       model: optional<string>
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptName: optional<string>
       promptVersion: optional<integer>
 
@@ -137,6 +139,8 @@ types:
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
       promptName: optional<string>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptVersion: optional<integer>
 
   ObservationBody:
@@ -312,3 +316,17 @@ types:
     properties:
       successes: list<IngestionSuccess>
       errors: list<IngestionError>
+
+  OpenAIUsageSchema:
+    properties:
+      prompt_tokens: integer
+      completion_tokens: integer
+      total_tokens: integer
+      prompt_tokens_details: optional<map<string, integer>>
+      completion_tokens_details: optional<map<string, integer>>
+
+  UsageDetails:
+    discriminated: false
+    union:
+      - map<string, integer>
+      - OpenAIUsageSchema

fern/apis/server/definition/media.yml

@@ -0,0 +1,161 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get a media record
+      method: GET
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      response: GetMediaResponse
+
+    patch:
+      docs: Patch a media record
+      method: PATCH
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      request: PatchMediaBody
+
+    getUploadUrl:
+      docs: Get a presigned upload URL for a media record
+      method: POST
+      path: /media
+      request: GetMediaUploadUrlRequest
+      response: GetMediaUploadUrlResponse
+
+types:
+  GetMediaResponse:
+    properties:
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+      contentType:
+        type: string
+        docs: The MIME type of the media record
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      url:
+        type: string
+        docs: The download URL of the media record
+      urlExpiry:
+        type: string
+        docs: The expiry date and time of the media record download URL
+
+  PatchMediaBody:
+    properties:
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      uploadHttpStatus:
+        type: integer
+        docs: The HTTP status code of the upload
+      uploadHttpError:
+        type: optional<string>
+        docs: The HTTP error message of the upload
+      uploadTimeMs:
+        type: optional<integer>
+        docs: The time in milliseconds it took to upload the media record
+
+  GetMediaUploadUrlRequest:
+    properties:
+      traceId:
+        type: string
+        docs: The trace ID associated with the media record
+      observationId:
+        type: optional<string>
+        docs: The observation ID associated with the media record. If the media record is associated directly with a trace, this will be null.
+      contentType: MediaContentType
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      sha256Hash:
+        type: string
+        docs: The SHA-256 hash of the media record
+      field:
+        type: string
+        docs: The trace / observation field the media record is associated with. This can be one of `input`, `output`, `metadata`
+
+  GetMediaUploadUrlResponse:
+    properties:
+      uploadUrl:
+        type: optional<string>
+        docs: The presigned upload URL. If the asset is already uploaded, this will be null
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+
+  MediaContentType:
+    enum:
+      - value: image/png
+        name: IMAGE_PNG
+      - value: image/jpeg
+        name: IMAGE_JPEG
+      - value: image/jpg
+        name: IMAGE_JPG
+      - value: image/webp
+        name: IMAGE_WEBP
+      - value: image/gif
+        name: IMAGE_GIF
+      - value: image/svg+xml
+        name: IMAGE_SVG_XML
+      - value: image/tiff
+        name: IMAGE_TIFF
+      - value: image/bmp
+        name: IMAGE_BMP
+      - value: audio/mpeg
+        name: AUDIO_MPEG
+      - value: audio/mp3
+        name: AUDIO_MP3
+      - value: audio/wav
+        name: AUDIO_WAV
+      - value: audio/ogg
+        name: AUDIO_OGG
+      - value: audio/oga
+        name: AUDIO_OGA
+      - value: audio/aac
+        name: AUDIO_AAC
+      - value: audio/mp4
+        name: AUDIO_MP4
+      - value: audio/flac
+        name: AUDIO_FLAC
+      - value: video/mp4
+        name: VIDEO_MP4
+      - value: video/webm
+        name: VIDEO_WEBM
+      - value: text/plain
+        name: TEXT_PLAIN
+      - value: text/html
+        name: TEXT_HTML
+      - value: text/css
+        name: TEXT_CSS
+      - value: text/csv
+        name: TEXT_CSV
+      - value: application/pdf
+        name: APPLICATION_PDF
+      - value: application/msword
+        name: APPLICATION_MSWORD
+      - value: application/vnd.ms-excel
+        name: APPLICATION_MS_EXCEL
+      - value: application/zip
+        name: APPLICATION_ZIP
+      - value: application/json
+        name: APPLICATION_JSON
+      - value: application/xml
+        name: APPLICATION_XML
+      - value: application/octet-stream
+        name: APPLICATION_OCTET_STREAM
+    docs: The MIME type of the media record

fern/apis/server/definition/models.yml

@@ -55,7 +55,7 @@ types:
         type: string
       startDate:
         docs: Apply only to generations which are newer than this ISO date.
-        type: optional<date>
+        type: optional<datetime>
       unit:
         docs: Unit used by this model.
         type: commons.ModelUsageUnit

fern/apis/server/definition/score.yml

@@ -52,10 +52,17 @@ service:
           configId:
             type: optional<string>
             docs: Retrieve only scores with a specific configId.
+          queueId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific annotation queueId.
           dataType:
             type: optional<commons.ScoreDataType>
             docs: Retrieve only scores with a specific dataType.
-      response: Scores
+          traceTags:
+            type: optional<list<string>>
+            allow-multiple: true
+            docs: Only scores linked to traces that include all of these tags will be returned.
+      response: GetScoresResponse
     get-by-id:
       docs: Get a score
       method: GET
@@ -132,7 +139,38 @@ types:
       id:
         type: string
         docs: The id of the created object in Langfuse
-  Scores:
+  GetScoresResponseTraceData:
     properties:
-      data: list<commons.Score>
+      userId:
+        type: optional<string>
+        docs: The user ID associated with the trace referenced by score
+      tags:
+        type: optional<list<string>>
+        docs: A list of tags associated with the trace referenced by score
+
+  GetScoresResponseDataNumeric:
+    extends: commons.NumericScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataCategorical:
+    extends: commons.CategoricalScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataBoolean:
+    extends: commons.BooleanScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseData:
+    discriminant: dataType
+    union:
+      NUMERIC: GetScoresResponseDataNumeric
+      CATEGORICAL: GetScoresResponseDataCategorical
+      BOOLEAN: GetScoresResponseDataBoolean
+
+  GetScoresResponse:
+    properties:
+      data: list<GetScoresResponseData>
       meta: pagination.MetaResponse

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.81.2",
+  "version": "3.2.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -9,15 +9,16 @@
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
-    "infra:dev:up": "docker compose -f ./docker-compose.dev.yml up -d",
+    "infra:dev:up": "docker compose -f ./docker-compose.dev.yml up -d --wait",
     "infra:dev:down": "docker compose -f ./docker-compose.dev.yml down",
+    "infra:dev:prune": "docker compose -f ./docker-compose.dev.yml down -v",
     "db:generate": "turbo run db:generate",
     "db:migrate": "turbo run db:migrate",
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:up && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset -f && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
@@ -33,7 +34,7 @@
     "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
     "husky": "^9.0.11",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
     "release-it": "^17.3.0",
     "turbo": "^1.13.4"
   },
@@ -80,5 +81,10 @@
       }
     }
   },
-  "packageManager": "pnpm@9.5.0"
+  "packageManager": "pnpm@9.5.0",
+  "pnpm": {
+    "overrides": {
+      "jsonpath-plus": "10.0.7"
+    }
+  }
 }

packages/config-eslint/package.json

@@ -11,7 +11,7 @@
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.6",
+    "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-only-warn": "^1.1.0",

packages/shared/clickhouse/migrations/clustered/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0001_traces.up.sql

@@ -0,0 +1,32 @@
+CREATE TABLE traces ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(LowCardinality(String), String),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    updated_at DateTime64(3) DEFAULT now(),
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+     project_id,
+     toDate(timestamp)
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/clustered/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0002_observations.up.sql

@@ -0,0 +1,47 @@
+CREATE TABLE observations ON CLUSTER default (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_usage_details` Map(LowCardinality(String), UInt64),
+    `usage_details` Map(LowCardinality(String), UInt64),
+    `provided_cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
+PRIMARY KEY (
+    project_id,
+    `type`,
+    toDate(start_time)
+)
+ORDER BY (
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
+

packages/shared/clickhouse/migrations/clustered/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0003_scores.up.sql

@@ -0,0 +1,33 @@
+CREATE TABLE scores ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `queue_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+    project_id,
+    toDate(timestamp),
+    name
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_id project_id TYPE bloom_filter() GRANULARITY 1;

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.up.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_project_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0001_traces.up.sql

@@ -3,24 +3,30 @@ CREATE TABLE traces (
     `timestamp` DateTime64(3),
     `name` String,
     `user_id` Nullable(String),
-    `metadata` Map(String, String) CODEC(ZSTD(1)),
+    `metadata` Map(LowCardinality(String), String),
     `release` Nullable(String),
     `version` Nullable(String),
     `project_id` String,
     `public` Bool,
     `bookmarked` Bool,
     `tags` Array(String),
-    `input` Nullable(String) CODEC(ZSTD(1)),
-    `output` Nullable(String) CODEC(ZSTD(1)),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
     `session_id` Nullable(String),
     `created_at` DateTime64(3) DEFAULT now(),
-    `updated_at` DateTime64(3) DEFAULT now(),
+    updated_at DateTime64(3) DEFAULT now(),
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
     INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
     INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
-) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+     project_id,
+     toDate(timestamp)
+)
 ORDER BY (
-        project_id,
-        toUnixTimestamp(timestamp),
-        id
-    );
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/unclustered/0002_observations.up.sql

@@ -7,7 +7,7 @@ CREATE TABLE observations (
     `start_time` DateTime64(3),
     `end_time` Nullable(DateTime64(3)),
     `name` String,
-    `metadata` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `metadata` Map(LowCardinality(String), String),
     `level` LowCardinality(String),
     `status_message` Nullable(String),
     `version` Nullable(String),
@@ -16,18 +16,10 @@ CREATE TABLE observations (
     `provided_model_name` Nullable(String),
     `internal_model_id` Nullable(String),
     `model_parameters` Nullable(String),
-    `provided_input_usage_units` Nullable(Decimal64(12)),
-    `provided_output_usage_units` Nullable(Decimal64(12)),
-    `provided_total_usage_units` Nullable(Decimal64(12)),
-    `input_usage_units` Nullable(Decimal64(12)),
-    `output_usage_units` Nullable(Decimal64(12)),
-    `total_usage_units` Nullable(Decimal64(12)),
-    `unit` Nullable(String),
-    `provided_input_cost` Nullable(Decimal64(12)),
-    `provided_output_cost` Nullable(Decimal64(12)),
-    `provided_total_cost` Nullable(Decimal64(12)),
-    `input_cost` Nullable(Decimal64(12)),
-    `output_cost` Nullable(Decimal64(12)),
+    `provided_usage_details` Map(LowCardinality(String), UInt64),
+    `usage_details` Map(LowCardinality(String), UInt64),
+    `provided_cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `cost_details` Map(LowCardinality(String), Decimal64(12)),
     `total_cost` Nullable(Decimal64(12)),
     `completion_start_time` Nullable(DateTime64(3)),
     `prompt_id` Nullable(String),
@@ -35,16 +27,21 @@ CREATE TABLE observations (
     `prompt_version` Nullable(UInt16),
     `created_at` DateTime64(3) DEFAULT now(),
     `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
     INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
     INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
-    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1,
-    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter() GRANULARITY 1,
-    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter() GRANULARITY 1
-) ENGINE = ReplacingMergeTree Partition by toYYYYMM(start_time)
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
+PRIMARY KEY (
+    project_id,
+    `type`,
+    toDate(start_time)
+)
 ORDER BY (
-        project_id,
-        `type`,
-        trace_id,
-        toUnixTimestamp(start_time),
-        id
-    );
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
+

packages/shared/clickhouse/migrations/unclustered/0003_scores.up.sql

@@ -12,14 +12,22 @@ CREATE TABLE scores (
     `config_id` Nullable(String),
     `data_type` String,
     `string_value` Nullable(String),
+    `queue_id` Nullable(String),
     `created_at` DateTime64(3) DEFAULT now(),
     `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    `is_deleted` UInt8,
     INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_project_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1
-) ENGINE = ReplacingMergeTree Partition by toYYYYMM(timestamp)
+    INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+    project_id,
+    toDate(timestamp),
+    name
+)
 ORDER BY (
-        project_id,
-        trace_id,
-        toUnixTimestamp(timestamp),
-        id
-    );
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/migrations/unclustered/0004_drop_observations_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ADD INDEX IF NOT EXISTS idx_project_id project_id TYPE bloom_filter() GRANULARITY 1;

packages/shared/clickhouse/migrations/unclustered/0004_drop_observations_index.up.sql

@@ -0,0 +1 @@
+ALTER TABLE observations DROP INDEX IF EXISTS idx_project_id;

packages/shared/clickhouse/migrations/unclustered/0005_add_session_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces DROP INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/unclustered/0005_add_session_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/down.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,10 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
-# Execute the down command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
+fi

packages/shared/clickhouse/scripts/seed.ts

@@ -0,0 +1,75 @@
+import {
+  clickhouseClient,
+  ObservationRecordReadType,
+} from "@langfuse/shared/src/server";
+import { prisma } from "../../src/db";
+import { redis } from "@langfuse/shared/src/server";
+import { prepareClickhouse } from "../../scripts/prepareClickhouse";
+import { createDatasets } from "../../prisma/seed";
+import { queryClickhouse } from "../../src/server/repositories/clickhouse";
+import { convertObservation } from "../../src/server/repositories/observations_converters";
+
+async function main() {
+  try {
+    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
+    if (
+      await prisma.project.findFirst({
+        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      })
+    ) {
+      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
+    }
+    await prepareClickhouse(projectIds, {
+      numberOfDays: 3,
+      totalObservations: 10000,
+    });
+
+    const project1 = await prisma.project.findFirst({
+      where: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const project2 =
+      projectIds.length > 1
+        ? await prisma.project.findFirst({
+            where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+          })
+        : await prisma.project.findFirst();
+
+    const query = `
+        SELECT *
+        FROM observations o
+        WHERE o.project_id IN ({projectIds: Array(String)})
+        LIMIT 2000;
+      `;
+
+    const res = await queryClickhouse<ObservationRecordReadType>({
+      query,
+      params: {
+        projectIds,
+      },
+    });
+
+    await createDatasets(
+      project1!,
+      project2!,
+      (await Promise.all(res.map(convertObservation))).map((o) => ({
+        ...o,
+        metadata: {},
+        modelParameters: {},
+        input: {},
+        output: {},
+      })),
+    );
+
+    console.log("Clickhouse preparation completed successfully.");
+  } catch (error) {
+    console.error("Error during Clickhouse preparation:", error);
+  } finally {
+    await clickhouseClient().close();
+    await prisma.$disconnect();
+    redis?.disconnect();
+    console.log("Disconnected from Clickhouse.");
+  }
+}
+
+main();

packages/shared/clickhouse/scripts/up.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,11 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
-else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
 
-# Execute the up command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+else
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
+fi

packages/shared/package.json

@@ -43,73 +43,85 @@
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
     "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
-    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
     "ch:up": "bash clickhouse/scripts/up.sh",
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
-    "ch:reset": "pnpm run ch:down && pnpm run ch:up"
+    "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed",
+    "ch:seed": "dotenv -e ../../.env -- tsx clickhouse/scripts/seed.ts",
+    "load:setup": "dotenv -e ../../.env -- tsx scripts/load-seed.ts"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.627.0",
-    "@aws-sdk/lib-storage": "^3.645.0",
-    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@aws-sdk/client-cloudwatch": "^3.675.0",
+    "@aws-sdk/client-s3": "^3.675.0",
+    "@aws-sdk/lib-storage": "^3.675.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.3",
-    "@langchain/openai": "^0.3.0",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "@prisma/client": "^5.17.0",
+    "@langchain/anthropic": "^0.3.8",
+    "@langchain/aws": "^0.1.2",
+    "@langchain/core": "^0.3.18",
+    "@langchain/google-vertexai": "^0.1.3",
+    "@langchain/openai": "^0.3.14",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "@prisma/client": "^5.22.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
+    "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.12.10",
-    "dd-trace": "^5.19.0",
+    "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.2",
+    "langchain": "^0.3.6",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
-    "nodemailer": "^6.9.13",
+    "next-auth": "^4.24.11",
+    "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
-    "winston": "^3.14.2",
-    "winston-transport": "^4.7.1",
+    "winston": "^3.15.0",
     "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.2"
+    "zod-to-json-schema": "^3.23.5"
   },
   "devDependencies": {
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
     "@types/node": "^20.11.29",
-    "@types/nodemailer": "^6.4.15",
-    "@types/pg": "^8.11.6",
+    "@types/nodemailer": "^6.4.16",
+    "@types/pg": "^8.11.10",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.1.3",
-    "prettier": "^3.2.5",
-    "prisma": "^5.17.0",
+    "kysely-codegen": "^0.16.8",
+    "nodemon": "^3.1.7",
+    "prettier": "^3.3.3",
+    "prisma": "^5.22.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
+    "tsx": "^4.19.1",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.3"
+    "vitest": "^2.1.2"
   },
   "peerDependencies": {
     "@types/react": "~18.2.79",
     "react": "~18.2.0"
+  },
+  "pnpm": {
+    "overrides": {
+      "jsonpath-plus": "10.0.7"
+    }
   }
 }

packages/shared/prettierrc.json

@@ -1,4 +0,0 @@
-{
-  "trailingComma": "es5",
-  "printWidth": 120
-}

packages/shared/prisma/generated/types.ts

@@ -37,6 +37,16 @@ export const ScoreDataType = {
     BOOLEAN: "BOOLEAN"
 } as const;
 export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
+export const AnnotationQueueStatus = {
+    PENDING: "PENDING",
+    COMPLETED: "COMPLETED"
+} as const;
+export type AnnotationQueueStatus = (typeof AnnotationQueueStatus)[keyof typeof AnnotationQueueStatus];
+export const AnnotationQueueObjectType = {
+    TRACE: "TRACE",
+    OBSERVATION: "OBSERVATION"
+} as const;
+export type AnnotationQueueObjectType = (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -81,6 +91,30 @@ export type Account = {
     id_token: string | null;
     session_state: string | null;
     refresh_token_expires_in: number | null;
+    created_at: number | null;
+};
+export type AnnotationQueue = {
+    id: string;
+    name: string;
+    description: string | null;
+    score_config_ids: Generated<string[]>;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type AnnotationQueueItem = {
+    id: string;
+    queue_id: string;
+    object_id: string;
+    object_type: AnnotationQueueObjectType;
+    status: Generated<AnnotationQueueStatus>;
+    locked_at: Timestamp | null;
+    locked_by_user_id: string | null;
+    annotator_user_id: string | null;
+    completed_at: Timestamp | null;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type ApiKey = {
     id: string;
@@ -109,6 +143,18 @@ export type AuditLog = {
     before: string | null;
     after: string | null;
 };
+export type BackgroundMigration = {
+    id: string;
+    name: string;
+    script: string;
+    args: unknown;
+    state: Generated<unknown>;
+    finished_at: Timestamp | null;
+    failed_at: Timestamp | null;
+    failed_reason: string | null;
+    worker_id: string | null;
+    locked_at: Timestamp | null;
+};
 export type BatchExport = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -232,6 +278,8 @@ export type JobExecution = {
     end_time: Timestamp | null;
     error: string | null;
     job_input_trace_id: string | null;
+    job_input_observation_id: string | null;
+    job_input_dataset_item_id: string | null;
     job_output_score_id: string | null;
 };
 export type LlmApiKeys = {
@@ -245,8 +293,23 @@ export type LlmApiKeys = {
     base_url: string | null;
     custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
+    config: unknown | null;
     project_id: string;
 };
+export type Media = {
+    id: string;
+    sha_256_hash: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    uploaded_at: Timestamp | null;
+    upload_http_status: number | null;
+    upload_http_error: string | null;
+    bucket_path: string;
+    bucket_name: string;
+    content_type: string;
+    content_length: string;
+};
 export type MembershipInvitation = {
     id: string;
     email: string;
@@ -269,7 +332,7 @@ export type Model = {
     input_price: string | null;
     output_price: string | null;
     total_price: string | null;
-    unit: string;
+    unit: string | null;
     tokenizer_id: string | null;
     tokenizer_config: unknown | null;
 };
@@ -307,6 +370,16 @@ export type Observation = {
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
+export type ObservationMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    observation_id: string;
+    field: string;
+};
 export type ObservationView = {
     id: string;
     trace_id: string | null;
@@ -367,11 +440,20 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
+export type Price = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    model_id: string;
+    usage_type: string;
+    price: string;
+};
 export type Project = {
     id: string;
     org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    deleted_at: Timestamp | null;
     name: string;
 };
 export type ProjectMembership = {
@@ -410,6 +492,7 @@ export type Score = {
     observation_id: string | null;
     config_id: string | null;
     string_value: string | null;
+    queue_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     data_type: Generated<ScoreDataType>;
@@ -459,6 +542,15 @@ export type Trace = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type TraceMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    field: string;
+};
 export type TraceSession = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -506,8 +598,11 @@ export type VerificationToken = {
 };
 export type DB = {
     Account: Account;
+    annotation_queue_items: AnnotationQueueItem;
+    annotation_queues: AnnotationQueue;
     api_keys: ApiKey;
     audit_logs: AuditLog;
+    background_migrations: BackgroundMigration;
     batch_exports: BatchExport;
     comments: Comment;
     cron_jobs: CronJobs;
@@ -520,13 +615,16 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
+    observation_media: ObservationMedia;
     observations: Observation;
     observations_view: ObservationView;
     organization_memberships: OrganizationMembership;
     organizations: Organization;
     posthog_integrations: PosthogIntegration;
+    prices: Price;
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
@@ -534,6 +632,7 @@ export type DB = {
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;
+    trace_media: TraceMedia;
     trace_sessions: TraceSession;
     traces: Trace;
     traces_view: TraceView;

packages/shared/prisma/migrations/20241009042557_auth_add_created_at_for_gitlab/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "created_at" INTEGER;

packages/shared/prisma/migrations/20241009110720_scores_add_nullable_queue_id_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "queue_id" TEXT;

packages/shared/prisma/migrations/20241009113245_add_annotation_queue/migration.sql

@@ -0,0 +1,75 @@
+-- CreateEnum
+CREATE TYPE "AnnotationQueueStatus" AS ENUM ('PENDING', 'COMPLETED');
+
+-- CreateEnum
+CREATE TYPE "AnnotationQueueObjectType" AS ENUM ('TRACE', 'OBSERVATION');
+
+-- CreateTable
+CREATE TABLE "annotation_queues" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT,
+    "score_config_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queues_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "annotation_queue_items" (
+    "id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "object_type" "AnnotationQueueObjectType" NOT NULL,
+    "status" "AnnotationQueueStatus" NOT NULL DEFAULT 'PENDING',
+    "locked_at" TIMESTAMP(3),
+    "locked_by_user_id" TEXT,
+    "annotator_user_id" TEXT,
+    "completed_at" TIMESTAMP(3),
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_items_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_id_project_id_idx" ON "annotation_queues"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queues_project_id_created_at_idx" ON "annotation_queues"("project_id", "created_at");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queues_project_id_name_key" ON "annotation_queues"("project_id", "name");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_id_project_id_idx" ON "annotation_queue_items"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_project_id_queue_id_status_idx" ON "annotation_queue_items"("project_id", "queue_id", "status");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_object_id_object_type_project_id_que_idx" ON "annotation_queue_items"("object_id", "object_type", "project_id", "queue_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_annotator_user_id_idx" ON "annotation_queue_items"("annotator_user_id");
+
+-- CreateIndex
+CREATE INDEX "annotation_queue_items_created_at_idx" ON "annotation_queue_items"("created_at");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queues" ADD CONSTRAINT "annotation_queues_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_locked_by_user_id_fkey" FOREIGN KEY ("locked_by_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_annotator_user_id_fkey" FOREIGN KEY ("annotator_user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_items" ADD CONSTRAINT "annotation_queue_items_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241010120245_llm_keys_add_config/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "llm_api_keys" ADD COLUMN "config" JSONB;

packages/shared/prisma/migrations/20241015110145_prompts_config_to_JSON/migration.sql

@@ -0,0 +1,107 @@
+-- Temporarily drop observations_view as prompts.config is a calculated column in the view and we need to update it to a JSON type
+DROP VIEW IF EXISTS "observations_view"; -- Drop view as column was added in 20240705154048_observation_view_add_created_at_updated_at and update view must have same columns
+
+-- Update prompts.config to JSON type. Use JSON as JSONB is reordering keys for optimization, which is undesired
+ALTER TABLE prompts ALTER COLUMN config TYPE JSON USING config::JSON;
+ALTER TABLE prompts ALTER COLUMN config SET DEFAULT '{}'::JSON;
+
+-- Recreate the view with the updated prompts.config column
+CREATE VIEW "observations_view" AS -- Specify the columns that should be returned in the view, as calculated columns are added but exist in the observations table already
+SELECT
+    o.id,
+    o.name,
+    o.start_time,
+    o.end_time,
+    o.parent_observation_id,
+    o.type,
+    o.trace_id,
+    o.metadata,
+    o.model,
+    o."modelParameters",
+    o.input,
+    o.output,
+    o.level,
+    o.status_message,
+    o.completion_start_time,
+    o.completion_tokens,
+    o.prompt_tokens,
+    o.total_tokens,
+    o.version,
+    o.project_id,
+    o.created_at,
+    o.updated_at,
+    o.unit,
+    o.prompt_id,
+    p.name as prompt_name,         -- added in this change
+    p.version as prompt_version,   -- added in this change
+    o.input_cost,
+    o.output_cost,
+    o.total_cost,
+    o.internal_model,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.calculated_input_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            COALESCE(o.calculated_input_cost, o.input_cost)
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.calculated_output_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            COALESCE(o.calculated_output_cost, o.output_cost)
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.calculated_total_cost IS NULL AND o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            COALESCE(o.calculated_total_cost, o.total_cost)
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+LEFT JOIN LATERAL (
+    SELECT
+        prompts.*
+    FROM
+        prompts
+    WHERE prompts.id = o.prompt_id
+    AND prompts.project_id = o.project_id
+    LIMIT 1
+) p ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20241022110145_add_claude_sonnet_35/migration.sql

@@ -0,0 +1,18 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://docs.anthropic.com/en/docs/about-claude/models#model-comparison-table
+  ('cm2krz1uf000208jjg5653iud', NULL, 'claude-3.5-haiku-20241022', '(?i)^(claude-3-5-sonnet-20241022|anthropic\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude', NULL),
+  ('cm2ks2vzn000308jjh4ze1w7q', NULL, 'claude-3.5-haiku-latest', '(?i)^(claude-3-5-sonnet-latest)$', NULL, 0.000003, 0.000015, NULL, 'TOKENS', 'claude', NULL)
+

packages/shared/prisma/migrations/20241023110145_update_claude_sonnet_35/migration.sql

@@ -0,0 +1,3 @@
+-- https://docs.anthropic.com/en/docs/about-claude/models#model-comparison-table
+UPDATE models SET model_name = 'claude-3.5-sonnet-20241022' WHERE id = 'cm2krz1uf000208jjg5653iud';
+UPDATE models SET model_name = 'claude-3.5-sonnet-latest' WHERE id = 'cm2ks2vzn000308jjh4ze1w7q';

packages/shared/prisma/migrations/20241024100928_add_prices_table/migration.sql

@@ -0,0 +1,62 @@
+-- CreateTable
+CREATE TABLE "prices" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "model_id" TEXT NOT NULL,
+    "usage_type" TEXT NOT NULL,
+    "price" DECIMAL(65,30) NOT NULL,
+
+    CONSTRAINT "prices_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "prices_model_id_idx" ON "prices"("model_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "prices_model_id_usage_type_key" ON "prices"("model_id", "usage_type");
+
+-- AddForeignKey
+ALTER TABLE "prices" ADD CONSTRAINT "prices_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "models"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Alter Table to make unit column nullable
+ALTER TABLE "models" ALTER COLUMN "unit" DROP NOT NULL;
+
+
+-- Create a temporary table to store the new prices for user-defined models
+CREATE TEMPORARY TABLE temp_prices AS
+SELECT 
+    id AS model_id,
+    'input' AS usage_type,
+    input_price AS price
+FROM models
+WHERE project_id IS NOT NULL AND input_price IS NOT NULL
+UNION ALL
+SELECT 
+    id AS model_id,
+    'output' AS usage_type,
+    output_price AS price
+FROM models
+WHERE project_id IS NOT NULL AND output_price IS NOT NULL
+UNION ALL
+SELECT 
+    id AS model_id,
+    'total' AS usage_type,
+    total_price AS price
+FROM models
+WHERE project_id IS NOT NULL AND total_price IS NOT NULL;
+
+-- Insert into prices table
+INSERT INTO prices (id, created_at, updated_at, model_id, usage_type, price)
+SELECT 
+    md5(random()::text || clock_timestamp()::text || model_id::text || usage_type::text)::uuid AS id,
+    NOW() AS created_at,
+    NOW() AS updated_at,
+    model_id,
+    usage_type,
+    price
+FROM temp_prices
+ON CONFLICT (model_id, usage_type) DO NOTHING;
+
+-- Drop the temporary table
+DROP TABLE temp_prices;

packages/shared/prisma/migrations/20241024111800_add_background_migrations_table/migration.sql

@@ -0,0 +1,18 @@
+-- CreateTable
+CREATE TABLE "background_migrations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "script" TEXT NOT NULL,
+    "args" JSONB NOT NULL,
+
+    "finished_at" TIMESTAMP(3),
+    "failed_at" TIMESTAMP(3),
+    "failed_reason" TEXT,
+    "worker_id" TEXT,
+    "locked_at" TIMESTAMP(3),
+
+    CONSTRAINT "background_migrations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "background_migrations_name_key" ON "background_migrations"("name");

packages/shared/prisma/migrations/20241024121500_add_generations_cost_backfill_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('32859a35-98f5-4a4a-b438-ebc579349e00', '20241024_1216_add_generations_cost_backfill', 'addGenerationsCostBackfill', '{}');

packages/shared/prisma/migrations/20241024173000_add_traces_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('5960f22a-748f-480c-b2f3-bc4f9d5d84bc', '20241024_1730_migrate_traces_from_pg_to_ch', 'migrateTracesFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20241024173700_add_observations_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('7526e7c9-0026-4595-af2c-369dfd9176ec', '20241024_1737_migrate_observations_from_pg_to_ch', 'migrateObservationsFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20241024173800_add_scores_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('94e50334-50d3-4e49-ad2e-9f6d92c85ef7', '20241024_1738_migrate_scores_from_pg_to_ch', 'migrateScoresFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20241029130802_prices_drop_excess_index/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY "prices_model_id_idx";

packages/shared/prisma/migrations/20241104111600_background_migrations_add_state_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "background_migrations" ADD COLUMN "state" jsonb NOT NULL DEFAULT '{}';

packages/shared/prisma/migrations/20241105110900_add_claude_haiku_35/migration.sql

@@ -0,0 +1,29 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://docs.anthropic.com/en/docs/about-claude/models#model-comparison-table
+  ('cm34aq60d000207ml0j1h31ar', NULL, 'claude-3-5-haiku-20241022', '(?i)^(claude-3-5-haiku-20241022|anthropic\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$', NULL, 0.000001, 0.000005, NULL, 'TOKENS', 'claude', NULL),
+  ('cm34aqb9h000307ml6nypd618', NULL, 'claude-3.5-haiku-latest', '(?i)^(claude-3-5-haiku-latest)$', NULL, 0.000001, 0.000005, NULL, 'TOKENS', 'claude', NULL);
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm34ax6mc000008jkfqed92mb', 'cm34aq60d000207ml0j1h31ar', 'input', 0.000001),
+  ('cm34axb2o000108jk09wn9b47', 'cm34aqb9h000307ml6nypd618', 'input', 0.000001),
+  ('cm34axeie000208jk8b2ke2t8', 'cm34aq60d000207ml0j1h31ar', 'output', 0.000005),
+  ('cm34axi67000308jk7x1a7qko', 'cm34aqb9h000307ml6nypd618', 'output', 0.000005);

packages/shared/prisma/migrations/20241106122605_add_media_tables/migration.sql

@@ -0,0 +1,71 @@
+-- CreateTable
+CREATE TABLE "media" (
+    "id" TEXT NOT NULL,
+    "sha_256_hash" CHAR(44) NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "uploaded_at" TIMESTAMP(3),
+    "upload_http_status" INTEGER,
+    "upload_http_error" TEXT,
+    "bucket_path" TEXT NOT NULL,
+    "bucket_name" TEXT NOT NULL,
+    "content_type" TEXT NOT NULL,
+    "content_length" BIGINT NOT NULL,
+
+    CONSTRAINT "media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "trace_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "trace_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "observation_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "observation_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "observation_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "media_project_id_sha_256_hash_key" ON "media"("project_id", "sha_256_hash");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "trace_media_project_id_trace_id_media_id_field_key" ON "trace_media"("project_id", "trace_id", "media_id", "field");
+
+-- CreateIndex
+CREATE INDEX "observation_media_project_id_observation_id_idx" ON "observation_media"("project_id", "observation_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "observation_media_project_id_trace_id_observation_id_media__key" ON "observation_media"("project_id", "trace_id", "observation_id", "media_id", "field");
+
+-- AddForeignKey
+ALTER TABLE "media" ADD CONSTRAINT "media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241114175010_job_executions_add_observation_dataset_item_cols/migration.sql

@@ -0,0 +1,6 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_input_trace_id_fkey";
+
+-- AlterTable
+ALTER TABLE "job_executions" ADD COLUMN     "job_input_dataset_item_id" TEXT,
+ADD COLUMN     "job_input_observation_id" TEXT;

packages/shared/prisma/migrations/20241124115100_add_projects_deleted_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN "deleted_at" TIMESTAMP(3);

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/prisma/migrations/20241206115829_remove_trace_score_observation_constraints/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_output_score_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/schema.prisma

@@ -49,6 +49,7 @@ model Account {
   id_token                 String? // @db.Text
   session_state            String?
   refresh_token_expires_in Int?
+  created_at               Int? // GitLab
   user                     User    @relation(fields: [userId], references: [id], onDelete: Cascade)
 
   @@unique([provider, providerAccountId])
@@ -79,6 +80,8 @@ model User {
   createdAt               DateTime                 @default(now()) @map("created_at")
   updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
   featureFlags            String[]                 @default([]) @map("feature_flags")
+  annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
+  annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
 
   @@map("users")
 }
@@ -106,31 +109,37 @@ model Organization {
 }
 
 model Project {
-  id                 String                 @id @default(cuid())
-  orgId              String                 @map("org_id")
-  createdAt          DateTime               @default(now()) @map("created_at")
-  updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
-  name               String
-  projectMembers     ProjectMembership[]
-  organization       Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
-  traces             Trace[]
-  observations       Observation[]
-  apiKeys            ApiKey[]
-  dataset            Dataset[]
-  RawEvents          Events[]
-  invitations        MembershipInvitation[]
-  sessions           TraceSession[]
-  Prompt             Prompt[]
-  Model              Model[]
-  EvalTemplate       EvalTemplate[]
-  JobConfiguration   JobConfiguration[]
-  JobExecution       JobExecution[]
-  LlmApiKeys         LlmApiKeys[]
-  PosthogIntegration PosthogIntegration[]
-  Score              Score[]
-  scoreConfig        ScoreConfig[]
-  BatchExport        BatchExport[]
-  comment            Comment[]
+  id                  String                 @id @default(cuid())
+  orgId               String                 @map("org_id")
+  createdAt           DateTime               @default(now()) @map("created_at")
+  updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+  deletedAt           DateTime?              @map("deleted_at")
+  name                String
+  projectMembers      ProjectMembership[]
+  organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+  traces              Trace[]
+  observations        Observation[]
+  apiKeys             ApiKey[]
+  dataset             Dataset[]
+  RawEvents           Events[]
+  invitations         MembershipInvitation[]
+  sessions            TraceSession[]
+  Prompt              Prompt[]
+  Model               Model[]
+  EvalTemplate        EvalTemplate[]
+  JobConfiguration    JobConfiguration[]
+  JobExecution        JobExecution[]
+  LlmApiKeys          LlmApiKeys[]
+  PosthogIntegration  PosthogIntegration[]
+  Score               Score[]
+  scoreConfig         ScoreConfig[]
+  BatchExport         BatchExport[]
+  comment             Comment[]
+  annotationQueue     AnnotationQueue[]
+  annotationQueueItem AnnotationQueueItem[]
+  TraceMedia          TraceMedia[]
+  Media               Media[]
+  ObservationMedia    ObservationMedia[]
 
   @@index([orgId])
   @@map("projects")
@@ -156,6 +165,21 @@ model ApiKey {
   @@map("api_keys")
 }
 
+model BackgroundMigration {
+  id           String    @id @default(cuid())
+  name         String    @unique
+  script       String    @map("script")
+  args         Json      @map("args")
+  state        Json      @default("{}") @map("state")
+  finishedAt   DateTime? @map("finished_at")
+  failedAt     DateTime? @map("failed_at")
+  failedReason String?   @map("failed_reason")
+  workerId     String?   @map("worker_id")
+  lockedAt     DateTime? @map("locked_at")
+
+  @@map("background_migrations")
+}
+
 model LlmApiKeys {
   id        String   @id @unique @default(cuid())
   createdAt DateTime @default(now()) @map("created_at")
@@ -168,6 +192,7 @@ model LlmApiKeys {
   baseURL           String?  @map("base_url")
   customModels      String[] @default([]) @map("custom_models")
   withDefaultModels Boolean  @default(true) @map("with_default_models")
+  config            Json?
 
   projectId String  @map("project_id")
   project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
@@ -279,8 +304,6 @@ model Trace {
   createdAt  DateTime      @default(now()) @map("created_at")
   updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-  JobExecution JobExecution[]
-
   @@index([projectId, timestamp])
   @@index([sessionId])
   @@index([name])
@@ -446,24 +469,24 @@ enum ObservationLevel {
 }
 
 model Score {
-  id            String         @id @default(cuid())
-  timestamp     DateTime       @default(now())
-  projectId     String         @map("project_id")
-  project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  id            String        @id @default(cuid())
+  timestamp     DateTime      @default(now())
+  projectId     String        @map("project_id")
+  project       Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        ScoreSource
-  authorUserId  String?        @map("author_user_id")
+  authorUserId  String?       @map("author_user_id")
   comment       String?
-  traceId       String         @map("trace_id")
-  observationId String?        @map("observation_id")
-  configId      String?        @map("config_id")
-  stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
-  createdAt     DateTime       @default(now()) @map("created_at")
-  updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
-  dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
-  JobExecution  JobExecution[]
-  scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
+  traceId       String        @map("trace_id")
+  observationId String?       @map("observation_id")
+  configId      String?       @map("config_id")
+  stringValue   String?       @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+  queueId       String?       @map("queue_id")
+  createdAt     DateTime      @default(now()) @map("created_at")
+  updatedAt     DateTime      @default(now()) @updatedAt @map("updated_at")
+  dataType      ScoreDataType @default(NUMERIC) @map("data_type")
+  scoreConfig   ScoreConfig?  @relation(fields: [configId], references: [id], onDelete: SetNull)
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -515,6 +538,59 @@ enum ScoreDataType {
   BOOLEAN
 }
 
+model AnnotationQueue {
+  id                  String                @id @default(cuid())
+  name                String
+  description         String?
+  scoreConfigIds      String[]              @default([]) @map("score_config_ids")
+  projectId           String                @map("project_id")
+  project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt           DateTime              @default(now()) @map("created_at")
+  updatedAt           DateTime              @default(now()) @updatedAt @map("updated_at")
+  annotationQueueItem AnnotationQueueItem[]
+
+  @@unique([projectId, name])
+  @@index([id, projectId])
+  @@index([projectId, createdAt])
+  @@map("annotation_queues")
+}
+
+model AnnotationQueueItem {
+  id              String                    @id @default(cuid())
+  queueId         String                    @map("queue_id")
+  queue           AnnotationQueue           @relation(fields: [queueId], references: [id], onDelete: Cascade)
+  objectId        String                    @map("object_id")
+  objectType      AnnotationQueueObjectType @map("object_type")
+  status          AnnotationQueueStatus     @default(PENDING)
+  lockedAt        DateTime?                 @map("locked_at")
+  lockedByUserId  String?                   @map("locked_by_user_id")
+  lockedByUser    User?                     @relation("LockedByUser", fields: [lockedByUserId], references: [id], onDelete: SetNull)
+  annotatorUserId String?                   @map("annotator_user_id")
+  annotatorUser   User?                     @relation("AnnotatorUser", fields: [annotatorUserId], references: [id], onDelete: SetNull)
+  completedAt     DateTime?                 @map("completed_at")
+  projectId       String                    @map("project_id")
+  project         Project                   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt       DateTime                  @default(now()) @map("created_at")
+  updatedAt       DateTime                  @default(now()) @updatedAt @map("updated_at")
+
+  @@index([id, projectId])
+  @@index([projectId, queueId, status])
+  @@index([objectId, objectType, projectId, queueId])
+  @@index([annotatorUserId])
+  @@index([createdAt])
+  @@map("annotation_queue_items")
+}
+
+enum AnnotationQueueStatus {
+  PENDING
+  COMPLETED
+}
+
+enum AnnotationQueueObjectType {
+  TRACE
+  OBSERVATION
+}
+
 model CronJobs {
   name         String    @id
   lastRun      DateTime? @map("last_run")
@@ -666,7 +742,7 @@ model Prompt {
   version  Int
   type     String   @default("text")
   isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-  config   Json     @default("{}")
+  config   Json     @default("{}") @db.Json
   tags     String[] @default([])
   labels   String[] @default([])
 
@@ -693,15 +769,29 @@ model Model {
   inputPrice      Decimal?  @map("input_price")
   outputPrice     Decimal?  @map("output_price")
   totalPrice      Decimal?  @map("total_price")
-  unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, REQUESTS, or IMAGES
+  unit            String? // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, REQUESTS, or IMAGES
   tokenizerId     String?   @map("tokenizer_id")
   tokenizerConfig Json?     @map("tokenizer_config")
+  Price           Price[]
 
   @@unique([projectId, modelName, startDate, unit])
   @@index(modelName)
   @@map("models")
 }
 
+model Price {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  modelId   String   @map("model_id") // Model is already linked to project (or default), so we don't need projectId here
+  Model     Model    @relation(fields: [modelId], references: [id], onDelete: Cascade)
+  usageType String   @map("usage_type")
+  price     Decimal
+
+  @@unique([modelId, usageType])
+  @@map("prices")
+}
+
 // No FK constraints to preserve audit logs
 model AuditLog {
   id              String   @id @default(cuid())
@@ -775,7 +865,7 @@ model JobConfiguration {
   filter          Json
   targetObject    String         @map("target_object")
   variableMapping Json           @map("variable_mapping")
-  sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+  sampling        Decimal // ratio of jobs that are executed for sampling (0..1)
   delay           Int // delay in milliseconds
   JobExecution    JobExecution[]
 
@@ -806,11 +896,13 @@ model JobExecution {
   endTime   DateTime?          @map("end_time")
   error     String?
 
-  jobInputTraceId String? @map("job_input_trace_id")
-  trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+  jobInputTraceId String? @map("job_input_trace_id") // no fk constraint - traces in ClickHouse, deletion handled via project cascade
+
+  jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
+
+  jobInputDatasetItemId String? @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
-  score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
   @@index([projectId, status])
   @@index([projectId, id])
@@ -872,3 +964,56 @@ model BatchExport {
   @@index([status])
   @@map("batch_exports")
 }
+
+model Media {
+  id               String             @id @default(cuid())
+  sha256Hash       String             @map("sha_256_hash") @db.Char(44)
+  projectId        String             @map("project_id")
+  project          Project            @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt        DateTime           @default(now()) @map("created_at")
+  updatedAt        DateTime           @default(now()) @updatedAt @map("updated_at")
+  uploadedAt       DateTime?          @map("uploaded_at")
+  uploadHttpStatus Int?               @map("upload_http_status")
+  uploadHttpError  String?            @map("upload_http_error")
+  bucketPath       String             @map("bucket_path")
+  bucketName       String             @map("bucket_name")
+  contentType      String             @map("content_type")
+  contentLength    BigInt             @map("content_length")
+  TraceMedia       TraceMedia[]
+  ObservationMedia ObservationMedia[]
+
+  @@unique([projectId, sha256Hash])
+  @@map("media")
+}
+
+model TraceMedia {
+  id        String   @id @default(cuid())
+  projectId String   @map("project_id")
+  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId   String   @map("media_id")
+  media     Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId   String   @map("trace_id")
+  field     String   @map("field")
+
+  @@unique([projectId, traceId, mediaId, field])
+  @@map("trace_media")
+}
+
+model ObservationMedia {
+  id            String   @id @default(cuid())
+  projectId     String   @map("project_id")
+  project       Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt     DateTime @default(now()) @map("created_at")
+  updatedAt     DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId       String   @map("media_id")
+  media         Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId       String   @map("trace_id")
+  observationId String   @map("observation_id")
+  field         String   @map("field")
+
+  @@unique([projectId, traceId, observationId, mediaId, field])
+  @@index([projectId, observationId])
+  @@map("observation_media")
+}

packages/shared/prisma/seed.ts

@@ -5,6 +5,7 @@ import {
   ObservationType,
   ScoreSource,
   ScoreDataType,
+  AnnotationQueueObjectType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -77,6 +78,9 @@ async function main() {
     create: {
       id: seedOrgId,
       name: "Seed Org",
+      cloudConfig: {
+        plan: "Team",
+      },
     },
   });
 
@@ -250,6 +254,11 @@ async function main() {
       project2,
     ]);
 
+    const queueIds = await generateQueuesForProject(
+      [project1, project2],
+      configIdsAndNames,
+    );
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -257,16 +266,24 @@ async function main() {
 
     const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;
 
-    const { traces, observations, scores, sessions, events, comments } =
-      createObjects(
-        traceVolume,
-        envTags,
-        colorTags,
-        project1,
-        project2,
-        promptIds,
-        configIdsAndNames,
-      );
+    const {
+      traces,
+      observations,
+      scores,
+      sessions,
+      events,
+      comments,
+      queueItems,
+    } = createObjects(
+      traceVolume,
+      envTags,
+      colorTags,
+      project1,
+      project2,
+      promptIds,
+      queueIds,
+      configIdsAndNames,
+    );
 
     logger.info(
       `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`,
@@ -279,6 +296,7 @@ async function main() {
       sessions,
       events,
       comments,
+      queueItems,
     );
 
     // If openai key is in environment, add it to the projects LLM API keys
@@ -369,16 +387,62 @@ async function main() {
       update: {},
     });
 
-    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
-      const dataset = await prisma.dataset.create({
-        data: {
-          name: `demo-dataset-${datasetNumber}`,
-          description:
-            datasetNumber === 0 ? "Dataset test description" : undefined,
-          projectId: project2.id,
-          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
-        },
-      });
+    await createDatasets(project1, project2, observations);
+  }
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+  })
+  .catch(async (e) => {
+    logger.error(e);
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+    process.exit(1);
+  });
+
+export async function createDatasets(
+  project1: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  project2: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  observations: Prisma.ObservationCreateManyInput[],
+) {
+  for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
+    for (const projectId of [project1.id, project2.id]) {
+      const datasetName = `demo-dataset-${datasetNumber}`;
+
+      // check if ds already exists
+      const dataset =
+        (await prisma.dataset.findFirst({
+          where: {
+            projectId,
+            name: datasetName,
+          },
+        })) ??
+        (await prisma.dataset.create({
+          data: {
+            name: datasetName,
+            description:
+              datasetNumber === 0 ? "Dataset test description" : undefined,
+            projectId,
+            metadata: datasetNumber === 0 ? { key: "value" } : undefined,
+          },
+        }));
 
       const datasetItemIds = [];
       for (let i = 0; i < 18; i++) {
@@ -386,9 +450,12 @@ async function main() {
           Math.random() > 0.3
             ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
+        if (!sourceObservation) {
+          continue;
+        }
         const datasetItem = await prisma.datasetItem.create({
           data: {
-            projectId: project2.id,
+            projectId,
             datasetId: dataset.id,
             sourceTraceId: sourceObservation?.traceId,
             sourceObservationId:
@@ -413,9 +480,16 @@ async function main() {
       }
 
       for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
-        const datasetRun = await prisma.datasetRuns.create({
-          data: {
-            projectId: project2.id,
+        const datasetRun = await prisma.datasetRuns.upsert({
+          where: {
+            datasetId_projectId_name: {
+              datasetId: dataset.id,
+              projectId,
+              name: `demo-dataset-run-${datasetRunNumber}`,
+            },
+          },
+          create: {
+            projectId,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
@@ -427,20 +501,24 @@ async function main() {
               ["tag1", "tag2"],
             ][datasetRunNumber % 5],
           },
+          update: {},
         });
 
         for (const datasetItemId of datasetItemIds) {
           const relevantObservations = observations.filter(
-            (o) => o.projectId === project2.id,
+            (o) => o.projectId === projectId,
           );
           const observation =
             relevantObservations[
               Math.floor(Math.random() * relevantObservations.length)
             ];
 
+          if (!observation) {
+            continue;
+          }
           await prisma.datasetRunItems.create({
             data: {
-              projectId: project2.id,
+              projectId,
               datasetItemId,
               traceId: observation.traceId as string,
               observationId: Math.random() > 0.5 ? observation.id : undefined,
@@ -453,20 +531,6 @@ async function main() {
   }
 }
 
-main()
-  .then(async () => {
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-  })
-  .catch(async (e) => {
-    logger.error(e);
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-    process.exit(1);
-  });
-
 async function uploadObjects(
   traces: Prisma.TraceCreateManyInput[],
   observations: Prisma.ObservationCreateManyInput[],
@@ -474,6 +538,7 @@ async function uploadObjects(
   sessions: Prisma.TraceSessionCreateManyInput[],
   events: Prisma.ObservationCreateManyInput[],
   comments: Prisma.CommentCreateManyInput[],
+  queueItems: Prisma.AnnotationQueueItemCreateManyInput[],
 ) {
   let promises: Prisma.PrismaPromise<unknown>[] = [];
 
@@ -581,6 +646,22 @@ async function uploadObjects(
       );
     await promises[i];
   }
+
+  promises = [];
+  chunk(queueItems, chunkSize).forEach((chunk) => {
+    promises.push(
+      prisma.annotationQueueItem.createMany({
+        data: chunk,
+      }),
+    );
+  });
+  for (let i = 0; i < promises.length; i++) {
+    if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
+      logger.info(
+        `Seeding of Annotation Queue Items ${((i + 1) / promises.length) * 100}% complete`,
+      );
+    await promises[i];
+  }
 }
 
 function createObjects(
@@ -590,6 +671,7 @@ function createObjects(
   project1: Project,
   project2: Project,
   promptIds: Map<string, string[]>,
+  queueIds: Map<string, string[]>,
   configParams: Map<
     string,
     {
@@ -607,6 +689,7 @@ function createObjects(
   const events: Prisma.ObservationCreateManyInput[] = [];
   const configs: Prisma.ScoreConfigCreateManyInput[] = [];
   const comments: Prisma.CommentCreateManyInput[] = [];
+  const queueItems: Prisma.AnnotationQueueItemCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -689,6 +772,21 @@ function createObjects(
       }),
     };
 
+    const queueItem = [
+      ...(Math.random() > 0.9 && queueIds.get(projectId)?.[0]
+        ? [
+            {
+              queueId: queueIds.get(projectId)?.[0] as string,
+              objectId: trace.id,
+              objectType: AnnotationQueueObjectType.TRACE,
+              projectId,
+            },
+          ]
+        : []),
+    ];
+
+    queueItems.push(...queueItem);
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
@@ -941,6 +1039,7 @@ function createObjects(
     observations,
     scores,
     configs,
+    queueItems,
     sessions: uniqueSessions,
     events,
     comments,
@@ -959,60 +1058,59 @@ async function generatePromptsForProject(projects: Project[]) {
   return promptIds;
 }
 
-async function generatePrompts(project: Project) {
-  const promptIds: string[] = [];
-  const prompts = [
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 1 content",
-      name: "Prompt 1",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 2 content",
-      name: "Prompt 2",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "API",
-      prompt: "Prompt 3 content",
-      name: "Prompt 3 by API",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 4 content",
-      name: "Prompt 4",
-      version: 1,
-      labels: ["production", "latest"],
-      tags: ["tag1", "tag2"],
-    },
-  ];
+export const SEED_PROMPTS = [
+  {
+    id: `prompt-123`,
+    createdBy: "user-1",
+    prompt: "Prompt 1 content",
+    name: "Prompt 1",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-456`,
+    createdBy: "user-1",
+    prompt: "Prompt 2 content",
+    name: "Prompt 2",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-789`,
+    createdBy: "API",
+    prompt: "Prompt 3 content",
+    name: "Prompt 3 by API",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: "Prompt 4 content",
+    name: "Prompt 4",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
 
-  for (const prompt of prompts) {
+export const PROMPT_IDS: string[] = [];
+
+async function generatePrompts(project: Project) {
+  const promptIds = [];
+  for (const prompt of SEED_PROMPTS) {
     await prisma.prompt.upsert({
       where: {
         projectId_name_version: {
-          projectId: prompt.projectId,
+          projectId: prompt.id + project.id,
           name: prompt.name,
           version: prompt.version,
         },
+        id: prompt.id + project.id,
       },
       create: {
-        id: prompt.id,
-        projectId: prompt.projectId,
+        id: prompt.id + project.id,
+        projectId: project.id,
         createdBy: prompt.createdBy,
         prompt: prompt.prompt,
         name: prompt.name,
@@ -1020,9 +1118,7 @@ async function generatePrompts(project: Project) {
         labels: prompt.labels,
         tags: prompt.tags,
       },
-      update: {
-        id: prompt.id,
-      },
+      update: {},
     });
     promptIds.push(prompt.id);
   }
@@ -1076,6 +1172,7 @@ async function generatePrompts(project: Project) {
           name: version.name,
           version: version.version,
         },
+        id: version.id,
       },
       create: {
         id: version.id,
@@ -1106,6 +1203,7 @@ async function generatePrompts(project: Project) {
           name: promptName,
           version: i,
         },
+        id: promptId,
       },
       create: {
         id: promptId,
@@ -1218,6 +1316,7 @@ async function generateConfigs(project: Project) {
 
   return configNameAndId;
 }
+
 function getGenerationInputOutput(): {
   input: Prisma.InputJsonValue;
   output: Prisma.InputJsonValue;
@@ -1278,3 +1377,64 @@ function getGenerationInputOutput(): {
 
   return { input, output };
 }
+
+async function generateQueuesForProject(
+  projects: Project[],
+  configIdsAndNames: Map<
+    string,
+    {
+      name: string;
+      id: string;
+      dataType: ScoreDataType;
+      categories: ConfigCategory[] | null;
+    }[]
+  >,
+) {
+  const projectIdsToQueues: Map<string, string[]> = new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const queueIds = await generateQueues(
+        project,
+        configIdsAndNames.get(project.id) ?? [],
+      );
+      projectIdsToQueues.set(project.id, queueIds);
+    }),
+  );
+  return projectIdsToQueues;
+}
+
+async function generateQueues(
+  project: Project,
+  configIdsAndNames: {
+    name: string;
+    id: string;
+    dataType: ScoreDataType;
+    categories: ConfigCategory[] | null;
+  }[],
+) {
+  const queue = {
+    id: `queue-${v4()}`,
+    name: "Default",
+    description: "Default queue",
+    scoreConfigIds: configIdsAndNames.map((config) => config.id),
+    projectId: project.id,
+  };
+
+  await prisma.annotationQueue.upsert({
+    where: {
+      projectId_name: {
+        projectId: queue.projectId,
+        name: queue.name,
+      },
+    },
+    create: {
+      ...queue,
+    },
+    update: {
+      id: queue.id,
+    },
+  });
+
+  return [queue.id];
+}

packages/shared/scripts/load-seed.ts

@@ -0,0 +1,134 @@
+import { randomUUID } from "crypto";
+import { prisma } from "../src/db";
+import {
+  clickhouseClient,
+  getDisplaySecretKey,
+  hashSecretKey,
+  logger,
+} from "../src/server";
+import { prepareClickhouse } from "./prepareClickhouse";
+import { redis } from "../src/server";
+
+const createRandomProjectId = () => randomUUID().toString();
+
+const prepareProjectsAndApiKeys = async (
+  numOfProjects: number,
+  opts: { requiredProjectIds: string[] },
+) => {
+  const { requiredProjectIds } = opts;
+  const projectsToCreate = numOfProjects - requiredProjectIds.length;
+  const projectIds = [...requiredProjectIds];
+
+  for (let i = 0; i < projectsToCreate; i++) {
+    projectIds.push(createRandomProjectId());
+  }
+
+  const operations = projectIds.map(async (projectId) => {
+    const orgId = `org-${projectId}`;
+
+    await prisma.organization.upsert({
+      where: { id: orgId },
+      update: {},
+      create: {
+        id: orgId,
+        name: `Organization for ${projectId}`,
+      },
+    });
+
+    await prisma.project.upsert({
+      where: { id: projectId },
+      update: {},
+      create: {
+        id: projectId,
+        name: `Project ${projectId}`,
+        orgId: orgId,
+      },
+    });
+
+    const apiKeyId = `api-key-${projectId}`;
+    const apiKeyExists = await prisma.apiKey.findUnique({
+      where: { id: apiKeyId },
+    });
+    if (!apiKeyExists) {
+      const sk = await hashSecretKey(
+        `sk-${Math.random().toString(36).substr(2, 9)}`,
+      );
+      await prisma.apiKey.create({
+        data: {
+          id: apiKeyId,
+          note: `API Key for ${projectId}`,
+          publicKey: `pk-${Math.random().toString(36).substr(2, 9)}`,
+          hashedSecretKey: sk,
+          displaySecretKey: getDisplaySecretKey(sk),
+          project: {
+            connect: {
+              id: projectId,
+            },
+          },
+        },
+      });
+    }
+  });
+
+  await Promise.all(operations);
+  return projectIds;
+};
+
+async function main() {
+  let numOfProjects = parseInt(process.argv[2], 10);
+  let numberOfDays = parseInt(process.argv[3], 10);
+  let totalObservations = parseInt(process.argv[4], 10);
+
+  logger.info(process.argv);
+  logger.info(
+    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`,
+  );
+
+  if (isNaN(totalObservations)) {
+    logger.warn(
+      "Total observations not provided or invalid. Defaulting to 1000 observations.",
+    );
+    totalObservations = 1000;
+  }
+
+  if (isNaN(numOfProjects)) {
+    logger.warn(
+      "Number of projects not provided or invalid. Defaulting to 10 projects.",
+    );
+    numOfProjects = 10;
+  }
+
+  if (isNaN(numberOfDays)) {
+    logger.warn(
+      "Number of days not provided or invalid. Defaulting to 3 days.",
+    );
+    numberOfDays = 3;
+  }
+
+  try {
+    const projectIds = [
+      "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      "239ad00f-562f-411d-af14-831c75ddd875",
+    ];
+
+    const createdProjectIds = await prepareProjectsAndApiKeys(numOfProjects, {
+      requiredProjectIds: projectIds,
+    });
+
+    await prepareClickhouse(createdProjectIds, {
+      numberOfDays,
+      totalObservations: totalObservations ?? 1000,
+    });
+
+    logger.info("Clickhouse preparation completed successfully.");
+  } catch (error) {
+    logger.error("Error during Clickhouse preparation:", error);
+  } finally {
+    await clickhouseClient().close();
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from Clickhouse.");
+  }
+}
+
+main();

packages/shared/scripts/prepareClickhouse.ts

@@ -0,0 +1,256 @@
+import { SEED_PROMPTS } from "../prisma/seed";
+import { prisma } from "../src/db";
+import { clickhouseClient, logger } from "../src/server";
+
+function randn_bm(min: number, max: number, skew: number) {
+  let u = 0,
+    v = 0;
+  while (u === 0) u = Math.random(); //Converting [0,1) to (0,1)
+  while (v === 0) v = Math.random();
+  let num = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+
+  num = num / 10.0 + 0.5; // Translate to 0 -> 1
+  if (num > 1 || num < 0)
+    num = randn_bm(min, max, skew); // resample between 0 and 1 if out of range
+  else {
+    num = Math.pow(num, skew); // Skew
+    num *= max - min; // Stretch to fill range
+    num += min; // offset to min
+  }
+  return num;
+}
+
+export const prepareClickhouse = async (
+  projectIds: string[],
+  opts: {
+    numberOfDays: number;
+    totalObservations: number;
+  },
+) => {
+  logger.info(
+    `Preparing Clickhouse for ${projectIds.length} projects and ${opts.numberOfDays} days.`,
+  );
+
+  const projectData = projectIds.map((projectId) => {
+    const observationsPerProject = Math.ceil(
+      randn_bm(0, opts.totalObservations, 2),
+    ); // Skew the number of observations
+
+    const tracesPerProject = Math.floor(observationsPerProject / 6); // On average, one trace should have 6 observations
+    const scoresPerProject = tracesPerProject * 10; // On average, one trace should have 10 scores
+    return {
+      projectId,
+      observationsPerProject,
+      tracesPerProject,
+      scoresPerProject,
+    };
+  });
+
+  for (const data of projectData) {
+    const {
+      projectId,
+      tracesPerProject,
+      observationsPerProject,
+      scoresPerProject,
+    } = data;
+    logger.info(
+      `Preparing Clickhouse for ${projectId}: Traces: ${tracesPerProject}, Scores: ${scoresPerProject}, Observations: ${observationsPerProject}`,
+    );
+
+    const tracesQuery = `
+    INSERT INTO traces
+    SELECT toString(number) AS id,
+      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+      concat('name_', toString(rand() % 100)) AS name,
+      concat('user_id_', toInt64(randExponential(1 / 100))) AS user_id,
+      map('key', 'value') AS metadata,
+      concat('release_', toString(randUniform(0, 100))) AS release,
+      concat('version_', toString(randUniform(0, 100))) AS version,
+      '${projectId}' AS project_id,
+      if(rand() < 0.8, true, false) as public,
+      if(rand() < 0.8, true, false) as bookmarked,
+      array('tag1', 'tag2') as tags,
+      repeat('input', toInt64(randExponential(1 / 100))) AS input,
+      repeat('output', toInt64(randExponential(1 / 100))) AS output,
+      if(randUniform(0, 1) < 0.2, NULL, concat('session_', toString(rand() % 1000))) AS session_id,
+      timestamp AS created_at,
+      timestamp AS updated_at,
+      timestamp AS event_ts,
+      0 AS is_deleted
+    FROM numbers(${tracesPerProject});
+  `;
+
+    const observationsQuery = `
+    INSERT INTO observations
+    SELECT toString(number) AS id,
+      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
+      '${projectId}' AS project_id,
+      if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
+      toString(rand()) AS parent_observation_id,
+      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
+      addSeconds(start_time, if(rand() < 0.6, floor(randUniform(0, 20)), floor(randUniform(0, 3600)))) AS end_time,
+      concat('name', toString(rand() % 100)) AS name,
+      map('key', 'value') AS metadata,
+      if(randUniform(0, 1) < 0.9, 'DEFAULT', if(randUniform(0, 1) < 0.5, 'ERROR', if(randUniform(0, 1) < 0.5, 'DEBUG', 'WARNING'))) AS level,
+      'status_message' AS status_message,
+      'version' AS version,
+      repeat('input', toInt64(randExponential(1 / 100))) AS input,
+      repeat('output', toInt64(randExponential(1 / 100))) AS output,
+      case
+        when number % 2 = 0 then 'claude-3-haiku-20240307'
+        else 'gpt-4'
+      end as provided_model_name,
+      case
+        when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
+        else 'clrntkjgy000f08jx79v9g1xj'
+      end as internal_model_id,
+      '{"temperature": 0.7, "max_tokens": 150}' AS model_parameters,
+      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS provided_usage_details,
+      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS usage_details,
+      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
+      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
+      toDecimal64(randUniform(0, 2000), 12) AS total_cost,
+      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
+      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,
+      array(${SEED_PROMPTS.map((p) => `'${p.name}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_name,
+      array(${SEED_PROMPTS.map((p) => `'${p.version}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_version,
+      start_time AS created_at,
+      start_time AS updated_at,
+      start_time AS event_ts,
+      0 AS is_deleted
+    FROM numbers(${observationsPerProject});
+  `;
+
+    console.log(observationsQuery);
+
+    const scoresQuery = `
+    INSERT INTO scores
+    SELECT toString(floor(randUniform(0, 100))) AS id,
+      toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+      '${projectId}' AS project_id,
+      toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
+      if(
+        rand() > 0.9,
+        toString(floor(randUniform(0, ${observationsPerProject}))),
+        NULL
+      ) AS observation_id,
+      concat('name_', toString(rand() % 10)) AS name,
+      randUniform(0, 100) as value,
+      'API' as source,
+      'comment' as comment,
+      toString(rand() % 100) as author_user_id,
+      toString(rand() % 100) as config_id,
+      if (rand() < 0.33, 'NUMERIC', if (rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) as data_type,
+      toString(rand() % 100) as string_value,
+      NULL as queue_id,
+      timestamp AS created_at,
+      timestamp AS updated_at,
+      timestamp AS event_ts,
+      0 AS is_deleted
+    FROM numbers(${scoresPerProject});
+  `;
+
+    const queries = [tracesQuery, scoresQuery, observationsQuery];
+
+    for (const query of queries) {
+      logger.info(`Executing query: ${query}`);
+      await clickhouseClient().command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        },
+      });
+    }
+    // we also need to upsert trace sessions in postgres
+
+    const sessionQuery = `
+      SELECT session_id, project_id
+      FROM traces 
+      WHERE session_id IS NOT NULL;
+    `;
+    const sessionResult = await clickhouseClient().query({
+      query: sessionQuery,
+      format: "JSONEachRow",
+    });
+
+    const sessionData = await sessionResult.json<{
+      session_id: string;
+      project_id: string;
+    }>();
+
+    const idProjectIdCombinations = sessionData.map((session) => ({
+      id: session.session_id,
+      projectId: session.project_id,
+      public: Math.random() < 0.1,
+      bookmarked: Math.random() < 0.1,
+    }));
+
+    await prisma.traceSession.createMany({
+      data: idProjectIdCombinations,
+      skipDuplicates: true,
+    });
+  }
+
+  const tables = ["traces", "scores", "observations"];
+  for (const table of tables) {
+    const query = `
+            SELECT
+            project_id,
+            count() AS per_project_count,
+            bar(per_project_count, 0, (
+                SELECT count(*)
+                FROM ${table}
+            ), 50) AS bar_representation
+          FROM ${table}
+          GROUP BY project_id
+          ORDER BY count() desc
+          `;
+
+    const result = await clickhouseClient().query({
+      query,
+      format: "TabSeparated",
+    });
+
+    logger.info(
+      `${table.charAt(0).toUpperCase() + table.slice(1)} per Project: \n` +
+        (await result.text()),
+    );
+  }
+
+  const tablesWithDateColumns = [
+    { name: "traces", dateColumn: "timestamp" },
+    { name: "scores", dateColumn: "timestamp" },
+    { name: "observations", dateColumn: "start_time" },
+  ];
+
+  for (const { name: table, dateColumn } of tablesWithDateColumns) {
+    const query = `
+            SELECT
+            toDate(${dateColumn}) AS event_date,
+            count() AS per_date_count,
+            bar(per_date_count, 0, (
+                SELECT count(*)
+                FROM ${table}
+            ), 50) AS bar_representation
+          FROM ${table}
+          GROUP BY event_date
+          ORDER BY event_date desc
+          `;
+
+    const result = await clickhouseClient().query({
+      query,
+      format: "TabSeparated",
+    });
+
+    logger.info(
+      `${table.charAt(0).toUpperCase() + table.slice(1)} per Date: \n` +
+        (await result.text()),
+    );
+  }
+};

packages/shared/src/db.ts

@@ -21,37 +21,40 @@ export class PrismaClientSingleton {
       return PrismaClientSingleton.instance;
     }
 
-    const client = new PrismaClient<
-      Prisma.PrismaClientOptions,
-      "warn" | "error" | "query"
-    >({
-      log: [
-        { emit: "event", level: "query" },
-        { emit: "event", level: "error" },
-        { emit: "event", level: "warn" },
-      ],
-    });
-
-    if (env.NODE_ENV === "development") {
-      client.$on("query", (event) => {
-        logger.info(`prisma:query ${event.query}, ${event.duration}ms`);
-      });
-    }
-
-    client.$on("warn", (event) => {
-      logger.warn(`prisma:warn ${event.message}`);
-    });
-
-    client.$on("error", (event) => {
-      logger.error(`prisma:error ${event.message}`);
-    });
-
-    PrismaClientSingleton.instance = client;
+    PrismaClientSingleton.instance = createPrismaInstance();
 
     return PrismaClientSingleton.instance;
   }
 }
 
+const createPrismaInstance = () => {
+  const client = new PrismaClient<
+    Prisma.PrismaClientOptions,
+    "warn" | "error" | "query"
+  >({
+    log: [
+      { emit: "event", level: "query" },
+      { emit: "event", level: "error" },
+      { emit: "event", level: "warn" },
+    ],
+  });
+
+  if (env.NODE_ENV === "development") {
+    client.$on("query", (event) => {
+      logger.info(`prisma:query ${event.query}, ${event.duration}ms`);
+    });
+  }
+
+  client.$on("warn", (event) => {
+    logger.warn(`prisma:warn ${event.message}`);
+  });
+
+  client.$on("error", (event) => {
+    logger.error(`prisma:error ${event.message}`);
+  });
+  return client;
+};
+
 export class KyselySingleton {
   private static instance: { $kysely: Kysely<DB> };
 
@@ -80,7 +83,33 @@ export class KyselySingleton {
   }
 }
 
-export const prisma = PrismaClientSingleton.getInstance();
-export const kyselyPrisma = KyselySingleton.getInstance();
+declare const globalThis: {
+  prismaGlobal: PrismaClient | undefined;
+  kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
+} & typeof global;
+
+if (process.env.NODE_ENV === "development") {
+  globalThis.prismaGlobal ??= createPrismaInstance(); // regular instantiation
+  globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    }),
+  );
+}
+
+export const prisma =
+  globalThis.prismaGlobal ?? PrismaClientSingleton.getInstance();
+export const kyselyPrisma =
+  globalThis.kyselyPrismaGlobal ?? KyselySingleton.getInstance();
 
 export * from "@prisma/client";

packages/shared/src/env.ts

@@ -1,9 +1,11 @@
 import { z } from "zod";
+import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
+  NEXTAUTH_URL: z.string().url().optional(),
   REDIS_HOST: z.string().nullish(),
   REDIS_PORT: z.coerce
     .number({
@@ -26,23 +28,47 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
-  CLICKHOUSE_URL: z.string().url().optional(),
-  CLICKHOUSE_USER: z.string().optional(),
-  CLICKHOUSE_PASSWORD: z.string().optional(),
-  LANGFUSE_INGESTION_FLUSH_DELAY_MS: z.coerce
+  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_USER: z.string(),
+  CLICKHOUSE_PASSWORD: z.string(),
+  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("true"),
+  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+
+  LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
     .number()
     .nonnegative()
-    .default(10000),
-  LANGFUSE_INGESTION_FLUSH_ATTEMPTS: z.coerce.number().positive().default(3),
-  LANGFUSE_INGESTION_BUFFER_TTL_SECONDS: z.coerce
-    .number()
-    .positive()
-    .default(60 * 10),
+    .default(15_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
     .optional(),
   LANGFUSE_LOG_FORMAT: z.enum(["text", "json"]).default("text"),
+  ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
+    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
+  }),
+  LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
+  LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
 });
 
-export const env = EnvSchema.parse(process.env);
+export const env: z.infer<typeof EnvSchema> =
+  process.env.DOCKER_BUILD === "1"
+    ? (process.env as any)
+    : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/annotation/types.ts

@@ -32,6 +32,7 @@ const CreateAnnotationScoreBase = z.object({
   configId: z.string().optional(),
   observationId: z.string().optional(),
   comment: z.string().nullish(),
+  queueId: z.string().nullish(),
 });
 
 const UpdateAnnotationScoreBase = CreateAnnotationScoreBase.extend({
@@ -57,3 +58,15 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
   UpdateAnnotationScoreBase.merge(CategoricalData),
   UpdateAnnotationScoreBase.merge(BooleanData),
 ]);
+
+// annotation queues
+
+export const CreateQueueData = z.object({
+  name: z.string().min(1).max(35),
+  description: z.string().max(1000).optional(),
+  scoreConfigIds: z.array(z.string()).min(1, {
+    message: "At least 1 score config must be selected",
+  }),
+});
+
+export type CreateQueue = z.infer<typeof CreateQueueData>;

packages/shared/src/features/comments/types.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+const MAX_COMMENT_LENGTH = 3000;
+
 const COMMENT_OBJECT_TYPES = [
   "TRACE",
   "OBSERVATION",
@@ -9,7 +11,7 @@ const COMMENT_OBJECT_TYPES = [
 
 export const CreateCommentData = z.object({
   projectId: z.string(),
-  content: z.string().trim().min(1).max(500),
+  content: z.string().trim().min(1).max(MAX_COMMENT_LENGTH),
   objectId: z.string(),
   objectType: z.enum(COMMENT_OBJECT_TYPES),
 });

packages/shared/src/features/entitlements/plans.ts

@@ -3,15 +3,17 @@ export const planLabels = {
   "cloud:hobby": "Hobby",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
-  "self-hosted:enterprise": "Enterprise",
+  "self-hosted:pro": "Pro (self-hosted)",
+  "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
 
 export type Plan = keyof typeof planLabels;
 
 export const plans = Object.keys(planLabels) as Plan[];
 
-// This function is kept here to ensure consistency when updating plan names in the future.
+// These functions are kept here to ensure consistency when updating plan names in the future.
 export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/features/evals/types.ts

@@ -1,11 +1,11 @@
 import z from "zod";
-import { LLMAdapter } from "../..";
 
 export const langfuseObjects = [
   "trace",
   "span",
   "generation",
   "event",
+  "dataset_item",
 ] as const;
 
 // variable mapping stored in the db for eval templates
@@ -22,7 +22,7 @@ export const variableMapping = z
     (value) => value.langfuseObject === "trace" || value.objectName !== null,
     {
       message: "objectName is required for langfuseObjects other than trace",
-    }
+    },
   );
 
 export const variableMappingList = z.array(variableMapping);
@@ -45,7 +45,7 @@ const observationCols = [
   { name: "Output", id: "output", internal: 'o."output"' },
 ];
 
-export const availableEvalVariables = [
+export const availableTraceEvalVariables = [
   {
     id: "trace",
     display: "Trace",
@@ -77,17 +77,27 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalModelsByAdapter: Record<LLMAdapter, string[]> = {
-  [LLMAdapter.OpenAI]: [
-    "gpt-4o",
-    "gpt-4o-mini",
-    "gpt-4o-2024-08-06", // cheaper than gpt-4o
-    "gpt-4-turbo-preview",
-    "gpt-3.5-turbo",
-  ] as const,
-  [LLMAdapter.Anthropic]: [] as const,
-  [LLMAdapter.Azure]: [] as const,
-};
+export const availableDatasetEvalVariables = [
+  {
+    id: "dataset_item",
+    display: "Dataset item",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'd."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'd."input"' },
+      {
+        name: "Expected output",
+        id: "expected_output",
+        internal: 'd."expected_output"',
+      },
+    ],
+  },
+  ...availableTraceEvalVariables,
+];
 
 export const OutputSchema = z.object({
   reasoning: z.string(),
@@ -96,6 +106,7 @@ export const OutputSchema = z.object({
 
 export enum EvalTargetObject {
   Trace = "trace",
+  Dataset = "dataset",
 }
 
 export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/features/experiments/utils.ts

@@ -0,0 +1,15 @@
+import { Prisma } from "../../db";
+
+export const datasetItemMatchesVariable = (
+  input: Prisma.JsonValue,
+  variable: string,
+) => {
+  if (
+    input === null ||
+    input === undefined ||
+    typeof input !== "object" ||
+    Array.isArray(input)
+  )
+    return false;
+  return Object.keys(input).includes(variable);
+};

packages/shared/src/features/scores/index.ts

@@ -0,0 +1,3 @@
+export * from "./scoreTypes";
+export * from "./types";
+export * from "./scoreConfigTypes";

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -6,7 +6,7 @@ import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 
 /**
@@ -17,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx
+  ctx: z.RefinementCtx,
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -94,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value
+          category.value === expectedCategories[index].value,
       );
     }),
 });
@@ -123,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      })
+      }),
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
@@ -170,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType
+  scoreConfig: ScoreConfigDbType,
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -205,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      })
+      }),
     ),
   ])
   .superRefine((data, ctx) => {
@@ -227,7 +227,7 @@ export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
 
 // GET /score-configs
 export const GetScoreConfigsQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
 });
 
 export const GetScoreConfigsResponse = z.object({

packages/shared/src/features/scores/scoreTypes.ts

@@ -6,7 +6,7 @@ import { isPresent, stringDateTime } from "../../utils/typeChecks";
 import {
   NonEmptyString,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 import { Category as ConfigCategory } from "./scoreConfigTypes";
 
@@ -55,6 +55,7 @@ const ScoreBase = z.object({
   configId: z.string().nullish(),
   createdAt: z.coerce.date(),
   updatedAt: z.coerce.date(),
+  queueId: z.string().nullish(),
 });
 
 const BaseScoreBody = z.object({
@@ -230,10 +231,12 @@ export const PostScoresResponse = z.object({ id: z.string() });
 
 // GET /scores
 export const GetScoresQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
   userId: z.string().nullish(),
   dataType: z.enum(ScoreDataType).nullish(),
   configId: z.string().nullish(),
+  queueId: z.string().nullish(),
+  traceTags: z.union([z.array(z.string()), z.string()]).nullish(),
   name: z.string().nullish(),
   fromTimestamp: stringDateTime,
   toTimestamp: stringDateTime,
@@ -255,6 +258,7 @@ const LegacyGetScoreResponseDataV1 = z.intersection(
   z.object({
     trace: z.object({
       userId: z.string().nullish(),
+      tags: z.array(z.string()).nullish(),
     }),
   }),
 );