chore: release v3.3.0

* feat: redirect specific projects to secondary ingestion queue

* feat: redirect specific projects to secondary ingestion queue

* push

* Update worker/src/queues/ingestionQueue.ts

Co-authored-by: Steffen Schmitz <steffen@langfuse.com>

* push

* push

---------

Co-authored-by: Steffen Schmitz <steffen@langfuse.com>

.env.dev-azure.example

@@ -0,0 +1,84 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_BATCH_EXPORT_REGION=auto
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.dev.example

@@ -11,6 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -33,18 +34,29 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -66,4 +78,4 @@ REDIS_AUTH="myredissecret"
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.dev.legacy.example

@@ -0,0 +1,86 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# # S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=true
+LANGFUSE_RETURN_FROM_CLICKHOUSE=false
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false

.env.local.example

@@ -1,22 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
-# Next Auth
-NEXTAUTH_SECRET="secret"
-NEXTAUTH_URL="http://localhost:3000"
-
-# feature flag to enable experimental features locally
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
-
-# Redis
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,6 +92,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
@@ -100,16 +109,16 @@ OTEL_SERVICE_NAME="langfuse"
 # Defines the connection url for smtp server. 
 # SMTP_CONNECTION_URL=
 
-# S3 storage, optional, used for exports from the UI
-# S3_ENDPOINT=
-# S3_ACCESS_KEY_ID=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION=
-# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
+# S3 Batch Exports
+# LANGFUSE_S3_BATCH_EXPORT_ENABLED=
+# LANGFUSE_S3_BATCH_EXPORT_BUCKET=
+# LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=
+# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=
+# LANGFUSE_S3_BATCH_EXPORT_REGION=
+# LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=
+# LANGFUSE_S3_BATCH_EXPORT_PREFIX=
 
 # S3 storage for events, optional, used to persist all incoming events
-# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
 # LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
 # Optional prefix to be used within the bucket. Must end with `/` if set
 # LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
@@ -242,7 +251,6 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
 # LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
 ## END Langfuse V3 Ingestion

.github/workflows/_deploy_ecs_service.yml

@@ -58,7 +58,6 @@ jobs:
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-
       - name: Render AWS ECS Task Definition
         id: render-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1

.github/workflows/codespell.yml

@@ -6,6 +6,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -10,7 +10,11 @@ on:
   merge_group:
   pull_request:
     branches:
-      - "*"
+      - "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:
@@ -41,6 +45,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -66,10 +71,10 @@ jobs:
         run: |
           timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
 
-  tests-web:
+  tests-web-sync:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
@@ -89,6 +94,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -105,7 +111,8 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -131,17 +138,90 @@ jobs:
           LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
           LANGFUSE_INIT_USER_NAME: "Demo User"
           LANGFUSE_INIT_USER_PASSWORD: "password"
+      - name: run test-sync
+        run: pnpm --filter=web run test-sync
+      - name: run test-client
+        run: pnpm --filter=web run test-client
+
+  tests-web-async:
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
+    strategy:
+      matrix:
+        node-version: [20]
+        postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+      - uses: actions/checkout@v4
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm install
+      - name: Load default env
+        run: |
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm --filter=shared ch:up
+      - name: Build
+        run: pnpm run build
+      - name: Start Langfuse
+        run: (pnpm run start&)
+        env:
+          LANGFUSE_INIT_ORG_ID: "seed-org-id"
+          LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+          LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
+          LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
+          LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890"
+          LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
+          LANGFUSE_INIT_USER_NAME: "Demo User"
+          LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run tests
         run: pnpm --filter=web run test
 
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -152,6 +232,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -172,12 +253,12 @@ jobs:
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status
@@ -211,6 +292,7 @@ jobs:
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -243,6 +325,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -258,10 +341,14 @@ jobs:
       - name: install dependencies
         run: |
           pnpm install
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
@@ -269,14 +356,29 @@ jobs:
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed:examples
       - name: Build
         run: pnpm run build
       - name: Run server
         run: (pnpm run start&)
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e:server
 
@@ -286,11 +388,12 @@ jobs:
     needs:
       [
         lint,
-        tests-web,
+        tests-web-sync,
         tests-worker,
         e2e-tests,
         test-docker-build,
         e2e-server-tests,
+        tests-web-async,
       ]
     if: always()
     steps:
@@ -349,13 +452,16 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
@@ -374,13 +480,16 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse-worker # GitHub
             langfuse/langfuse-worker # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.github/workflows/snyk.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   snyk:
@@ -15,13 +20,13 @@ jobs:
       - name: Build a Docker image
         run: docker compose -f docker-compose.build.yml up -d
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
         continue-on-error: true
         uses: snyk/actions/docker@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-server
+          image: langfuse-langfuse-web
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
@@ -36,7 +41,7 @@ jobs:
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-worker
+          image: langfuse-langfuse-worker
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning

.gitignore

@@ -37,8 +37,9 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env*
 !.env.dev.example
-!.env.local.example
+!.env.dev-azure.example
 !.env.prod.example
+!.env.dev.legacy.example
 
 # vercel
 .vercel

LICENSE

@@ -2,8 +2,8 @@ Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
-- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- All content that resides under the "ee/", "web/src/ee/", and/or "worker/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Langfuse Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -42,9 +42,7 @@
 
 ## Langfuse Overview
 
-_Unmute video for voice-over_
-
-https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+[![Langfuse Overview Video](https://github.com/user-attachments/assets/3926b288-ff61-4b95-8aa1-45d041c70866)](https://langfuse.com/watch-demo)
 
 ### Develop
 
@@ -191,3 +189,54 @@ You can opt-out by setting `TELEMETRY_ENABLED=false`.
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
  </picture>
 </a>
+
+### Open Source Projects Using Langfuse
+
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents)):
+
+| Repository                                                                                                                                                                                                                                                          | Stars |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 54865 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 51531 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/131470832?s=40&v=4" width="20" height="20" alt=""> &nbsp; [lobehub](https://github.com/lobehub) / [lobe-chat](https://github.com/lobehub/lobe-chat)                                           | 49003 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 39093 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37368 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/139558948?s=40&v=4" width="20" height="20" alt=""> &nbsp; [chatchat-space](https://github.com/chatchat-space) / [Langchain-Chatchat](https://github.com/chatchat-space/Langchain-Chatchat)    | 32486 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/128289781?s=40&v=4" width="20" height="20" alt=""> &nbsp; [FlowiseAI](https://github.com/FlowiseAI) / [Flowise](https://github.com/FlowiseAI/Flowise)                                         | 32448 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26931 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/119600397?s=40&v=4" width="20" height="20" alt=""> &nbsp; [twentyhq](https://github.com/twentyhq) / [twenty](https://github.com/twentyhq/twenty)                                              | 24195 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22618 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 15151 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/179202840?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mediar-ai](https://github.com/mediar-ai) / [screenpipe](https://github.com/mediar-ai/screenpipe)                                   | 11037 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/105877416?s=40&v=4" width="20" height="20" alt=""> &nbsp; [formbricks](https://github.com/formbricks) / [formbricks](https://github.com/formbricks/formbricks)                                |  9386 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/76263028?s=40&v=4" width="20" height="20" alt=""> &nbsp; [anthropics](https://github.com/anthropics) / [courses](https://github.com/anthropics/courses)                                       |  8385 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7374 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5391 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/137907881?s=40&v=4" width="20" height="20" alt=""> &nbsp; [promptfoo](https://github.com/promptfoo) / [promptfoo](https://github.com/promptfoo/promptfoo)                                     |  4976 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/157326433?s=40&v=4" width="20" height="20" alt=""> &nbsp; [onlook-dev](https://github.com/onlook-dev) / [onlook](https://github.com/onlook-dev/onlook)                                        |  4141 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2526 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  2061 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1161 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1100 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/18422723?s=40&v=4" width="20" height="20" alt=""> &nbsp; [alishobeiri](https://github.com/alishobeiri) / [thread](https://github.com/alishobeiri/thread)                                      |  1074 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   971 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   823 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   677 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   639 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   514 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   394 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   384 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   370 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   305 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/154247157?s=40&v=4" width="20" height="20" alt=""> &nbsp; [vespperhq](https://github.com/vespperhq) / [vespper](https://github.com/vespperhq/vespper)                                         |   304 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   295 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   291 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/2357342?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dmayboroda](https://github.com/dmayboroda) / [minima](https://github.com/dmayboroda/minima)                                          |   221 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   172 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/148684274?s=40&v=4" width="20" height="20" alt=""> &nbsp; [i-am-alice](https://github.com/i-am-alice) / [3rd-devs](https://github.com/i-am-alice/3rd-devs)                                    |   148 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   138 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/104478511?s=40&v=4" width="20" height="20" alt=""> &nbsp; [babelcloud](https://github.com/babelcloud) / [LLM-RGB](https://github.com/babelcloud/LLM-RGB)                                      |   135 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/15125613?s=40&v=4" width="20" height="20" alt=""> &nbsp; [souzatharsis](https://github.com/souzatharsis) / [tamingLLMs](https://github.com/souzatharsis/tamingLLMs)                           |   129 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169401942?s=40&v=4" width="20" height="20" alt=""> &nbsp; [LibreChat-AI](https://github.com/LibreChat-AI) / [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)                      |   128 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   126 |

docker-compose.build.yml

@@ -1,34 +1,53 @@
-version: "3.5"
-
 services:
-  server:
+  langfuse-web:
     build:
       dockerfile: ./web/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: &langfuse-depends-on
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
     ports:
       - "3000:3000"
-    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+    environment: &langfuse-web-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      NEXTAUTH_SECRET: mysecret
+      SALT: mysalt
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: http://localhost:3000
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -36,26 +55,17 @@ services:
       timeout: 10s
       retries: 3
 
-  worker:
+  langfuse-worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: *langfuse-depends-on
     ports:
       - "3030:3030"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - PORT=${PORT:-3030}
-      - REDIS_HOST=${REDIS_HOST:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      <<: *langfuse-web-env
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
@@ -63,26 +73,85 @@ services:
       timeout: 10s
       retries: 3
 
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
-    image: redis:7.2.4
+    image: redis:7
+    restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
-    restart: always
     ports:
       - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.v3beta.yml

@@ -1,131 +0,0 @@
-services:
-  langfuse-worker:
-    image: langfuse/langfuse-worker:latest
-    depends_on: &langfuse-depends-on
-      postgres:
-        condition: service_healthy
-      minio:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "3030:3030"
-    environment: &langfuse-worker-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      SALT: "mysalt"
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
-      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
-      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
-      LANGFUSE_ASYNC_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_INGESTION_PROCESSING:-true}
-      LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:-true}
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
-      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
-
-  langfuse-web:
-    image: langfuse/langfuse:latest
-    depends_on: *langfuse-depends-on
-    ports:
-      - "3000:3000"
-    environment:
-      <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
-      NEXTAUTH_SECRET: mysecret
-      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
-      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
-      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
-      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
-      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
-      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
-      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-
-  clickhouse:
-    image: clickhouse/clickhouse-server
-    user: "101:101"
-    container_name: clickhouse
-    hostname: clickhouse
-    environment:
-      CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
-    volumes:
-      - langfuse_clickhouse_data:/var/lib/clickhouse
-      - langfuse_clickhouse_logs:/var/log/clickhouse-server
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    depends_on:
-      - postgres
-
-  minio:
-    image: minio/minio
-    container_name: minio
-    entrypoint: sh
-    # create the 'langfuse' bucket before starting the service
-    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
-    environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
-    ports:
-      - "9090:9000"
-      - "9091:9001"
-    volumes:
-      - langfuse_minio_data:/data
-    healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
-      interval: 1s
-      timeout: 5s
-      retries: 5
-      start_period: 1s
-
-  redis:
-    image: redis:7
-    restart: always
-    command: >
-      --requirepass ${REDIS_AUTH:-myredissecret}
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: [ 'CMD', 'redis-cli', 'ping' ]
-      interval: 3s
-      timeout: 10s
-      retries: 10
-
-  postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
-    restart: always
-    healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
-      interval: 3s
-      timeout: 3s
-      retries: 10
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
-    ports:
-      - 5432:5432
-    volumes:
-      - langfuse_postgres_data:/var/lib/postgresql/data
-
-volumes:
-  langfuse_postgres_data:
-    driver: local
-  langfuse_clickhouse_data:
-    driver: local
-  langfuse_clickhouse_logs:
-    driver: local
-  langfuse_minio_data:
-    driver: local

docker-compose.yml

@@ -1,31 +1,131 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:3
+    restart: always
+    depends_on: &langfuse-depends-on
+      postgres:
         condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:3
+    restart: always
+    depends_on: *langfuse-depends-on
     ports:
       - "3000:3000"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
 
-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    restart: always
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -33,14 +133,20 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

ee/package.json

@@ -28,7 +28,7 @@
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
     "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +47,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
   }
 }

ee/src/env.ts

@@ -1,12 +1,9 @@
 import { z } from "zod";
-import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
+import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = {
-  ...sharedEnv,
-  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
-};
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

fern/apis/server/definition/comments.yml

@@ -58,7 +58,7 @@ types:
         docs: The id of the object to attach the comment to. If this does not reference a valid existing object, an error will be thrown.
       content:
         type: string
-        docs: The content of the comment. May include markdown. Currently limited to 500 characters.
+        docs: The content of the comment. May include markdown. Currently limited to 3000 characters.
       authorUserId:
         type: optional<string>
         docs: The id of the user who created the comment.

fern/apis/server/definition/commons.yml

@@ -126,7 +126,7 @@ types:
         docs: The output data of the observation
       usage:
         type: optional<Usage>
-        docs: The usage data of the observation
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The usage data of the observation
       level:
         type: ObservationLevel
         docs: The level of the observation
@@ -139,6 +139,12 @@ types:
       promptId:
         type: optional<string>
         docs: The prompt ID associated with the observation
+      usageDetails:
+        type: optional<map<string, integer>>
+        docs: The usage details of the observation. Key is the name of the usage metric, value is the number of units consumed. The total key is the sum of all (non-total) usage metrics or the total value ingested.
+      costDetails:
+        type: optional<map<string, double>>
+        docs: The cost details of the observation. Key is the name of the cost metric, value is the cost in USD. The total key is the sum of all (non-total) cost metrics or the total value ingested.
 
   ObservationsView:
     extends: Observation
@@ -163,13 +169,13 @@ types:
         docs: The total price in USD.
       calculatedInputCost:
         type: optional<double>
-        docs: The calculated cost of the input in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the input in USD
       calculatedOutputCost:
         type: optional<double>
-        docs: The calculated cost of the output in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the output in USD
       calculatedTotalCost:
         type: optional<double>
-        docs: The calculated total cost in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated total cost in USD
       latency:
         type: optional<double>
         docs: The latency in seconds.
@@ -178,7 +184,7 @@ types:
         docs: The time to the first token in seconds
 
   Usage:
-    docs: Standard interface for usage and cost
+    docs: (Deprecated. Use usageDetails and costDetails instead.) Standard interface for usage and cost
     properties:
       input:
         docs: Number of input units (e.g. tokens)

fern/apis/server/definition/ingestion.yml

@@ -126,6 +126,8 @@ types:
       model: optional<string>
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptName: optional<string>
       promptVersion: optional<integer>
 
@@ -137,6 +139,8 @@ types:
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
       promptName: optional<string>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptVersion: optional<integer>
 
   ObservationBody:
@@ -312,3 +316,17 @@ types:
     properties:
       successes: list<IngestionSuccess>
       errors: list<IngestionError>
+
+  OpenAIUsageSchema:
+    properties:
+      prompt_tokens: integer
+      completion_tokens: integer
+      total_tokens: integer
+      prompt_tokens_details: optional<map<string, integer>>
+      completion_tokens_details: optional<map<string, integer>>
+
+  UsageDetails:
+    discriminated: false
+    union:
+      - map<string, integer>
+      - OpenAIUsageSchema

fern/apis/server/definition/media.yml

@@ -0,0 +1,161 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get a media record
+      method: GET
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      response: GetMediaResponse
+
+    patch:
+      docs: Patch a media record
+      method: PATCH
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      request: PatchMediaBody
+
+    getUploadUrl:
+      docs: Get a presigned upload URL for a media record
+      method: POST
+      path: /media
+      request: GetMediaUploadUrlRequest
+      response: GetMediaUploadUrlResponse
+
+types:
+  GetMediaResponse:
+    properties:
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+      contentType:
+        type: string
+        docs: The MIME type of the media record
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      url:
+        type: string
+        docs: The download URL of the media record
+      urlExpiry:
+        type: string
+        docs: The expiry date and time of the media record download URL
+
+  PatchMediaBody:
+    properties:
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      uploadHttpStatus:
+        type: integer
+        docs: The HTTP status code of the upload
+      uploadHttpError:
+        type: optional<string>
+        docs: The HTTP error message of the upload
+      uploadTimeMs:
+        type: optional<integer>
+        docs: The time in milliseconds it took to upload the media record
+
+  GetMediaUploadUrlRequest:
+    properties:
+      traceId:
+        type: string
+        docs: The trace ID associated with the media record
+      observationId:
+        type: optional<string>
+        docs: The observation ID associated with the media record. If the media record is associated directly with a trace, this will be null.
+      contentType: MediaContentType
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      sha256Hash:
+        type: string
+        docs: The SHA-256 hash of the media record
+      field:
+        type: string
+        docs: The trace / observation field the media record is associated with. This can be one of `input`, `output`, `metadata`
+
+  GetMediaUploadUrlResponse:
+    properties:
+      uploadUrl:
+        type: optional<string>
+        docs: The presigned upload URL. If the asset is already uploaded, this will be null
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+
+  MediaContentType:
+    enum:
+      - value: image/png
+        name: IMAGE_PNG
+      - value: image/jpeg
+        name: IMAGE_JPEG
+      - value: image/jpg
+        name: IMAGE_JPG
+      - value: image/webp
+        name: IMAGE_WEBP
+      - value: image/gif
+        name: IMAGE_GIF
+      - value: image/svg+xml
+        name: IMAGE_SVG_XML
+      - value: image/tiff
+        name: IMAGE_TIFF
+      - value: image/bmp
+        name: IMAGE_BMP
+      - value: audio/mpeg
+        name: AUDIO_MPEG
+      - value: audio/mp3
+        name: AUDIO_MP3
+      - value: audio/wav
+        name: AUDIO_WAV
+      - value: audio/ogg
+        name: AUDIO_OGG
+      - value: audio/oga
+        name: AUDIO_OGA
+      - value: audio/aac
+        name: AUDIO_AAC
+      - value: audio/mp4
+        name: AUDIO_MP4
+      - value: audio/flac
+        name: AUDIO_FLAC
+      - value: video/mp4
+        name: VIDEO_MP4
+      - value: video/webm
+        name: VIDEO_WEBM
+      - value: text/plain
+        name: TEXT_PLAIN
+      - value: text/html
+        name: TEXT_HTML
+      - value: text/css
+        name: TEXT_CSS
+      - value: text/csv
+        name: TEXT_CSV
+      - value: application/pdf
+        name: APPLICATION_PDF
+      - value: application/msword
+        name: APPLICATION_MSWORD
+      - value: application/vnd.ms-excel
+        name: APPLICATION_MS_EXCEL
+      - value: application/zip
+        name: APPLICATION_ZIP
+      - value: application/json
+        name: APPLICATION_JSON
+      - value: application/xml
+        name: APPLICATION_XML
+      - value: application/octet-stream
+        name: APPLICATION_OCTET_STREAM
+    docs: The MIME type of the media record

fern/apis/server/definition/score.yml

@@ -147,10 +147,29 @@ types:
       tags:
         type: optional<list<string>>
         docs: A list of tags associated with the trace referenced by score
-  GetScoresResponseData:
+
+  GetScoresResponseDataNumeric:
+    extends: commons.NumericScore
     properties:
-      <<: commons.Score
       trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataCategorical:
+    extends: commons.CategoricalScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataBoolean:
+    extends: commons.BooleanScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseData:
+    discriminant: dataType
+    union:
+      NUMERIC: GetScoresResponseDataNumeric
+      CATEGORICAL: GetScoresResponseDataCategorical
+      BOOLEAN: GetScoresResponseDataBoolean
+
   GetScoresResponse:
     properties:
       data: list<GetScoresResponseData>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.89.0",
+  "version": "3.3.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -81,10 +81,10 @@
       }
     }
   },
+  "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
-  },
-  "packageManager": "pnpm@9.5.0"
+  }
 }

packages/shared/clickhouse/migrations/clustered/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0001_traces.up.sql

@@ -0,0 +1,32 @@
+CREATE TABLE traces ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(LowCardinality(String), String),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    updated_at DateTime64(3) DEFAULT now(),
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+     project_id,
+     toDate(timestamp)
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/clustered/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0002_observations.up.sql

@@ -0,0 +1,47 @@
+CREATE TABLE observations ON CLUSTER default (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_usage_details` Map(LowCardinality(String), UInt64),
+    `usage_details` Map(LowCardinality(String), UInt64),
+    `provided_cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
+PRIMARY KEY (
+    project_id,
+    `type`,
+    toDate(start_time)
+)
+ORDER BY (
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
+

packages/shared/clickhouse/migrations/clustered/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0003_scores.up.sql

@@ -0,0 +1,33 @@
+CREATE TABLE scores ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `queue_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+    project_id,
+    toDate(timestamp),
+    name
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_id project_id TYPE bloom_filter() GRANULARITY 1;

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.up.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_project_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0001_traces.up.sql

@@ -22,11 +22,11 @@ CREATE TABLE traces (
     INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
 PRIMARY KEY (
-        project_id,
-        toDate(timestamp)
+     project_id,
+     toDate(timestamp)
 )
 ORDER BY (
-        project_id,
-        toDate(timestamp),
-        id
-    );
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/unclustered/0002_observations.up.sql

@@ -34,14 +34,14 @@ CREATE TABLE observations (
     INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
 PRIMARY KEY (
-        project_id,
-        `type`,
-        toDate(start_time)
-    )
+    project_id,
+    `type`,
+    toDate(start_time)
+)
 ORDER BY (
-        project_id,
-        `type`,
-        toDate(start_time),
-        id
-    );
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
 

packages/shared/clickhouse/migrations/unclustered/0003_scores.up.sql

@@ -21,13 +21,13 @@ CREATE TABLE scores (
     INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
 PRIMARY KEY (
-        project_id,
-        toDate(timestamp),
-        name
-    )
+    project_id,
+    toDate(timestamp),
+    name
+)
 ORDER BY (
-        project_id,
-        toDate(timestamp),
-        name,
-        id
-    )
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/down.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,10 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
-# Execute the down command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
+fi

packages/shared/clickhouse/scripts/seed.ts

@@ -1,21 +1,71 @@
-import { clickhouseClient } from "@langfuse/shared/src/server";
+import {
+  clickhouseClient,
+  ObservationRecordReadType,
+} from "@langfuse/shared/src/server";
 import { prisma } from "../../src/db";
 import { redis } from "@langfuse/shared/src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
+import { createDatasets } from "../../prisma/seed";
+import { queryClickhouse } from "../../src/server/repositories/clickhouse";
+import { convertObservation } from "../../src/server/repositories/observations_converters";
 
 async function main() {
   try {
     const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
+    if (
+      await prisma.project.findFirst({
+        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      })
+    ) {
+      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
+    }
     await prepareClickhouse(projectIds, {
       numberOfDays: 3,
-      totalObservations: 1000,
+      totalObservations: 10000,
     });
 
+    const project1 = await prisma.project.findFirst({
+      where: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const project2 =
+      projectIds.length > 1
+        ? await prisma.project.findFirst({
+            where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+          })
+        : await prisma.project.findFirst();
+
+    const query = `
+        SELECT *
+        FROM observations o
+        WHERE o.project_id IN ({projectIds: Array(String)})
+        LIMIT 2000;
+      `;
+
+    const res = await queryClickhouse<ObservationRecordReadType>({
+      query,
+      params: {
+        projectIds,
+      },
+    });
+
+    await createDatasets(
+      project1!,
+      project2!,
+      (await Promise.all(res.map(convertObservation))).map((o) => ({
+        ...o,
+        metadata: {},
+        modelParameters: {},
+        input: {},
+        output: {},
+      })),
+    );
+
     console.log("Clickhouse preparation completed successfully.");
   } catch (error) {
     console.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     console.log("Disconnected from Clickhouse.");

packages/shared/clickhouse/scripts/up.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,11 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
-else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
 
-# Execute the up command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+else
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
+fi

packages/shared/package.json

@@ -43,7 +43,6 @@
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
     "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
-    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
     "ch:up": "bash clickhouse/scripts/up.sh",
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
@@ -60,13 +59,15 @@
     "@aws-sdk/client-s3": "^3.675.0",
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.1",
-    "@langchain/aws": "^0.1.0",
-    "@langchain/core": "^0.3.9",
-    "@langchain/openai": "^0.3.0",
+    "@langchain/anthropic": "^0.3.8",
+    "@langchain/aws": "^0.1.2",
+    "@langchain/core": "^0.3.18",
+    "@langchain/google-vertexai": "^0.1.3",
+    "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^5.20.0",
+    "@prisma/client": "^5.22.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
@@ -78,9 +79,10 @@
     "exponential-backoff": "^3.1.1",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.2",
+    "langchain": "^0.3.6",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
@@ -104,7 +106,7 @@
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
     "prettier": "^3.3.3",
-    "prisma": "^5.20.0",
+    "prisma": "^5.22.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
@@ -119,7 +121,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -278,6 +278,8 @@ export type JobExecution = {
     end_time: Timestamp | null;
     error: string | null;
     job_input_trace_id: string | null;
+    job_input_observation_id: string | null;
+    job_input_dataset_item_id: string | null;
     job_output_score_id: string | null;
 };
 export type LlmApiKeys = {
@@ -294,6 +296,20 @@ export type LlmApiKeys = {
     config: unknown | null;
     project_id: string;
 };
+export type Media = {
+    id: string;
+    sha_256_hash: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    uploaded_at: Timestamp | null;
+    upload_http_status: number | null;
+    upload_http_error: string | null;
+    bucket_path: string;
+    bucket_name: string;
+    content_type: string;
+    content_length: string;
+};
 export type MembershipInvitation = {
     id: string;
     email: string;
@@ -354,6 +370,16 @@ export type Observation = {
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
+export type ObservationMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    observation_id: string;
+    field: string;
+};
 export type ObservationView = {
     id: string;
     trace_id: string | null;
@@ -427,6 +453,7 @@ export type Project = {
     org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    deleted_at: Timestamp | null;
     name: string;
 };
 export type ProjectMembership = {
@@ -515,6 +542,15 @@ export type Trace = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type TraceMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    field: string;
+};
 export type TraceSession = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -579,8 +615,10 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
+    observation_media: ObservationMedia;
     observations: Observation;
     observations_view: ObservationView;
     organization_memberships: OrganizationMembership;
@@ -594,6 +632,7 @@ export type DB = {
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;
+    trace_media: TraceMedia;
     trace_sessions: TraceSession;
     traces: Trace;
     traces_view: TraceView;

packages/shared/prisma/migrations/20241106122605_add_media_tables/migration.sql

@@ -0,0 +1,71 @@
+-- CreateTable
+CREATE TABLE "media" (
+    "id" TEXT NOT NULL,
+    "sha_256_hash" CHAR(44) NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "uploaded_at" TIMESTAMP(3),
+    "upload_http_status" INTEGER,
+    "upload_http_error" TEXT,
+    "bucket_path" TEXT NOT NULL,
+    "bucket_name" TEXT NOT NULL,
+    "content_type" TEXT NOT NULL,
+    "content_length" BIGINT NOT NULL,
+
+    CONSTRAINT "media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "trace_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "trace_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "observation_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "observation_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "observation_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "media_project_id_sha_256_hash_key" ON "media"("project_id", "sha_256_hash");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "trace_media_project_id_trace_id_media_id_field_key" ON "trace_media"("project_id", "trace_id", "media_id", "field");
+
+-- CreateIndex
+CREATE INDEX "observation_media_project_id_observation_id_idx" ON "observation_media"("project_id", "observation_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "observation_media_project_id_trace_id_observation_id_media__key" ON "observation_media"("project_id", "trace_id", "observation_id", "media_id", "field");
+
+-- AddForeignKey
+ALTER TABLE "media" ADD CONSTRAINT "media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241114175010_job_executions_add_observation_dataset_item_cols/migration.sql

@@ -0,0 +1,6 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_input_trace_id_fkey";
+
+-- AlterTable
+ALTER TABLE "job_executions" ADD COLUMN     "job_input_dataset_item_id" TEXT,
+ADD COLUMN     "job_input_observation_id" TEXT;

packages/shared/prisma/migrations/20241124115100_add_projects_deleted_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN "deleted_at" TIMESTAMP(3);

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/prisma/migrations/20241206115829_remove_trace_score_observation_constraints/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_output_score_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/schema.prisma

@@ -113,6 +113,7 @@ model Project {
   orgId               String                 @map("org_id")
   createdAt           DateTime               @default(now()) @map("created_at")
   updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+  deletedAt           DateTime?              @map("deleted_at")
   name                String
   projectMembers      ProjectMembership[]
   organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
@@ -136,6 +137,9 @@ model Project {
   comment             Comment[]
   annotationQueue     AnnotationQueue[]
   annotationQueueItem AnnotationQueueItem[]
+  TraceMedia          TraceMedia[]
+  Media               Media[]
+  ObservationMedia    ObservationMedia[]
 
   @@index([orgId])
   @@map("projects")
@@ -300,8 +304,6 @@ model Trace {
   createdAt  DateTime      @default(now()) @map("created_at")
   updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-  JobExecution JobExecution[]
-
   @@index([projectId, timestamp])
   @@index([sessionId])
   @@index([name])
@@ -467,25 +469,24 @@ enum ObservationLevel {
 }
 
 model Score {
-  id            String         @id @default(cuid())
-  timestamp     DateTime       @default(now())
-  projectId     String         @map("project_id")
-  project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  id            String        @id @default(cuid())
+  timestamp     DateTime      @default(now())
+  projectId     String        @map("project_id")
+  project       Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        ScoreSource
-  authorUserId  String?        @map("author_user_id")
+  authorUserId  String?       @map("author_user_id")
   comment       String?
-  traceId       String         @map("trace_id")
-  observationId String?        @map("observation_id")
-  configId      String?        @map("config_id")
-  stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
-  queueId       String?        @map("queue_id")
-  createdAt     DateTime       @default(now()) @map("created_at")
-  updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
-  dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
-  JobExecution  JobExecution[]
-  scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
+  traceId       String        @map("trace_id")
+  observationId String?       @map("observation_id")
+  configId      String?       @map("config_id")
+  stringValue   String?       @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+  queueId       String?       @map("queue_id")
+  createdAt     DateTime      @default(now()) @map("created_at")
+  updatedAt     DateTime      @default(now()) @updatedAt @map("updated_at")
+  dataType      ScoreDataType @default(NUMERIC) @map("data_type")
+  scoreConfig   ScoreConfig?  @relation(fields: [configId], references: [id], onDelete: SetNull)
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -895,11 +896,13 @@ model JobExecution {
   endTime   DateTime?          @map("end_time")
   error     String?
 
-  jobInputTraceId String? @map("job_input_trace_id")
-  trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+  jobInputTraceId String? @map("job_input_trace_id") // no fk constraint - traces in ClickHouse, deletion handled via project cascade
+
+  jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
+
+  jobInputDatasetItemId String? @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
-  score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
   @@index([projectId, status])
   @@index([projectId, id])
@@ -961,3 +964,56 @@ model BatchExport {
   @@index([status])
   @@map("batch_exports")
 }
+
+model Media {
+  id               String             @id @default(cuid())
+  sha256Hash       String             @map("sha_256_hash") @db.Char(44)
+  projectId        String             @map("project_id")
+  project          Project            @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt        DateTime           @default(now()) @map("created_at")
+  updatedAt        DateTime           @default(now()) @updatedAt @map("updated_at")
+  uploadedAt       DateTime?          @map("uploaded_at")
+  uploadHttpStatus Int?               @map("upload_http_status")
+  uploadHttpError  String?            @map("upload_http_error")
+  bucketPath       String             @map("bucket_path")
+  bucketName       String             @map("bucket_name")
+  contentType      String             @map("content_type")
+  contentLength    BigInt             @map("content_length")
+  TraceMedia       TraceMedia[]
+  ObservationMedia ObservationMedia[]
+
+  @@unique([projectId, sha256Hash])
+  @@map("media")
+}
+
+model TraceMedia {
+  id        String   @id @default(cuid())
+  projectId String   @map("project_id")
+  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId   String   @map("media_id")
+  media     Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId   String   @map("trace_id")
+  field     String   @map("field")
+
+  @@unique([projectId, traceId, mediaId, field])
+  @@map("trace_media")
+}
+
+model ObservationMedia {
+  id            String   @id @default(cuid())
+  projectId     String   @map("project_id")
+  project       Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt     DateTime @default(now()) @map("created_at")
+  updatedAt     DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId       String   @map("media_id")
+  media         Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId       String   @map("trace_id")
+  observationId String   @map("observation_id")
+  field         String   @map("field")
+
+  @@unique([projectId, traceId, observationId, mediaId, field])
+  @@index([projectId, observationId])
+  @@map("observation_media")
+}

packages/shared/prisma/seed.ts

@@ -387,16 +387,62 @@ async function main() {
       update: {},
     });
 
-    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
-      const dataset = await prisma.dataset.create({
-        data: {
-          name: `demo-dataset-${datasetNumber}`,
-          description:
-            datasetNumber === 0 ? "Dataset test description" : undefined,
-          projectId: project2.id,
-          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
-        },
-      });
+    await createDatasets(project1, project2, observations);
+  }
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+  })
+  .catch(async (e) => {
+    logger.error(e);
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+    process.exit(1);
+  });
+
+export async function createDatasets(
+  project1: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  project2: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  observations: Prisma.ObservationCreateManyInput[],
+) {
+  for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
+    for (const projectId of [project1.id, project2.id]) {
+      const datasetName = `demo-dataset-${datasetNumber}`;
+
+      // check if ds already exists
+      const dataset =
+        (await prisma.dataset.findFirst({
+          where: {
+            projectId,
+            name: datasetName,
+          },
+        })) ??
+        (await prisma.dataset.create({
+          data: {
+            name: datasetName,
+            description:
+              datasetNumber === 0 ? "Dataset test description" : undefined,
+            projectId,
+            metadata: datasetNumber === 0 ? { key: "value" } : undefined,
+          },
+        }));
 
       const datasetItemIds = [];
       for (let i = 0; i < 18; i++) {
@@ -404,9 +450,12 @@ async function main() {
           Math.random() > 0.3
             ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
+        if (!sourceObservation) {
+          continue;
+        }
         const datasetItem = await prisma.datasetItem.create({
           data: {
-            projectId: project2.id,
+            projectId,
             datasetId: dataset.id,
             sourceTraceId: sourceObservation?.traceId,
             sourceObservationId:
@@ -431,9 +480,16 @@ async function main() {
       }
 
       for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
-        const datasetRun = await prisma.datasetRuns.create({
-          data: {
-            projectId: project2.id,
+        const datasetRun = await prisma.datasetRuns.upsert({
+          where: {
+            datasetId_projectId_name: {
+              datasetId: dataset.id,
+              projectId,
+              name: `demo-dataset-run-${datasetRunNumber}`,
+            },
+          },
+          create: {
+            projectId,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
@@ -445,20 +501,24 @@ async function main() {
               ["tag1", "tag2"],
             ][datasetRunNumber % 5],
           },
+          update: {},
         });
 
         for (const datasetItemId of datasetItemIds) {
           const relevantObservations = observations.filter(
-            (o) => o.projectId === project2.id,
+            (o) => o.projectId === projectId,
           );
           const observation =
             relevantObservations[
               Math.floor(Math.random() * relevantObservations.length)
             ];
 
+          if (!observation) {
+            continue;
+          }
           await prisma.datasetRunItems.create({
             data: {
-              projectId: project2.id,
+              projectId,
               datasetItemId,
               traceId: observation.traceId as string,
               observationId: Math.random() > 0.5 ? observation.id : undefined,
@@ -471,20 +531,6 @@ async function main() {
   }
 }
 
-main()
-  .then(async () => {
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-  })
-  .catch(async (e) => {
-    logger.error(e);
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-    process.exit(1);
-  });
-
 async function uploadObjects(
   traces: Prisma.TraceCreateManyInput[],
   observations: Prisma.ObservationCreateManyInput[],
@@ -1012,60 +1058,59 @@ async function generatePromptsForProject(projects: Project[]) {
   return promptIds;
 }
 
-async function generatePrompts(project: Project) {
-  const promptIds: string[] = [];
-  const prompts = [
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 1 content",
-      name: "Prompt 1",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 2 content",
-      name: "Prompt 2",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "API",
-      prompt: "Prompt 3 content",
-      name: "Prompt 3 by API",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 4 content",
-      name: "Prompt 4",
-      version: 1,
-      labels: ["production", "latest"],
-      tags: ["tag1", "tag2"],
-    },
-  ];
+export const SEED_PROMPTS = [
+  {
+    id: `prompt-123`,
+    createdBy: "user-1",
+    prompt: "Prompt 1 content",
+    name: "Prompt 1",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-456`,
+    createdBy: "user-1",
+    prompt: "Prompt 2 content",
+    name: "Prompt 2",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-789`,
+    createdBy: "API",
+    prompt: "Prompt 3 content",
+    name: "Prompt 3 by API",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: "Prompt 4 content",
+    name: "Prompt 4",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
 
-  for (const prompt of prompts) {
+export const PROMPT_IDS: string[] = [];
+
+async function generatePrompts(project: Project) {
+  const promptIds = [];
+  for (const prompt of SEED_PROMPTS) {
     await prisma.prompt.upsert({
       where: {
         projectId_name_version: {
-          projectId: prompt.projectId,
+          projectId: prompt.id + project.id,
           name: prompt.name,
           version: prompt.version,
         },
+        id: prompt.id + project.id,
       },
       create: {
-        id: prompt.id,
-        projectId: prompt.projectId,
+        id: prompt.id + project.id,
+        projectId: project.id,
         createdBy: prompt.createdBy,
         prompt: prompt.prompt,
         name: prompt.name,
@@ -1073,9 +1118,7 @@ async function generatePrompts(project: Project) {
         labels: prompt.labels,
         tags: prompt.tags,
       },
-      update: {
-        id: prompt.id,
-      },
+      update: {},
     });
     promptIds.push(prompt.id);
   }
@@ -1129,6 +1172,7 @@ async function generatePrompts(project: Project) {
           name: version.name,
           version: version.version,
         },
+        id: version.id,
       },
       create: {
         id: version.id,
@@ -1159,6 +1203,7 @@ async function generatePrompts(project: Project) {
           name: promptName,
           version: i,
         },
+        id: promptId,
       },
       create: {
         id: promptId,

packages/shared/scripts/load-seed.ts

@@ -105,10 +105,6 @@ async function main() {
     numberOfDays = 3;
   }
 
-  logger.info(
-    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`,
-  );
-
   try {
     const projectIds = [
       "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
@@ -128,7 +124,7 @@ async function main() {
   } catch (error) {
     logger.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     logger.info("Disconnected from Clickhouse.");

packages/shared/scripts/prepareClickhouse.ts

@@ -1,3 +1,4 @@
+import { SEED_PROMPTS } from "../prisma/seed";
 import { prisma } from "../src/db";
 import { clickhouseClient, logger } from "../src/server";
 
@@ -109,10 +110,16 @@ export const prepareClickhouse = async (
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
       toDecimal64(randUniform(0, 2000), 12) AS total_cost,
-      start_time AS completion_start_time,
-      toString(rand()) AS prompt_id,
-      toString(rand()) AS prompt_name,
-      1000 AS prompt_version,
+      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
+      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,
+      array(${SEED_PROMPTS.map((p) => `'${p.name}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_name,
+      array(${SEED_PROMPTS.map((p) => `'${p.version}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_version,
       start_time AS created_at,
       start_time AS updated_at,
       start_time AS event_ts,
@@ -120,6 +127,8 @@ export const prepareClickhouse = async (
     FROM numbers(${observationsPerProject});
   `;
 
+    console.log(observationsQuery);
+
     const scoresQuery = `
     INSERT INTO scores
     SELECT toString(floor(randUniform(0, 100))) AS id,
@@ -151,7 +160,7 @@ export const prepareClickhouse = async (
 
     for (const query of queries) {
       logger.info(`Executing query: ${query}`);
-      await clickhouseClient.command({
+      await clickhouseClient().command({
         query,
         clickhouse_settings: {
           wait_end_of_query: 1,
@@ -165,7 +174,7 @@ export const prepareClickhouse = async (
       FROM traces 
       WHERE session_id IS NOT NULL;
     `;
-    const sessionResult = await clickhouseClient.query({
+    const sessionResult = await clickhouseClient().query({
       query: sessionQuery,
       format: "JSONEachRow",
     });
@@ -203,7 +212,7 @@ export const prepareClickhouse = async (
           ORDER BY count() desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });
@@ -234,7 +243,7 @@ export const prepareClickhouse = async (
           ORDER BY event_date desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });

packages/shared/src/env.ts

@@ -5,6 +5,7 @@ const EnvSchema = z.object({
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
+  NEXTAUTH_URL: z.string().url().optional(),
   REDIS_HOST: z.string().nullish(),
   REDIS_PORT: z.coerce
     .number({
@@ -27,9 +28,23 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
-  CLICKHOUSE_URL: z.string().url().optional(),
-  CLICKHOUSE_USER: z.string().optional(),
-  CLICKHOUSE_PASSWORD: z.string().optional(),
+  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_USER: z.string(),
+  CLICKHOUSE_PASSWORD: z.string(),
+  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("true"),
+  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+
+  LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(15_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -38,8 +53,9 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
+    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
+  }),
   LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
   LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
   LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
@@ -48,6 +64,11 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
 });
 
-export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));
+export const env: z.infer<typeof EnvSchema> =
+  process.env.DOCKER_BUILD === "1"
+    ? (process.env as any)
+    : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/comments/types.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+const MAX_COMMENT_LENGTH = 3000;
+
 const COMMENT_OBJECT_TYPES = [
   "TRACE",
   "OBSERVATION",
@@ -9,7 +11,7 @@ const COMMENT_OBJECT_TYPES = [
 
 export const CreateCommentData = z.object({
   projectId: z.string(),
-  content: z.string().trim().min(1).max(500),
+  content: z.string().trim().min(1).max(MAX_COMMENT_LENGTH),
   objectId: z.string(),
   objectType: z.enum(COMMENT_OBJECT_TYPES),
 });

packages/shared/src/features/entitlements/plans.ts

@@ -3,15 +3,17 @@ export const planLabels = {
   "cloud:hobby": "Hobby",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
-  "self-hosted:enterprise": "Enterprise",
+  "self-hosted:pro": "Pro (self-hosted)",
+  "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
 
 export type Plan = keyof typeof planLabels;
 
 export const plans = Object.keys(planLabels) as Plan[];
 
-// This function is kept here to ensure consistency when updating plan names in the future.
+// These functions are kept here to ensure consistency when updating plan names in the future.
 export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/features/evals/types.ts

@@ -5,6 +5,7 @@ export const langfuseObjects = [
   "span",
   "generation",
   "event",
+  "dataset_item",
 ] as const;
 
 // variable mapping stored in the db for eval templates
@@ -21,7 +22,7 @@ export const variableMapping = z
     (value) => value.langfuseObject === "trace" || value.objectName !== null,
     {
       message: "objectName is required for langfuseObjects other than trace",
-    }
+    },
   );
 
 export const variableMappingList = z.array(variableMapping);
@@ -44,7 +45,7 @@ const observationCols = [
   { name: "Output", id: "output", internal: 'o."output"' },
 ];
 
-export const availableEvalVariables = [
+export const availableTraceEvalVariables = [
   {
     id: "trace",
     display: "Trace",
@@ -76,6 +77,28 @@ export const availableEvalVariables = [
   },
 ];
 
+export const availableDatasetEvalVariables = [
+  {
+    id: "dataset_item",
+    display: "Dataset item",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'd."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'd."input"' },
+      {
+        name: "Expected output",
+        id: "expected_output",
+        internal: 'd."expected_output"',
+      },
+    ],
+  },
+  ...availableTraceEvalVariables,
+];
+
 export const OutputSchema = z.object({
   reasoning: z.string(),
   score: z.string(),
@@ -83,6 +106,7 @@ export const OutputSchema = z.object({
 
 export enum EvalTargetObject {
   Trace = "trace",
+  Dataset = "dataset",
 }
 
 export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/features/experiments/utils.ts

@@ -0,0 +1,15 @@
+import { Prisma } from "../../db";
+
+export const datasetItemMatchesVariable = (
+  input: Prisma.JsonValue,
+  variable: string,
+) => {
+  if (
+    input === null ||
+    input === undefined ||
+    typeof input !== "object" ||
+    Array.isArray(input)
+  )
+    return false;
+  return Object.keys(input).includes(variable);
+};

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -6,7 +6,7 @@ import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 
 /**
@@ -17,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx
+  ctx: z.RefinementCtx,
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -94,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value
+          category.value === expectedCategories[index].value,
       );
     }),
 });
@@ -123,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      })
+      }),
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
@@ -170,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType
+  scoreConfig: ScoreConfigDbType,
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -205,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      })
+      }),
     ),
   ])
   .superRefine((data, ctx) => {
@@ -227,7 +227,7 @@ export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
 
 // GET /score-configs
 export const GetScoreConfigsQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
 });
 
 export const GetScoreConfigsResponse = z.object({

packages/shared/src/features/scores/scoreTypes.ts

@@ -6,7 +6,7 @@ import { isPresent, stringDateTime } from "../../utils/typeChecks";
 import {
   NonEmptyString,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 import { Category as ConfigCategory } from "./scoreConfigTypes";
 
@@ -84,13 +84,13 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
     z.object({
       value: z.number(),
       dataType: z.literal("NUMERIC"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -98,7 +98,7 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
         message: "Value must be either 0 or 1",
       }),
       dataType: z.literal("BOOLEAN"),
-    })
+    }),
   ),
 ]);
 
@@ -162,7 +162,7 @@ export const ScorePropsAgainstConfig = z.union([
  */
 export const filterAndValidateDbScoreList = (
   scores: Score[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): APIScore[] =>
   scores.reduce((acc, ts) => {
     const result = APIScoreSchema.safeParse(ts);
@@ -199,14 +199,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       value: z.number(),
       dataType: z.literal("NUMERIC"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -216,14 +216,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       }),
       dataType: z.literal("BOOLEAN"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.union([z.string(), z.number()]),
       dataType: z.undefined(),
       configId: z.string().nullish(),
-    })
+    }),
   ),
 ]);
 
@@ -231,7 +231,7 @@ export const PostScoresResponse = z.object({ id: z.string() });
 
 // GET /scores
 export const GetScoresQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
   userId: z.string().nullish(),
   dataType: z.enum(ScoreDataType).nullish(),
   configId: z.string().nullish(),
@@ -260,7 +260,7 @@ const LegacyGetScoreResponseDataV1 = z.intersection(
       userId: z.string().nullish(),
       tags: z.array(z.string()).nullish(),
     }),
-  })
+  }),
 );
 export const GetScoresResponse = z.object({
   data: z.array(LegacyGetScoreResponseDataV1),
@@ -269,7 +269,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
     (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
@@ -282,7 +282,7 @@ export const legacyFilterAndValidateV1GetScoreList = (
       }
       return acc;
     },
-    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[]
+    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[],
   );
 
 // GET /scores/{scoreId}

packages/shared/src/index.ts

@@ -7,15 +7,14 @@ export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
 export * from "./tableDefinitions/tracesTable";
-export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
-export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";
@@ -33,6 +32,9 @@ export * from "./features/scores";
 // comments
 export * from "./features/comments/types";
 
+// experiments
+export * from "./features/experiments/utils";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -10,3 +10,19 @@ export const BedrockCredentialSchema = z
   })
   .optional();
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
+
+export const GCPServiceAccountKeySchema = z.object({
+  type: z.literal("service_account"),
+  project_id: z.string(),
+  private_key_id: z.string(),
+  private_key: z.string(),
+  client_email: z.string(),
+  client_id: z.string(),
+  auth_uri: z.string(),
+  token_uri: z.string(),
+  auth_provider_x509_cert_url: z.string(),
+  client_x509_cert_url: z.string(),
+});
+
+export type GCPServiceAccountKey = z.infer<typeof GCPServiceAccountKeySchema>;
+export default GCPServiceAccountKeySchema;

packages/shared/src/interfaces/filters.ts

@@ -15,6 +15,7 @@ export const filterOperators = {
   ],
   numberObject: ["=", ">", "<", ">=", "<="],
   boolean: ["=", "<>"],
+  null: ["is null", "is not null"],
 } as const;
 
 export const timeFilter = z.object({
@@ -68,6 +69,12 @@ export const booleanFilter = z.object({
   operator: z.enum(filterOperators.boolean),
   value: z.boolean(),
 });
+export const nullFilter = z.object({
+  type: z.literal("null"),
+  column: z.string(),
+  operator: z.enum(filterOperators.null),
+  value: z.literal(""),
+});
 export const singleFilter = z.discriminatedUnion("type", [
   timeFilter,
   stringFilter,
@@ -77,4 +84,5 @@ export const singleFilter = z.discriminatedUnion("type", [
   stringObjectFilter,
   numberObjectFilter,
   booleanFilter,
+  nullFilter,
 ]);

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/clickhouse/client.ts

@@ -1,24 +1,28 @@
 import { createClient } from "@clickhouse/client";
 import { env } from "../../env";
+import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
 export type ClickhouseClientType = ReturnType<typeof createClient>;
 
-export const clickhouseClient = createClient({
-  url: env.CLICKHOUSE_URL,
-  username: env.CLICKHOUSE_USER,
-  password: env.CLICKHOUSE_PASSWORD,
-  database: "default",
-  clickhouse_settings: {
-    async_insert: 1,
-    wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
-  },
-});
+export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
+  createClient({
+    ...opts,
+    url: env.CLICKHOUSE_URL,
+    username: env.CLICKHOUSE_USER,
+    password: env.CLICKHOUSE_PASSWORD,
+    database: "default",
+    clickhouse_settings: {
+      async_insert: 1,
+      wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
+    },
+  });
 
+export const defaultClickhouseClient = clickhouseClient();
 /**
  * Accepts a JavaScript date and returns the DateTime in format YYYY-MM-DD HH:MM:SS
  */
 
 export const convertDateToClickhouseDateTime = (date: Date): string => {
-  // 2024-11-06T20:37:00.000Z -> 2024-11-06 21:37:00
-  return date.toISOString().slice(0, 19).replace("T", " ");
+  // 2024-11-06T20:37:00.123Z -> 2024-11-06 21:37:00.123
+  return date.toISOString().replace("T", " ").replace("Z", "");
 };

packages/shared/src/server/filterToPrisma.ts

@@ -106,10 +106,12 @@ export function tableColumnsToSqlFilter(
           ", ",
         )}] `;
         break;
-
       case "boolean":
         valuePrisma = Prisma.sql`${filter.value}`;
         break;
+      case "null":
+        valuePrisma = Prisma.sql``;
+        break;
     }
     const jsonKeyPrisma =
       filter.type === "stringObject" || filter.type === "numberObject"

packages/shared/src/server/index.ts

@@ -1,10 +1,12 @@
-export * from "./services/S3StorageService";
+export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
+export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -15,13 +17,22 @@ export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
 export * from "./ingestion/modelMatch";
+export * from "./ingestion/processEventBatch";
 export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
+export * from "./redis/cloudUsageMeteringQueue";
+export * from "./redis/getQueue";
+export * from "./redis/traceDelete";
+export * from "./redis/projectDelete";
+export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/postHogIntegrationQueue";
+export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
 export * from "./ingestion/legacy/index";
 export * from "./queues";
@@ -32,3 +43,7 @@ export * from "./instrumentation";
 export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
+export * from "./redis/evalExecutionQueue";
+
+// test utils
+export * from "./test-utils";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -20,6 +20,9 @@ import { jsonSchema } from "../../../utils/zod";
 import { prisma } from "../../../db";
 import { LegacyIngestionAccessScope } from ".";
 import { logger } from "../../logger";
+import { env } from "../../../env";
+import { upsertTrace } from "../../repositories";
+import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
 
 export interface EventProcessor {
   auth(apiScope: LegacyIngestionAccessScope): void;
@@ -131,22 +134,9 @@ export class ObservationProcessor implements EventProcessor {
           })
         : undefined;
 
-    const traceId =
-      !this.event.body.traceId && !existingObservation
-        ? // Create trace if no traceid
-          (
-            await prisma.trace.create({
-              data: {
-                projectId: apiScope.projectId,
-                name: this.event.body.name,
-              },
-            })
-          ).id
-        : this.event.body.traceId;
-
     // Token counts
     const [newInputCount, newOutputCount] =
-      "usage" in this.event.body
+      "usage" in this.event.body || "usageDetails" in this.event.body
         ? await this.calculateTokenCounts(
             apiScope.projectId,
             this.event.body,
@@ -157,8 +147,9 @@ export class ObservationProcessor implements EventProcessor {
         : [undefined, undefined];
 
     const newTotalCount =
-      "usage" in this.event.body
+      "usage" in this.event.body || "usageDetails" in this.event.body
         ? (this.event.body.usage?.total ??
+          this.event.body.usageDetails?.total ??
           (newInputCount != null || newOutputCount != null
             ? (newInputCount ?? 0) + (newOutputCount ?? 0)
             : undefined))
@@ -168,15 +159,24 @@ export class ObservationProcessor implements EventProcessor {
       inputCost:
         "usage" in this.event.body && this.event.body.usage?.inputCost != null // inputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.inputCost)
-          : existingObservation?.inputCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.input != null
+            ? new Decimal(this.event.body.costDetails?.input)
+            : existingObservation?.inputCost,
       outputCost:
         "usage" in this.event.body && this.event.body.usage?.outputCost != null // outputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.outputCost)
-          : existingObservation?.outputCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.output != null
+            ? new Decimal(this.event.body.costDetails?.output)
+            : existingObservation?.outputCost,
       totalCost:
         "usage" in this.event.body && this.event.body.usage?.totalCost != null // totalCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.totalCost)
-          : existingObservation?.totalCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.total != null
+            ? new Decimal(this.event.body.costDetails?.total)
+            : existingObservation?.totalCost,
     };
 
     const tokenCounts = {
@@ -230,11 +230,46 @@ export class ObservationProcessor implements EventProcessor {
         return newId;
       })();
 
+    let traceId = this.event.body?.traceId;
+    if (!this.event.body.traceId && !existingObservation) {
+      // Create trace if no traceId
+      traceId = observationId;
+
+      // Insert trace into postgres
+      await prisma.trace.upsert({
+        where: {
+          id: observationId,
+        },
+        create: {
+          projectId: apiScope.projectId,
+          name: this.event.body.name,
+          id: observationId,
+          timestamp: this.event.body.startTime || new Date(),
+        },
+        update: {},
+      });
+
+      if (env.CLICKHOUSE_URL) {
+        // Insert trace into clickhouse if enabled
+        await upsertTrace({
+          id: observationId,
+          project_id: apiScope.projectId,
+          timestamp: convertDateToClickhouseDateTime(
+            this.event.body.startTime
+              ? new Date(this.event.body.startTime)
+              : new Date(),
+          ),
+          created_at: convertDateToClickhouseDateTime(new Date()),
+          updated_at: convertDateToClickhouseDateTime(new Date()),
+        });
+      }
+    }
+
     return {
       id: observationId,
       create: {
         id: observationId,
-        traceId: traceId,
+        traceId,
         type: type,
         name: this.event.body.name,
         startTime: this.event.body.startTime
@@ -359,7 +394,7 @@ export class ObservationProcessor implements EventProcessor {
     model?: Model,
     existingObservation?: Omit<Observation, "input" | "output">,
   ) {
-    let newPromptTokens = body.usage?.input;
+    let newPromptTokens = body.usage?.input || body.usageDetails?.input;
     if (newPromptTokens === undefined && model && model.tokenizerId) {
       if (body.input) {
         newPromptTokens = calculateTokenDelegate({
@@ -384,7 +419,7 @@ export class ObservationProcessor implements EventProcessor {
       }
     }
 
-    let newCompletionTokens = body.usage?.output;
+    let newCompletionTokens = body.usage?.output || body.usageDetails?.output;
 
     if (newCompletionTokens === undefined && model && model.tokenizerId) {
       if (body.output) {
@@ -600,19 +635,32 @@ export class TraceProcessor implements EventProcessor {
           : undefined;
 
     if (body.sessionId) {
-      await prisma.traceSession.upsert({
-        where: {
-          id_projectId: {
+      try {
+        await prisma.traceSession.upsert({
+          where: {
+            id_projectId: {
+              id: body.sessionId,
+              projectId: apiScope.projectId,
+            },
+          },
+          create: {
             id: body.sessionId,
             projectId: apiScope.projectId,
           },
-        },
-        create: {
-          id: body.sessionId,
-          projectId: apiScope.projectId,
-        },
-        update: {},
-      });
+          update: {},
+        });
+      } catch (e) {
+        if (
+          e instanceof Prisma.PrismaClientKnownRequestError &&
+          e.code === "P2002"
+        ) {
+          logger.warn(
+            `Failed to upsert session. Session ${body.sessionId} in project ${apiScope.projectId} already exists`,
+          );
+        } else {
+          throw e;
+        }
+      }
     }
 
     // Do not use nested upserts or multiple where conditions as this should be a single native database upsert

packages/shared/src/server/ingestion/legacy/index.ts

@@ -3,13 +3,7 @@ import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
 import { getProcessorForEvent } from "./EventProcessor";
-import { TraceUpsertEventType } from "../../queues";
-import {
-  convertTraceUpsertEventsToRedisEvents,
-  TraceUpsertQueue,
-} from "../../redis/traceUpsert";
 import { ApiAccessScope } from "../../auth/types";
-import { redis } from "../../redis/redis";
 import { backOff } from "exponential-backoff";
 import { Model } from "../../..";
 import { logger } from "../../logger";
@@ -89,7 +83,7 @@ export const handleBatch = async (
 
 async function retry<T>(request: () => Promise<T>): Promise<T> {
   return await backOff(request, {
-    numOfAttempts: env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" ? 5 : 3,
+    numOfAttempts: 5,
     retry: (e: Error, attemptNumber: number) => {
       if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
         logger.info("not retrying auth error");
@@ -163,42 +157,5 @@ export function cleanEvent(obj: unknown): unknown {
   }
 }
 
-export const isNotNullOrUndefined = <T>(
-  val?: T | null,
-): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
-
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
-
-export const addTracesToTraceUpsertQueue = async (
-  batchResults: BatchResult[],
-  projectId: string,
-): Promise<void> => {
-  const traceEvents: TraceUpsertEventType[] = batchResults
-    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
-    .map((result) =>
-      result.result &&
-      typeof result.result === "object" &&
-      "id" in result.result
-        ? // ingestion API only gets traces for one projectId
-          { traceId: result.result.id as string, projectId }
-        : null,
-    )
-    .filter(isNotNullOrUndefined);
-
-  try {
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
-
-      const queue = TraceUpsertQueue.getInstance();
-      if (!queue) {
-        logger.error("TraceUpsertQueue not initialized");
-        return;
-      }
-
-      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    }
-  } catch (error) {
-    logger.error("Error sending events to worker", error);
-  }
-};

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -0,0 +1,426 @@
+import { randomUUID } from "crypto";
+import { z } from "zod";
+
+import { type Model } from "../../db";
+import { env } from "../../env";
+import {
+  InvalidRequestError,
+  LangfuseNotFoundError,
+  UnauthorizedError,
+} from "../../errors";
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
+import {
+  getCurrentSpan,
+  instrumentAsync,
+  instrumentSync,
+  recordIncrement,
+  traceException,
+} from "../instrumentation";
+import { logger } from "../logger";
+import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { IngestionQueue } from "../redis/ingestionQueue";
+import { LegacyIngestionQueue } from "../redis/legacyIngestion";
+import { redis } from "../redis/redis";
+import { handleBatch } from "./legacy";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
+import { getProcessorForEvent } from "./legacy/EventProcessor";
+import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+
+export type TokenCountDelegate = (p: {
+  model: Model;
+  text: unknown;
+}) => number | undefined;
+
+let s3StorageServiceClient: StorageService;
+
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
+  }
+  return s3StorageServiceClient;
+};
+
+export const processEventBatch = async (
+  input: unknown[],
+  authCheck: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): Promise<{
+  successes: { id: string; status: number }[];
+  errors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[];
+}> => {
+  // add context of api call to the span
+  const currentSpan = getCurrentSpan();
+  recordIncrement("langfuse.ingestion.event", input.length);
+  currentSpan?.setAttribute("event_count", input.length);
+
+  /**************
+   * VALIDATION *
+   **************/
+  const validationErrors: { id: string; error: unknown }[] = [];
+  const authenticationErrors: { id: string; error: unknown }[] = [];
+
+  const batch: z.infer<typeof ingestionEvent>[] = input
+    .flatMap((event) => {
+      const parsed = instrumentSync(
+        { name: "ingestion-zod-parse-individual-event" },
+        (span) => {
+          const parsedBody = ingestionEvent.safeParse(event);
+          if (parsedBody.data?.id !== undefined) {
+            span.setAttribute("object.id", parsedBody.data.id);
+          }
+          return parsedBody;
+        },
+      );
+      if (!parsed.success) {
+        validationErrors.push({
+          id:
+            typeof event === "object" && event && "id" in event
+              ? typeof event.id === "string"
+                ? event.id
+                : "unknown"
+              : "unknown",
+          error: new InvalidRequestError(parsed.error.message),
+        });
+        return [];
+      }
+      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+        authenticationErrors.push({
+          id: parsed.data.id,
+          error: new UnauthorizedError("Access Scope Denied"),
+        });
+        return [];
+      }
+      return [parsed.data];
+    })
+    .flatMap((event) => {
+      if (event.type === eventTypes.SDK_LOG) {
+        // Log SDK_LOG events, but remove them from further processing
+        logger.info("SDK Log Event", { event });
+        return [];
+      }
+      return [event];
+    });
+
+  const sortedBatch = sortBatch(batch);
+
+  // We group events by eventBodyId which allows us to store and process them
+  // as one which reduces infra interactions per event. Only used in the S3 case.
+  const sortedBatchByEventBodyId = sortedBatch.reduce(
+    (
+      acc: Record<
+        string,
+        {
+          data: IngestionEventType[];
+          key: string;
+          eventBodyId: string;
+          type: (typeof eventTypes)[keyof typeof eventTypes];
+        }
+      >,
+      event,
+    ) => {
+      if (!event.body?.id) {
+        return acc;
+      }
+      const key = `${getClickhouseEntityType(event.type)}-${event.body.id}`;
+      if (!acc[key]) {
+        acc[key] = {
+          data: [],
+          key: event.id,
+          type: event.type,
+          eventBodyId: event.body.id,
+        };
+      }
+      acc[key].data.push(event);
+      return acc;
+    },
+    {},
+  );
+
+  /********************
+   * ASYNC PROCESSING *
+   ********************/
+  let s3UploadErrored = false;
+  await instrumentAsync({ name: "s3-upload-events" }, async () => {
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    // S3 Event Upload is blocking, but non-failing.
+    // If a promise rejects, we log it below, but do not throw an error.
+    // In this case, we upload the full batch into the Redis queue.
+    const results = await Promise.allSettled(
+      Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+        // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+        // That way we batch updates from the same invocation into a single file and reduce
+        // write operations on S3.
+        const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+        return s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+          data,
+        );
+      }),
+    );
+    results.forEach((result) => {
+      if (result.status === "rejected") {
+        s3UploadErrored = true;
+        logger.error("Failed to upload event to S3", {
+          error: result.reason,
+        });
+      }
+    });
+  });
+
+  // This is a workaround to allow us to disable async ingestion processing for SDK CI testing
+  // TODO: remove this block after SDKs are ready for V3 async ingestion processing
+  if (env.LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED === "true") {
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
+  }
+
+  // Send each event individually to IngestionQueue for ClickHouse processing
+  if (env.LANGFUSE_CLICKHOUSE_INGESTION_ENABLED === "true") {
+    if (s3UploadErrored) {
+      throw new Error(
+        "Failed to upload events to blob storage, aborting event processing",
+      );
+    }
+
+    if (redis) {
+      const queue = IngestionQueue.getInstance();
+      await Promise.all(
+        Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+          queue
+            ? queue.add(
+                QueueJobs.IngestionJob,
+                {
+                  id: randomUUID(),
+                  timestamp: new Date(),
+                  name: QueueJobs.IngestionJob as const,
+                  payload: {
+                    data: {
+                      type: sortedBatchByEventBodyId[id].type,
+                      eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                    },
+                    authCheck,
+                  },
+                },
+                {
+                  delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+                },
+              )
+            : Promise.reject("Failed to instantiate queue"),
+        ),
+      );
+      if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED !== "true") {
+        // If postgres ingestion is disabled, we return early
+        return aggregateBatchResult(
+          [...validationErrors, ...authenticationErrors],
+          sortedBatch.map((event) => ({ id: event.id, result: event })),
+          authCheck.scope.projectId,
+        );
+      }
+    }
+  }
+
+  if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED === "true") {
+    // As part of the legacy processing we sent the entire batch to the worker.
+    if (redis) {
+      const queue = LegacyIngestionQueue.getInstance();
+
+      if (queue) {
+        let addToQueueFailed = false;
+
+        const queuePayload: LegacyIngestionEventType = !s3UploadErrored
+          ? {
+              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
+                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+                return {
+                  type,
+                  eventBodyId,
+                  eventId: key,
+                };
+              }),
+              authCheck,
+              useS3EventStore: true,
+            }
+          : { data: sortedBatch, authCheck, useS3EventStore: false };
+
+        try {
+          await queue.add(QueueJobs.LegacyIngestionJob, {
+            payload: queuePayload,
+            id: randomUUID(),
+            timestamp: new Date(),
+            name: QueueJobs.LegacyIngestionJob as const,
+          });
+        } catch (e: unknown) {
+          logger.warn(
+            "Failed to add batch to queue, falling back to sync processing",
+            e,
+          );
+          addToQueueFailed = true;
+        }
+
+        if (!addToQueueFailed) {
+          return aggregateBatchResult(
+            // we are not sending additional server errors to the client in case of early return
+            [...validationErrors, ...authenticationErrors],
+            sortedBatch.map((event) => ({ id: event.id, result: event })),
+            authCheck.scope.projectId,
+          );
+        }
+      } else {
+        logger.warn(
+          "Ingestion queue not initialized, falling back to sync processing",
+        );
+      }
+    }
+
+    /*******************
+     * SYNC PROCESSING *
+     *******************/
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
+  }
+
+  throw new Error(
+    "Either Clickhouse or Postgres ingestion (or both) must be enabled",
+  );
+};
+
+const isAuthorized = (
+  event: IngestionEventType,
+  authScope: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): boolean => {
+  try {
+    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+    return true;
+  } catch (error) {
+    return false;
+  }
+};
+
+/**
+ * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
+ */
+const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+  const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
+    eventTypes.GENERATION_UPDATE,
+    eventTypes.SPAN_UPDATE,
+    eventTypes.OBSERVATION_UPDATE, // legacy event type
+  ];
+  const updates = batch
+    .filter((event) => updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+  const others = batch
+    .filter((event) => !updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+
+  // Return the array with non-update events first, followed by update events
+  return [...others, ...updates];
+};
+
+export const aggregateBatchResult = (
+  errors: Array<{ id: string; error: unknown }>,
+  results: Array<{ id: string; result: unknown }>,
+  projectId?: string,
+) => {
+  const returnedErrors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[] = [];
+
+  const successes: {
+    id: string;
+    status: number;
+  }[] = [];
+
+  errors.forEach((error) => {
+    if (error.error instanceof InvalidRequestError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 400,
+        message: "Invalid request data",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof UnauthorizedError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 401,
+        message: "Authentication error",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof LangfuseNotFoundError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 404,
+        message: "Resource not found",
+        error: error.error.message,
+      });
+    } else {
+      returnedErrors.push({
+        id: error.id,
+        status: 500,
+        error: "Internal Server Error",
+      });
+    }
+  });
+
+  if (returnedErrors.length > 0) {
+    traceException(errors);
+    logger.error("Error processing events", {
+      errors: returnedErrors,
+      "langfuse.project.id": projectId,
+    });
+  }
+
+  results.forEach((result) => {
+    successes.push({
+      id: result.id,
+      status: 201,
+    });
+  });
+
+  return { successes, errors: returnedErrors };
+};

packages/shared/src/server/ingestion/types.ts

@@ -56,10 +56,66 @@ export const usage = MixedUsage.nullish()
   // ensure output is always of new usage model
   .pipe(Usage.nullish());
 
+const RawUsageOrCostDetails = z.record(
+  z.string(),
+  z.number().nonnegative().nullish(),
+);
+
+const OpenAIUsageSchema = z
+  .object({
+    prompt_tokens: z.number().nonnegative(),
+    completion_tokens: z.number().nonnegative(),
+    total_tokens: z.number().nonnegative(),
+    prompt_tokens_details: z.record(z.string(), z.number().nonnegative()),
+    completion_tokens_details: z.record(z.string(), z.number().nonnegative()),
+  })
+  .strict()
+  .transform((v) => {
+    if (!v) return;
+
+    const {
+      prompt_tokens,
+      completion_tokens,
+      total_tokens,
+      prompt_tokens_details,
+      completion_tokens_details,
+    } = v;
+    const result: z.infer<typeof RawUsageOrCostDetails> & {
+      input: number;
+      output: number;
+      total: number;
+    } = {
+      input: prompt_tokens,
+      output: completion_tokens,
+      total: total_tokens,
+    };
+
+    if (prompt_tokens_details) {
+      for (const [key, value] of Object.entries(prompt_tokens_details)) {
+        result[`input_${key}`] = value;
+        result.input = Math.max(result.input - (value ?? 0), 0);
+      }
+    }
+
+    if (completion_tokens_details) {
+      for (const [key, value] of Object.entries(completion_tokens_details)) {
+        result[`output_${key}`] = value;
+        result.output = Math.max(result.output - (value ?? 0), 0);
+      }
+    }
+
+    return result;
+  })
+  .pipe(RawUsageOrCostDetails);
+
+export const UsageOrCostDetails = z
+  .union([OpenAIUsageSchema, RawUsageOrCostDetails])
+  .nullish();
+
 export const TraceBody = z.object({
   id: z.string().nullish(),
   timestamp: stringDateTime,
-  name: z.string().nullish(),
+  name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -119,6 +175,8 @@ export const CreateGenerationBody = CreateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -147,6 +205,8 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -298,6 +358,8 @@ export const LegacyObservationBody = z.object({
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   metadata: jsonSchema.nullish(),
   parentObservationId: z.string().nullish(),
   level: z.nativeEnum(ObservationLevel).nullish(),

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -17,7 +17,7 @@ type ValidateAndInflateScoreParams = {
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
 ): Promise<Score> {
-  const { body, projectId } = params;
+  const { body, projectId, scoreId } = params;
 
   if (body.configId) {
     const config = await prisma.scoreConfig.findFirst({
@@ -32,10 +32,22 @@ export async function validateAndInflateScore(
         "The configId you provided does not match a valid config in this project",
       );
 
-    validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+    // Override some fields in the score body with config fields
+    // We ignore the set fields in the body
+    const bodyWithConfigOverrides = {
+      ...body,
+      name: config.name,
+    };
+
+    validateConfigAgainstBody(
+      bodyWithConfigOverrides,
+      config as ValidatedScoreConfig,
+    );
 
     return inflateScoreBody({
-      ...params,
+      projectId,
+      scoreId,
+      body: bodyWithConfigOverrides,
       config: config as ValidatedScoreConfig,
     });
   }

packages/shared/src/server/instrumentation/index.ts

@@ -1,10 +1,10 @@
-import * as opentelemetry from "@opentelemetry/api";
-import * as dd from "dd-trace";
-import { env } from "../../env";
 import {
   CloudWatchClient,
   PutMetricDataCommand,
 } from "@aws-sdk/client-cloudwatch";
+import * as opentelemetry from "@opentelemetry/api";
+import * as dd from "dd-trace";
+import { env } from "../../env";
 import { logger } from "../logger";
 
 // type CallbackFn<T> = () => T;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,9 +1,13 @@
 import type { ZodSchema } from "zod";
 
+import { CallbackHandler } from "langfuse-langchain";
+
 import { ChatAnthropic } from "@langchain/anthropic";
+import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
   AIMessage,
+  BaseMessage,
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
@@ -13,15 +17,31 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
-import {
+import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import {
+  processEventBatch,
+  type TokenCountDelegate,
+} from "../ingestion/processEventBatch";
+import { logger } from "../logger";
 import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
 
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
+type ProcessTracedEvents = () => Promise<void>;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};
+
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
@@ -31,6 +51,7 @@ type LLMCompletionParams = {
   apiKey: string;
   maxRetries?: number;
   config?: Record<string, string> | null;
+  traceParams?: TraceParams;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -40,25 +61,34 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
-  }
-): Promise<IterableReadableStream<Uint8Array>>;
+  },
+): Promise<{
+  completion: IterableReadableStream<Uint8Array>;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
-  }
-): Promise<string>;
+  },
+): Promise<{ completion: string; processTracedEvents: ProcessTracedEvents }>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
     structuredOutputSchema: ZodSchema;
-  }
-): Promise<unknown>;
+  },
+): Promise<{
+  completion: unknown;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
-  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
+  params: FetchLLMCompletionParams,
+): Promise<{
+  completion: string | IterableReadableStream<Uint8Array> | unknown;
+  processTracedEvents: ProcessTracedEvents;
+}> {
   // the apiKey must never be printed to the console
   const {
     messages,
@@ -69,18 +99,61 @@ export async function fetchLLMCompletion(
     baseURL,
     maxRetries,
     config,
+    traceParams,
   } = params;
 
-  const finalMessages = messages.map((message) => {
-    if (message.role === ChatMessageRole.User)
-      return new HumanMessage(message.content);
-    if (message.role === ChatMessageRole.System)
-      return new SystemMessage(message.content);
+  let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
+  let processTracedEvents: ProcessTracedEvents = () => Promise.resolve();
 
-    return new AIMessage(message.content);
-  });
+  if (traceParams) {
+    const handler = new CallbackHandler({
+      _projectId: traceParams.projectId,
+      _isLocalEventExportEnabled: true,
+      tags: traceParams.tags,
+    });
 
-  let chatModel: ChatOpenAI | ChatAnthropic | ChatBedrockConverse;
+    finalCallbacks.push(handler);
+
+    processTracedEvents = async () => {
+      try {
+        const events = await handler.langfuse._exportLocalEvents(
+          traceParams.projectId,
+        );
+        await processEventBatch(
+          JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
+          traceParams.authCheck,
+          traceParams.tokenCountDelegate,
+        );
+      } catch (e) {
+        logger.error("Failed to process traced events", { error: e });
+      }
+    };
+  }
+
+  finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
+
+  let finalMessages: BaseMessage[];
+  // VertexAI requires at least 1 user message
+  if (modelParams.adapter === LLMAdapter.VertexAI && messages.length === 1) {
+    finalMessages = [new HumanMessage(messages[0].content)];
+  } else {
+    finalMessages = messages.map((message) => {
+      if (message.role === ChatMessageRole.User)
+        return new HumanMessage(message.content);
+      if (message.role === ChatMessageRole.System)
+        return new SystemMessage(message.content);
+
+      return new AIMessage(message.content);
+    });
+  }
+
+  finalMessages = finalMessages.filter((m) => m.content.length > 0);
+
+  let chatModel:
+    | ChatOpenAI
+    | ChatAnthropic
+    | ChatBedrockConverse
+    | ChatVertexAI;
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
@@ -89,8 +162,8 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
-      clientOptions: { maxRetries },
+      callbacks: finalCallbacks,
+      clientOptions: { maxRetries, timeout: 1000 * 60 * 2 }, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
@@ -99,11 +172,13 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      streamUsage: false, // https://github.com/langchain-ai/langchainjs/issues/6533
+      callbacks: finalCallbacks,
       maxRetries,
       configuration: {
         baseURL,
       },
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Azure) {
     chatModel = new ChatOpenAI({
@@ -114,8 +189,9 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
@@ -128,8 +204,26 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
+    });
+  } else if (modelParams.adapter === LLMAdapter.VertexAI) {
+    const credentials = GCPServiceAccountKeySchema.parse(JSON.parse(apiKey));
+
+    // Requests time out after 60 seconds for both public and private endpoints by default
+    // Reference: https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions#send-request
+    chatModel = new ChatVertexAI({
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxOutputTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks: finalCallbacks,
+      maxRetries,
+      authOptions: {
+        projectId: credentials.project_id,
+        credentials,
+      },
     });
   } else {
     // eslint-disable-next-line no-unused-vars
@@ -137,10 +231,19 @@ export async function fetchLLMCompletion(
     throw new Error("This model provider is not supported.");
   }
 
+  const runConfig = {
+    callbacks: finalCallbacks,
+    runId: traceParams?.traceId,
+    runName: traceParams?.traceName,
+  };
+
   if (params.structuredOutputSchema) {
-    return await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-      .withStructuredOutput(params.structuredOutputSchema)
-      .invoke(finalMessages);
+    return {
+      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+        .withStructuredOutput(params.structuredOutputSchema)
+        .invoke(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
   /*
@@ -156,27 +259,42 @@ export async function fetchLLMCompletion(
   Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
   */
   if (modelParams.model.startsWith("o1-")) {
-    return await new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      temperature: 1,
-      maxTokens: undefined,
-      topP: undefined,
-      callbacks,
-      maxRetries,
-      configuration: {
-        baseURL,
-      },
-    })
-      .pipe(new StringOutputParser())
-      .invoke(
-        finalMessages.filter((message) => message._getType() !== "system")
-      );
+    return {
+      completion: await new ChatOpenAI({
+        openAIApiKey: apiKey,
+        modelName: modelParams.model,
+        temperature: 1,
+        maxTokens: undefined,
+        topP: undefined,
+        callbacks,
+        maxRetries,
+        configuration: {
+          baseURL,
+        },
+        timeout: 1000 * 60 * 2, // 2 minutes timeout
+      })
+        .pipe(new StringOutputParser())
+        .invoke(
+          finalMessages.filter((message) => message._getType() !== "system"),
+          runConfig,
+        ),
+      processTracedEvents,
+    };
   }
 
   if (streaming) {
-    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
+    return {
+      completion: await chatModel
+        .pipe(new BytesOutputParser())
+        .stream(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
-  return await chatModel.pipe(new StringOutputParser()).invoke(finalMessages);
+  return {
+    completion: await chatModel
+      .pipe(new StringOutputParser())
+      .invoke(finalMessages, runConfig),
+    processTracedEvents,
+  };
 }

packages/shared/src/server/llm/types.ts

@@ -16,6 +16,7 @@ export enum LLMAdapter {
   OpenAI = "openai",
   Azure = "azure",
   Bedrock = "bedrock",
+  VertexAI = "vertex-ai",
 }
 
 export enum ChatMessageRole {
@@ -26,6 +27,20 @@ export enum ChatMessageRole {
 
 export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
 
+const ChatMessageSchema = z.object({
+  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+
+export const PromptContentSchema = z.union([
+  ChatMessageListSchema,
+  TextPromptSchema,
+]);
+export type PromptContent = z.infer<typeof PromptContentSchema>;
+
 export type ModelParams = {
   provider: string;
   adapter: LLMAdapter;
@@ -49,6 +64,17 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
+// Experiment config
+export const ExperimentMetadataSchema = z
+  .object({
+    prompt_id: z.string(),
+    provider: z.string(),
+    model: z.string(),
+    model_params: ZodModelConfig,
+  })
+  .strict();
+export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
+
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",
@@ -89,10 +115,18 @@ export const anthropicModels = [
   "claude-instant-1.2",
 ] as const;
 
+export const vertexAIModels = [
+  "gemini-1.5-pro",
+  "gemini-1.5-flash",
+  "gemini-1.0-pro",
+] as const;
+
 export type AnthropicModel = (typeof anthropicModels)[number];
+export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
   [LLMAdapter.Anthropic]: anthropicModels,
   [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.VertexAI]: vertexAIModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
 } as const;

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -1,20 +1,13 @@
 import { filterOperators } from "../../../interfaces/filters";
+import { clickhouseCompliantRandomCharacters } from "../../repositories";
 
-function randomCharacters() {
-  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-  let result = "";
-  const randomArray = new Uint8Array(5);
-  crypto.getRandomValues(randomArray);
-  randomArray.forEach((number) => {
-    result += chars[number % chars.length];
-  });
-  return result;
-}
-
+export type ClickhouseOperator =
+  | (typeof filterOperators)[keyof typeof filterOperators][number]
+  | "!=";
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
-  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  operator: ClickhouseOperator;
   field: string;
 }
 type ClickhouseFilter = {
@@ -44,7 +37,7 @@ export class StringFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varName = `stringFilter${randomCharacters()}`;
+    const varName = `stringFilter${clickhouseCompliantRandomCharacters()}`;
 
     const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     let query: string;
@@ -79,29 +72,33 @@ export class NumberFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
   public value: number;
-  public operator: (typeof filterOperators)["number"][number];
+  public operator: (typeof filterOperators)["number"][number] | "!=";
+  public clickhouseTypeOverwrite?: string;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["number"][number];
+    operator: (typeof filterOperators)["number"][number] | "!=";
     value: number;
     tablePrefix?: string;
+    clickhouseTypeOverwrite?: string;
   }) {
     this.clickhouseTable = opts.clickhouseTable;
     this.field = opts.field;
     this.value = opts.value;
     this.operator = opts.operator;
     this.tablePrefix = opts.tablePrefix;
+    this.clickhouseTypeOverwrite = opts.clickhouseTypeOverwrite;
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `numberFilter${uid}`;
+    const type = this.clickhouseTypeOverwrite ?? "Decimal64(12)";
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Decimal}`,
-      params: { [varName]: this.value },
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: ${type}}`,
+      params: { [varName]: this.value.toString() },
     };
   }
 }
@@ -128,7 +125,7 @@ export class DateTimeFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `dateTimeFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: DateTime64(3)}`,
@@ -159,13 +156,13 @@ export class StringOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `stringOptionsFilter${uid}`;
     return {
       query:
         this.operator === "any of"
-          ? `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`
-          : `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`,
+          ? `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} IN ({${varName}: Array(String)})`
+          : `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} NOT IN ({${varName}: Array(String)})`,
       params: { [varName]: this.values },
     };
   }
@@ -198,8 +195,8 @@ export class StringObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `stringObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `stringObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `stringObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `stringObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
 
     //  const query: `${column}['{varKeyName: String}'] ${this.operator} {${varValueName}: String}`,
@@ -254,7 +251,7 @@ export class ArrayOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `arrayOptionsFilter${uid}`;
     let query: string;
 
@@ -266,7 +263,7 @@ export class ArrayOptionsFilter implements Filter {
         query = `hasAny({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`;
         break;
       case "all of":
-        query = `arrayAll(x -> has({${varName}: Array(String)}, x), ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`;
+        query = `hasAll(${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}, {${varName}: Array(String)}) = True`;
         break;
       default:
         throw new Error(`Unsupported operator: ${this.operator}`);
@@ -279,18 +276,44 @@ export class ArrayOptionsFilter implements Filter {
   }
 }
 
-export class NumberObjectFilter implements Filter {
+export class NullFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
-  public key: string;
-  public value: number;
-  public operator: (typeof filterOperators)["numberObject"][number];
+  public operator: (typeof filterOperators)["null"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["numberObject"][number];
+    operator: (typeof filterOperators)["null"][number];
+    tablePrefix?: string;
+  }) {
+    this.clickhouseTable = opts.clickhouseTable;
+    this.field = opts.field;
+    this.operator = opts.operator;
+    this.tablePrefix = opts.tablePrefix;
+  }
+
+  apply(): ClickhouseFilter {
+    return {
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator}`,
+      params: {},
+    };
+  }
+}
+
+export class NumberObjectFilter implements Filter {
+  public clickhouseTable: string;
+  public field: string;
+  public key: string;
+  public value: number;
+  public operator: (typeof filterOperators)["numberObject"][number] | "!=";
+  protected tablePrefix?: string;
+
+  constructor(opts: {
+    clickhouseTable: string;
+    field: string;
+    operator: (typeof filterOperators)["numberObject"][number] | "!=";
     key: string;
     value: number;
     tablePrefix?: string;
@@ -304,11 +327,11 @@ export class NumberObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `numberObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `numberObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `numberObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `numberObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     return {
-      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal})), ${column})) = 0`,
+      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal64(12)})), ${column})) = 0`,
       params: { [varKeyName]: this.key, [varValueName]: this.value },
     };
   }
@@ -336,7 +359,7 @@ export class BooleanFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `booleanFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Boolean}`,
@@ -348,7 +371,7 @@ export class BooleanFilter implements Filter {
 export class FilterList {
   private filters: Filter[];
 
-  constructor(filters: Filter[]) {
+  constructor(filters: Filter[] = []) {
     this.filters = filters;
   }
 
@@ -360,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  length() {
+    return this.filters.length;
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -14,6 +14,7 @@ import {
   BooleanFilter,
   NumberObjectFilter,
   StringObjectFilter,
+  NullFilter,
 } from "./clickhouse-filter";
 
 export class QueryBuilderError extends Error {
@@ -66,6 +67,7 @@ export const createFilterFromFilterState = (
           operator: frontEndFilter.operator,
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
+          clickhouseTypeOverwrite: column.clickhouseTypeOverwrite,
         });
       case "arrayOptions":
         return new ArrayOptionsFilter({
@@ -101,9 +103,16 @@ export const createFilterFromFilterState = (
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
         });
-
+      case "null":
+        return new NullFilter({
+          clickhouseTable: column.clickhouseTableName,
+          field: column.clickhouseSelect,
+          operator: frontEndFilter.operator,
+          tablePrefix: column.queryPrefix,
+        });
       default:
-        logger.error(`Invalid filter type: ${JSON.stringify(frontEndFilter)}`);
+        const exhaustiveCheck: never = frontEndFilter;
+        logger.error(`Invalid filter type: ${JSON.stringify(exhaustiveCheck)}`);
         throw new QueryBuilderError(`Invalid filter type`);
     }
   });

packages/shared/src/server/queries/clickhouse-sql/orderby-factory.ts

@@ -3,31 +3,53 @@ import { OrderByState } from "../../../interfaces/orderBy";
 import { UiColumnMapping } from "../../../tableDefinitions";
 import { logger } from "../../logger";
 
+type OrderByStateNotNull = Exclude<OrderByState, null>;
+
 export function orderByToClickhouseSql(
-  orderBy: OrderByState,
+  orderBy: OrderByState | OrderByState[] = [],
   tableColumns: UiColumnMapping[],
 ): string {
-  if (!orderBy) {
+  if (
+    !orderBy ||
+    (Array.isArray(orderBy) && orderBy.filter(Boolean).length === 0)
+  ) {
     return "";
   }
-  // Get column definition to map column to internal name, e.g. "t.id"
-  const col = tableColumns.find(
-    (c) => c.uiTableName === orderBy.column || c.uiTableId === orderBy.column,
-  );
 
-  if (!col) {
-    logger.warn("Invalid order by column", orderBy.column);
-    throw new Error("Invalid order by column: " + orderBy.column);
+  if (!Array.isArray(orderBy)) {
+    orderBy = [orderBy];
+  }
+  // Initialize an array to hold order by clauses
+  const orderByClauses: string[] = [];
+
+  // Loop through each orderBy entry
+  for (const ob of orderBy.filter((o): o is OrderByStateNotNull =>
+    Boolean(o),
+  )) {
+    // Get column definition to map column to internal name, e.g. "t.id"
+    const col = tableColumns.find(
+      (c) => c.uiTableName === ob.column || c.uiTableId === ob.column,
+    );
+
+    if (!col) {
+      logger.warn("Invalid order by column", ob.column);
+      throw new Error("Invalid order by column: " + ob.column);
+    }
+
+    // Assert that ob.order is either "asc" or "desc"
+    const orderByOrder = z.enum(["ASC", "DESC"]);
+    const order = orderByOrder.safeParse(ob.order);
+    if (!order.success) {
+      logger.warn("Invalid order", ob.order);
+      throw new Error("Invalid order: " + ob.order);
+    }
+
+    // Append the order by clause to the array
+    orderByClauses.push(
+      `${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`,
+    );
   }
 
-  // Assert that orderBy.order is either "asc" or "desc"
-  const orderByOrder = z.enum(["ASC", "DESC"]);
-  const order = orderByOrder.safeParse(orderBy.order);
-  if (!order.success) {
-    logger.warn("Invalid order", orderBy.order);
-    throw new Error("Invalid order: " + orderBy.order);
-  }
-
-  // Both column and order are safe, can use raw SQL
-  return `ORDER BY ${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`;
+  // Join all order by clauses with a comma and return
+  return `ORDER BY ${orderByClauses.join(", ")}`;
 }

packages/shared/src/server/queries/createGenerationsQuery.ts

@@ -10,6 +10,8 @@ import { TableFilters } from "./types";
 type AdditionalObservationFields = {
   traceName: string | null;
   traceTags: Array<string>;
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
 };
 
 export type FullObservation = AdditionalObservationFields & ObservationView;

packages/shared/src/server/queries/createTracesQuery.ts

@@ -11,13 +11,13 @@ export function parseTraceAllFilters(input: TableFilters) {
   const filterCondition = tableColumnsToSqlFilterAndPrefix(
     input.filter ?? [],
     tracesTableCols,
-    "traces"
+    "traces",
   );
   const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
 
   // to improve query performance, add timeseries filter to observation queries as well
   const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime"
+    (f) => f.column === "Timestamp" && f.type === "datetime",
   );
 
   const observationTimeseriesFilter =
@@ -25,7 +25,7 @@ export function parseTraceAllFilters(input: TableFilters) {
       ? datetimeFilterToPrismaSql(
           "start_time",
           timeseriesFilter.operator,
-          timeseriesFilter.value
+          timeseriesFilter.value,
         )
       : Prisma.empty;
 
@@ -130,6 +130,6 @@ export function createTracesQuery({
     ${filterCondition}
   ${orderByCondition}
   ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
+  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
 `;
 }

packages/shared/src/server/queries/index.ts

@@ -7,3 +7,17 @@ export {
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
+export {
+  FilterList,
+  StringFilter,
+  DateTimeFilter,
+  StringOptionsFilter,
+  NumberFilter,
+  ArrayOptionsFilter,
+  BooleanFilter,
+  NumberObjectFilter,
+  StringObjectFilter,
+  NullFilter,
+  type ClickhouseOperator,
+} from "./clickhouse-sql/clickhouse-filter";
+export { orderByToClickhouseSql } from "./clickhouse-sql/orderby-factory";

packages/shared/src/server/queues.ts

@@ -1,14 +1,6 @@
 import { z } from "zod";
 import { eventTypes, ingestionBatchEvent } from ".";
 
-export enum EventName {
-  TraceUpsert = "TraceUpsert",
-  BatchExport = "BatchExport",
-  EvaluationExecution = "EvaluationExecution",
-  LegacyIngestion = "LegacyIngestion",
-  CloudUsageMetering = "CloudUsageMetering",
-}
-
 export const LegacyIngestionEventFull = z.object({
   useS3EventStore: z.literal(false),
   data: ingestionBatchEvent,
@@ -62,64 +54,113 @@ export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
 });
-export const TraceUpsertEventSchema = z.object({
+export const TraceQueueEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
+export const TracesQueueEventSchema = z.object({
+  projectId: z.string(),
+  traceIds: z.array(z.string()),
+});
+export const ProjectQueueEventSchema = z.object({
+  projectId: z.string(),
+  orgId: z.string(),
+});
+export const DatasetRunItemUpsertEventSchema = z.object({
+  projectId: z.string(),
+  datasetItemId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().optional(),
+});
 export const EvalExecutionEvent = z.object({
   projectId: z.string(),
   jobExecutionId: z.string(),
   delay: z.number().nullish(),
 });
+export const PostHogIntegrationProcessingEventSchema = z.object({
+  projectId: z.string(),
+});
+export const ExperimentCreateEventSchema = z.object({
+  projectId: z.string(),
+  datasetId: z.string(),
+  runId: z.string(),
+  description: z.string().optional(),
+});
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
-export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
+export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
+export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
+export type DatasetRunItemUpsertEventType = z.infer<
+  typeof DatasetRunItemUpsertEventSchema
+>;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
 export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
-
-export const EventBodySchema = z.union([
-  z.object({
-    name: z.literal(EventName.TraceUpsert),
-    payload: z.array(TraceUpsertEventSchema),
-  }),
-  z.object({
-    name: z.literal(EventName.EvaluationExecution),
-    payload: EvalExecutionEvent,
-  }),
-  z.object({
-    name: z.literal(EventName.BatchExport),
-    payload: BatchExportJobSchema,
-  }),
-]);
-export type EventBodyType = z.infer<typeof EventBodySchema>;
+export type ExperimentCreateEventType = z.infer<
+  typeof ExperimentCreateEventSchema
+>;
+export type PostHogIntegrationProcessingEventType = z.infer<
+  typeof PostHogIntegrationProcessingEventSchema
+>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+  DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
+  IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
   LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  ExperimentCreate = "experiment-create-queue",
+  PostHogIntegrationQueue = "posthog-integration-queue",
+  PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
 }
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
+  DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
-  EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
   LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  IngestionSecondaryJob = "secondary-ingestion-job",
+  ExperimentCreateJob = "experiment-create-job",
+  PostHogIntegrationJob = "posthog-integration-job",
+  PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
 }
 
 export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
     timestamp: Date;
     id: string;
-    payload: TraceUpsertEventType;
+    payload: TraceQueueEventType;
     name: QueueJobs.TraceUpsert;
   };
+  [QueueName.TraceDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: TracesQueueEventType | TraceQueueEventType;
+    name: QueueJobs.TraceDelete;
+  };
+  [QueueName.ProjectDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: ProjectQueueEventType;
+    name: QueueJobs.ProjectDelete;
+  };
+  [QueueName.DatasetRunItemUpsert]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetRunItemUpsertEventType;
+    name: QueueJobs.DatasetRunItemUpsert;
+  };
   [QueueName.EvaluationExecution]: {
     timestamp: Date;
     id: string;
@@ -144,4 +185,22 @@ export type TQueueJobTypes = {
     payload: IngestionEventQueueType;
     name: QueueJobs.IngestionJob;
   };
+  [QueueName.IngestionSecondaryQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: IngestionEventQueueType;
+    name: QueueJobs.IngestionJob;
+  };
+  [QueueName.ExperimentCreate]: {
+    timestamp: Date;
+    id: string;
+    payload: ExperimentCreateEventType;
+    name: QueueJobs.ExperimentCreateJob;
+  };
+  [QueueName.PostHogIntegrationProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: PostHogIntegrationProcessingEventType;
+    name: QueueJobs.PostHogIntegrationProcessingJob;
+  };
 };

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -0,0 +1,61 @@
+import { Queue } from "bullmq";
+import { env } from "../../env";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class CloudUsageMeteringQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudUsageMeteringQueue.instance) {
+      return CloudUsageMeteringQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudUsageMeteringQueue.instance = newRedis
+      ? new Queue(QueueName.CloudUsageMeteringQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudUsageMeteringQueue.instance?.on("error", (err) => {
+      logger.error("CloudUsageMeteringQueue error", err);
+    });
+
+    if (CloudUsageMeteringQueue.instance) {
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {
+          repeat: { pattern: "5 * * * *" },
+        },
+      );
+
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {},
+      );
+    }
+
+    return CloudUsageMeteringQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -0,0 +1,47 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DatasetRunItemUpsertQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetRunItemUpsert]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetRunItemUpsert]
+  > | null {
+    if (DatasetRunItemUpsertQueue.instance)
+      return DatasetRunItemUpsertQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetRunItemUpsertQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetRunItemUpsert]>(
+          QueueName.DatasetRunItemUpsert,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 5,
+              delay: 30_000, // 30 seconds
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetRunItemUpsertQueue.instance?.on("error", (err) => {
+      logger.error("DatasetRunItemUpsertQueue error", err);
+    });
+
+    return DatasetRunItemUpsertQueue.instance;
+  }
+}

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class EvalExecutionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null {
+    if (EvalExecutionQueue.instance) return EvalExecutionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    EvalExecutionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.EvaluationExecution]>(
+          QueueName.EvaluationExecution,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 10,
+              backoff: {
+                type: "exponential",
+                delay: 1000,
+              },
+            },
+          },
+        )
+      : null;
+
+    EvalExecutionQueue.instance?.on("error", (err) => {
+      logger.error("EvalExecutionQueue error", err);
+    });
+
+    return EvalExecutionQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class ExperimentCreateQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null {
+    if (ExperimentCreateQueue.instance) return ExperimentCreateQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ExperimentCreateQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ExperimentCreate]>(
+          QueueName.ExperimentCreate,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 10,
+              backoff: {
+                type: "exponential",
+                delay: 1000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ExperimentCreateQueue.instance?.on("error", (err) => {
+      logger.error("ExperimentCreateQueue error", err);
+    });
+
+    return ExperimentCreateQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -0,0 +1,48 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { BatchExportQueue } from "./batchExport";
+import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
+import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
+import { EvalExecutionQueue } from "./evalExecutionQueue";
+import { ExperimentCreateQueue } from "./experimentCreateQueue";
+import { IngestionQueue, SecondaryIngestionQueue } from "./ingestionQueue";
+import { LegacyIngestionQueue } from "./legacyIngestion";
+import { TraceUpsertQueue } from "./traceUpsert";
+import { TraceDeleteQueue } from "./traceDelete";
+import { ProjectDeleteQueue } from "./projectDelete";
+import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
+import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
+
+export function getQueue(queueName: QueueName): Queue | null {
+  switch (queueName) {
+    case QueueName.LegacyIngestionQueue:
+      return LegacyIngestionQueue.getInstance();
+    case QueueName.BatchExport:
+      return BatchExportQueue.getInstance();
+    case QueueName.CloudUsageMeteringQueue:
+      return CloudUsageMeteringQueue.getInstance();
+    case QueueName.DatasetRunItemUpsert:
+      return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.EvaluationExecution:
+      return EvalExecutionQueue.getInstance();
+    case QueueName.ExperimentCreate:
+      return ExperimentCreateQueue.getInstance();
+    case QueueName.TraceUpsert:
+      return TraceUpsertQueue.getInstance();
+    case QueueName.TraceDelete:
+      return TraceDeleteQueue.getInstance();
+    case QueueName.IngestionQueue:
+      return IngestionQueue.getInstance();
+    case QueueName.ProjectDelete:
+      return ProjectDeleteQueue.getInstance();
+    case QueueName.PostHogIntegrationQueue:
+      return PostHogIntegrationQueue.getInstance();
+    case QueueName.PostHogIntegrationProcessingQueue:
+      return PostHogIntegrationProcessingQueue.getInstance();
+    case QueueName.IngestionSecondaryQueue:
+      return SecondaryIngestionQueue.getInstance();
+    default:
+      const exhaustiveCheckDefault: never = queueName;
+      throw new Error(`Queue ${queueName} not found`);
+  }
+}

packages/shared/src/server/redis/ingestionQueue.ts

@@ -43,3 +43,45 @@ export class IngestionQueue {
     return IngestionQueue.instance;
   }
 }
+
+export class SecondaryIngestionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null {
+    if (SecondaryIngestionQueue.instance)
+      return SecondaryIngestionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    SecondaryIngestionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(
+          QueueName.IngestionSecondaryQueue,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    SecondaryIngestionQueue.instance?.on("error", (err) => {
+      logger.error("SecondaryIngestionQueue error", err);
+    });
+
+    return SecondaryIngestionQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationProcessingQueue.instance) {
+      return PostHogIntegrationProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100_000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationProcessingQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationProcessingQueue error", err);
+    });
+
+    return PostHogIntegrationProcessingQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationQueue.instance) {
+      return PostHogIntegrationQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationQueue error", err);
+    });
+
+    if (PostHogIntegrationQueue.instance) {
+      logger.debug("Scheduling jobs for PostHogIntegrationQueue");
+      PostHogIntegrationQueue.instance
+        .add(
+          QueueJobs.PostHogIntegrationJob,
+          {},
+          {
+            repeat: { pattern: "30 * * * *" }, // every hour at 30 minutes past
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding PostHogIntegrationJob schedule", err);
+        });
+    }
+
+    return PostHogIntegrationQueue.instance;
+  }
+}