chore: release v3.0.0

Signed-off-by: STRRL <im@strrl.dev>

.env.dev-azure.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -35,17 +35,18 @@ EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
 # DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
-# S3 storage
-S3_ENDPOINT=http://localhost:10000/devstoreaccount1
-S3_ACCESS_KEY_ID=devstoreaccount1
-S3_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-S3_BUCKET_NAME=langfuse
-S3_REGION=auto
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_BATCH_EXPORT_REGION=auto
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:10000/devstoreaccount1
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
@@ -57,7 +58,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
@@ -82,11 +82,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=false
-LANGFUSE_RETURN_FROM_CLICKHOUSE=true
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
-
-LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -34,17 +34,18 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -56,7 +57,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -79,11 +79,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=false
-LANGFUSE_RETURN_FROM_CLICKHOUSE=true
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
-
-LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.legacy.example

@@ -34,17 +34,18 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -56,7 +57,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -78,4 +78,9 @@ REDIS_AUTH="myredissecret"
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=true
+LANGFUSE_RETURN_FROM_CLICKHOUSE=false
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false

.env.local.example

@@ -1,22 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
-# Next Auth
-NEXTAUTH_SECRET="secret"
-NEXTAUTH_URL="http://localhost:3000"
-
-# feature flag to enable experimental features locally
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
-
-# Redis
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -67,6 +67,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,6 +91,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
@@ -100,16 +108,16 @@ OTEL_SERVICE_NAME="langfuse"
 # Defines the connection url for smtp server. 
 # SMTP_CONNECTION_URL=
 
-# S3 storage, optional, used for exports from the UI
-# S3_ENDPOINT=
-# S3_ACCESS_KEY_ID=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION=
-# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
+# S3 Batch Exports
+# LANGFUSE_S3_BATCH_EXPORT_ENABLED=
+# LANGFUSE_S3_BATCH_EXPORT_BUCKET=
+# LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=
+# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=
+# LANGFUSE_S3_BATCH_EXPORT_REGION=
+# LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=
+# LANGFUSE_S3_BATCH_EXPORT_PREFIX=
 
 # S3 storage for events, optional, used to persist all incoming events
-# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
 # LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
 # Optional prefix to be used within the bucket. Must end with `/` if set
 # LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
@@ -242,7 +250,6 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
 # LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
 ## END Langfuse V3 Ingestion

.github/workflows/_deploy_ecs_service.yml

@@ -58,7 +58,6 @@ jobs:
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-
       - name: Render AWS ECS Task Definition
         id: render-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1

.github/workflows/codespell.yml

@@ -6,6 +6,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -10,7 +10,11 @@ on:
   merge_group:
   pull_request:
     branches:
-      - "*"
+      - "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:
@@ -41,6 +45,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -89,6 +94,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -105,7 +111,8 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -133,6 +140,8 @@ jobs:
           LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run test-sync
         run: pnpm --filter=web run test-sync
+      - name: run test-client
+        run: pnpm --filter=web run test-client
 
   tests-web-async:
     timeout-minutes: 20
@@ -158,6 +167,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -174,7 +184,7 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev${{ matrix.blob-provider }}.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -222,6 +232,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -281,6 +292,7 @@ jobs:
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -313,6 +325,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -336,8 +349,6 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=true" >> .env
-          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
@@ -469,6 +480,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse-worker # GitHub
             langfuse/langfuse-worker # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -476,6 +489,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/snyk.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   snyk:
@@ -15,13 +20,13 @@ jobs:
       - name: Build a Docker image
         run: docker compose -f docker-compose.build.yml up -d
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
         continue-on-error: true
         uses: snyk/actions/docker@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-server
+          image: langfuse-langfuse-web
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
@@ -36,7 +41,7 @@ jobs:
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-worker
+          image: langfuse-langfuse-worker
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning

.gitignore

@@ -38,7 +38,6 @@ yarn-error.log*
 .env*
 !.env.dev.example
 !.env.dev-azure.example
-!.env.local.example
 !.env.prod.example
 !.env.dev.legacy.example
 

LICENSE

@@ -2,8 +2,8 @@ Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
-- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- All content that resides under the "ee/", "web/src/ee/", and/or "worker/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Langfuse Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -191,3 +191,40 @@ You can opt-out by setting `TELEMETRY_ENABLED=false`.
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
  </picture>
 </a>
+
+### Open Source Projects Using Langfuse
+
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents/python.md)):
+
+| Repository                                                                                                                                                                                                                                                          | Stars |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 53464 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 49387 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37087 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 36690 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26871 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22348 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 14508 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7319 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5352 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2084 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  1936 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1140 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1044 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   945 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   770 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   675 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   583 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   505 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   383 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   379 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   357 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   300 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   278 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   237 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   164 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   133 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   122 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/134601687?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langfuse](https://github.com/langfuse) / [langfuse-python](https://github.com/langfuse/langfuse-python)                            |   108 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/42655961?s=40&v=4" width="20" height="20" alt=""> &nbsp; [zozoheir](https://github.com/zozoheir) / [tinyllm](https://github.com/zozoheir/tinyllm)                                             |    95 |

docker-compose.build.yml

@@ -1,34 +1,53 @@
-version: "3.5"
-
 services:
-  server:
+  langfuse-web:
     build:
       dockerfile: ./web/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: &langfuse-depends-on
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
     ports:
       - "3000:3000"
-    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+    environment: &langfuse-web-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      NEXTAUTH_SECRET: mysecret
+      SALT: mysalt
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: http://localhost:3000
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -36,26 +55,17 @@ services:
       timeout: 10s
       retries: 3
 
-  worker:
+  langfuse-worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: *langfuse-depends-on
     ports:
       - "3030:3030"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - PORT=${PORT:-3030}
-      - REDIS_HOST=${REDIS_HOST:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      <<: *langfuse-web-env
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
@@ -63,26 +73,86 @@ services:
       timeout: 10s
       retries: 3
 
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
-    image: redis:7.2.4
+    image: redis:7
+    restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
-    restart: always
     ports:
       - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
     driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
+    driver: local
+

docker-compose.v3preview.yml

@@ -1,145 +0,0 @@
-services:
-  langfuse-worker:
-    image: langfuse/langfuse-worker:latest
-    depends_on: &langfuse-depends-on
-      postgres:
-        condition: service_healthy
-      minio:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "3030:3030"
-    environment: &langfuse-worker-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      SALT: "mysalt"
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
-      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
-      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
-      LANGFUSE_ASYNC_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_INGESTION_PROCESSING:-true}
-      LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:-true}
-      LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: ${LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE:-true}
-      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
-      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
-      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
-      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
-      LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENABLED: ${LANGFUSE_S3_MEDIA_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
-
-  langfuse-web:
-    image: langfuse/langfuse:latest
-    depends_on: *langfuse-depends-on
-    ports:
-      - "3000:3000"
-    environment:
-      <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
-      NEXTAUTH_SECRET: mysecret
-      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
-      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
-      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
-      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
-      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
-      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
-      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-
-  clickhouse:
-    image: clickhouse/clickhouse-server
-    user: "101:101"
-    container_name: clickhouse
-    hostname: clickhouse
-    environment:
-      CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
-    volumes:
-      - langfuse_clickhouse_data:/var/lib/clickhouse
-      - langfuse_clickhouse_logs:/var/log/clickhouse-server
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    depends_on:
-      - postgres
-
-  minio:
-    image: minio/minio
-    container_name: minio
-    entrypoint: sh
-    # create the 'langfuse' bucket before starting the service
-    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
-    environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
-    ports:
-      - "9090:9000"
-      - "9091:9001"
-    volumes:
-      - langfuse_minio_data:/data
-    healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
-      interval: 1s
-      timeout: 5s
-      retries: 5
-      start_period: 1s
-
-  redis:
-    image: redis:7
-    restart: always
-    command: >
-      --requirepass ${REDIS_AUTH:-myredissecret}
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 3s
-      timeout: 10s
-      retries: 10
-
-  postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
-    restart: always
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 3s
-      timeout: 3s
-      retries: 10
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
-    ports:
-      - 5432:5432
-    volumes:
-      - langfuse_postgres_data:/var/lib/postgresql/data
-
-volumes:
-  langfuse_postgres_data:
-    driver: local
-  langfuse_clickhouse_data:
-    driver: local
-  langfuse_clickhouse_logs:
-    driver: local
-  langfuse_minio_data:
-    driver: local

docker-compose.yml

@@ -1,31 +1,127 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:main # TODO: Switch to :3 after go-live
+    depends_on: &langfuse-depends-on
+      postgres:
         condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:main # TODO: Switch to :3 after go-live
+    depends_on: *langfuse-depends-on
     ports:
       - "3000:3000"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
 
-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -33,14 +129,20 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

ee/src/env.ts

@@ -1,12 +1,9 @@
 import { z } from "zod";
-import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
+import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = {
-  ...sharedEnv,
-  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
-};
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

fern/apis/server/definition/comments.yml

@@ -58,7 +58,7 @@ types:
         docs: The id of the object to attach the comment to. If this does not reference a valid existing object, an error will be thrown.
       content:
         type: string
-        docs: The content of the comment. May include markdown. Currently limited to 500 characters.
+        docs: The content of the comment. May include markdown. Currently limited to 3000 characters.
       authorUserId:
         type: optional<string>
         docs: The id of the user who created the comment.

fern/apis/server/definition/media.yml

@@ -99,5 +99,63 @@ types:
         docs: The unique langfuse identifier of a media record
 
   MediaContentType:
-    type: literal<"image/png","image/jpeg","image/jpg","image/webp","audio/mpeg","audio/mp3","audio/wav","text/plain","application/pdf">
+    enum:
+      - value: image/png
+        name: IMAGE_PNG
+      - value: image/jpeg
+        name: IMAGE_JPEG
+      - value: image/jpg
+        name: IMAGE_JPG
+      - value: image/webp
+        name: IMAGE_WEBP
+      - value: image/gif
+        name: IMAGE_GIF
+      - value: image/svg+xml
+        name: IMAGE_SVG_XML
+      - value: image/tiff
+        name: IMAGE_TIFF
+      - value: image/bmp
+        name: IMAGE_BMP
+      - value: audio/mpeg
+        name: AUDIO_MPEG
+      - value: audio/mp3
+        name: AUDIO_MP3
+      - value: audio/wav
+        name: AUDIO_WAV
+      - value: audio/ogg
+        name: AUDIO_OGG
+      - value: audio/oga
+        name: AUDIO_OGA
+      - value: audio/aac
+        name: AUDIO_AAC
+      - value: audio/mp4
+        name: AUDIO_MP4
+      - value: audio/flac
+        name: AUDIO_FLAC
+      - value: video/mp4
+        name: VIDEO_MP4
+      - value: video/webm
+        name: VIDEO_WEBM
+      - value: text/plain
+        name: TEXT_PLAIN
+      - value: text/html
+        name: TEXT_HTML
+      - value: text/css
+        name: TEXT_CSS
+      - value: text/csv
+        name: TEXT_CSV
+      - value: application/pdf
+        name: APPLICATION_PDF
+      - value: application/msword
+        name: APPLICATION_MSWORD
+      - value: application/vnd.ms-excel
+        name: APPLICATION_MS_EXCEL
+      - value: application/zip
+        name: APPLICATION_ZIP
+      - value: application/json
+        name: APPLICATION_JSON
+      - value: application/xml
+        name: APPLICATION_XML
+      - value: application/octet-stream
+        name: APPLICATION_OCTET_STREAM
     docs: The MIME type of the media record

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.92.0",
+  "version": "3.0.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/down.sh

@@ -19,16 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
-
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
-else
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
@@ -37,4 +28,13 @@ else
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+else
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
 fi

packages/shared/clickhouse/scripts/seed.ts

@@ -2,7 +2,7 @@ import {
   clickhouseClient,
   ObservationRecordReadType,
 } from "@langfuse/shared/src/server";
-import { Prisma, prisma } from "../../src/db";
+import { prisma } from "../../src/db";
 import { redis } from "@langfuse/shared/src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
 import { createDatasets } from "../../prisma/seed";

packages/shared/clickhouse/scripts/up.sh

@@ -19,16 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
-
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
-else
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
@@ -37,4 +28,13 @@ else
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+else
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
 fi

packages/shared/package.json

@@ -64,6 +64,7 @@
     "@langchain/anthropic": "^0.3.8",
     "@langchain/aws": "^0.1.2",
     "@langchain/core": "^0.3.18",
+    "@langchain/google-vertexai": "^0.1.3",
     "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "@prisma/client": "^5.22.0",

packages/shared/prisma/generated/types.ts

@@ -453,6 +453,7 @@ export type Project = {
     org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    deleted_at: Timestamp | null;
     name: string;
 };
 export type ProjectMembership = {

packages/shared/prisma/migrations/20241124115100_add_projects_deleted_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN "deleted_at" TIMESTAMP(3);

packages/shared/prisma/migrations/20241206115829_remove_trace_score_observation_constraints/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_output_score_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/schema.prisma

@@ -113,6 +113,7 @@ model Project {
   orgId               String                 @map("org_id")
   createdAt           DateTime               @default(now()) @map("created_at")
   updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+  deletedAt           DateTime?              @map("deleted_at")
   name                String
   projectMembers      ProjectMembership[]
   organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
@@ -468,25 +469,24 @@ enum ObservationLevel {
 }
 
 model Score {
-  id            String         @id @default(cuid())
-  timestamp     DateTime       @default(now())
-  projectId     String         @map("project_id")
-  project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  id            String        @id @default(cuid())
+  timestamp     DateTime      @default(now())
+  projectId     String        @map("project_id")
+  project       Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        ScoreSource
-  authorUserId  String?        @map("author_user_id")
+  authorUserId  String?       @map("author_user_id")
   comment       String?
-  traceId       String         @map("trace_id")
-  observationId String?        @map("observation_id")
-  configId      String?        @map("config_id")
-  stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
-  queueId       String?        @map("queue_id")
-  createdAt     DateTime       @default(now()) @map("created_at")
-  updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
-  dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
-  JobExecution  JobExecution[]
-  scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
+  traceId       String        @map("trace_id")
+  observationId String?       @map("observation_id")
+  configId      String?       @map("config_id")
+  stringValue   String?       @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+  queueId       String?       @map("queue_id")
+  createdAt     DateTime      @default(now()) @map("created_at")
+  updatedAt     DateTime      @default(now()) @updatedAt @map("updated_at")
+  dataType      ScoreDataType @default(NUMERIC) @map("data_type")
+  scoreConfig   ScoreConfig?  @relation(fields: [configId], references: [id], onDelete: SetNull)
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -900,10 +900,9 @@ model JobExecution {
 
   jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
 
-  jobInputDatasetItemId String?      @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
+  jobInputDatasetItemId String? @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
-  score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
   @@index([projectId, status])
   @@index([projectId, id])

packages/shared/prisma/seed.ts

@@ -450,6 +450,9 @@ export async function createDatasets(
           Math.random() > 0.3
             ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
+        if (!sourceObservation) {
+          continue;
+        }
         const datasetItem = await prisma.datasetItem.create({
           data: {
             projectId,
@@ -510,6 +513,9 @@ export async function createDatasets(
               Math.floor(Math.random() * relevantObservations.length)
             ];
 
+          if (!observation) {
+            continue;
+          }
           await prisma.datasetRunItems.create({
             data: {
               projectId,

packages/shared/scripts/prepareClickhouse.ts

@@ -110,7 +110,7 @@ export const prepareClickhouse = async (
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
       toDecimal64(randUniform(0, 2000), 12) AS total_cost,
-      start_time AS completion_start_time,
+      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
       array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
         ",",
       )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,

packages/shared/src/env.ts

@@ -27,15 +27,19 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
-  CLICKHOUSE_URL: z.string().url().optional(),
-  CLICKHOUSE_USER: z.string().optional(),
-  CLICKHOUSE_PASSWORD: z.string().optional(),
-  LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
+  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_USER: z.string(),
+  CLICKHOUSE_PASSWORD: z.string(),
+  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_ASYNC_INGESTION_PROCESSING: z
+  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("true"),
+  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
+
   LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
     .number()
     .nonnegative()
@@ -48,8 +52,9 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
+    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
+  }),
   LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
   LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
   LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
@@ -62,4 +67,7 @@ const EnvSchema = z.object({
   STRIPE_SECRET_KEY: z.string().optional(),
 });
 
-export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));
+export const env: z.infer<typeof EnvSchema> =
+  process.env.DOCKER_BUILD === "1"
+    ? (process.env as any)
+    : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/comments/types.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+const MAX_COMMENT_LENGTH = 3000;
+
 const COMMENT_OBJECT_TYPES = [
   "TRACE",
   "OBSERVATION",
@@ -9,7 +11,7 @@ const COMMENT_OBJECT_TYPES = [
 
 export const CreateCommentData = z.object({
   projectId: z.string(),
-  content: z.string().trim().min(1).max(500),
+  content: z.string().trim().min(1).max(MAX_COMMENT_LENGTH),
   objectId: z.string(),
   objectType: z.enum(COMMENT_OBJECT_TYPES),
 });

packages/shared/src/features/entitlements/plans.ts

@@ -3,15 +3,17 @@ export const planLabels = {
   "cloud:hobby": "Hobby",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
-  "self-hosted:enterprise": "Enterprise",
+  "self-hosted:pro": "Pro (self-hosted)",
+  "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
 
 export type Plan = keyof typeof planLabels;
 
 export const plans = Object.keys(planLabels) as Plan[];
 
-// This function is kept here to ensure consistency when updating plan names in the future.
+// These functions are kept here to ensure consistency when updating plan names in the future.
 export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/index.ts

@@ -7,7 +7,6 @@ export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
 export * from "./tableDefinitions/tracesTable";
-export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
@@ -16,7 +15,6 @@ export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
-export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -10,3 +10,19 @@ export const BedrockCredentialSchema = z
   })
   .optional();
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
+
+export const GCPServiceAccountKeySchema = z.object({
+  type: z.literal("service_account"),
+  project_id: z.string(),
+  private_key_id: z.string(),
+  private_key: z.string(),
+  client_email: z.string(),
+  client_id: z.string(),
+  auth_uri: z.string(),
+  token_uri: z.string(),
+  auth_provider_x509_cert_url: z.string(),
+  client_x509_cert_url: z.string(),
+});
+
+export type GCPServiceAccountKey = z.infer<typeof GCPServiceAccountKeySchema>;
+export default GCPServiceAccountKeySchema;

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/index.ts

@@ -6,6 +6,7 @@ export * from "./services/PromptService";
 export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -21,8 +22,10 @@ export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
-export * from "./redis/CloudUsageMeteringQueue";
+export * from "./redis/cloudUsageMeteringQueue";
 export * from "./redis/getQueue";
+export * from "./redis/traceDelete";
+export * from "./redis/projectDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
@@ -39,3 +42,6 @@ export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
+
+// test utils
+export * from "./test-utils";

packages/shared/src/server/ingestion/legacy/index.ts

@@ -83,7 +83,7 @@ export const handleBatch = async (
 
 async function retry<T>(request: () => Promise<T>): Promise<T> {
   return await backOff(request, {
-    numOfAttempts: env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" ? 5 : 3,
+    numOfAttempts: 5,
     retry: (e: Error, attemptNumber: number) => {
       if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
         logger.info("not retrying auth error");

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -156,91 +156,106 @@ export const processEventBatch = async (
    * ASYNC PROCESSING *
    ********************/
   let s3UploadErrored = false;
-  if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-    await instrumentAsync({ name: "s3-upload-events" }, async () => {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      // S3 Event Upload is currently blocking, but non-failing.
-      // If a promise rejects, we log it below, but do not throw an error.
-      // In this case, we upload the full batch into the Redis queue.
-      const results = await Promise.allSettled(
-        Object.keys(sortedBatchByEventBodyId).map(async (id) => {
-          // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
-          // That way we batch updates from the same invocation into a single file and reduce
-          // write operations on S3.
-          const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-          return s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
-            data,
-          );
-        }),
-      );
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          s3UploadErrored = true;
-          logger.error("Failed to upload event to S3", {
-            error: result.reason,
-          });
-        }
-      });
-    });
-  }
-
-  // Send each event individually to IngestionQueue for new processing
-  if (
-    env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" &&
-    env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" &&
-    env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING === "true" &&
-    redis &&
-    !s3UploadErrored
-  ) {
-    const queue = IngestionQueue.getInstance();
+  await instrumentAsync({ name: "s3-upload-events" }, async () => {
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    // S3 Event Upload is blocking, but non-failing.
+    // If a promise rejects, we log it below, but do not throw an error.
+    // In this case, we upload the full batch into the Redis queue.
     const results = await Promise.allSettled(
-      Object.keys(sortedBatchByEventBodyId).map(async (id) =>
-        queue
-          ? queue.add(
-              QueueJobs.IngestionJob,
-              {
-                id: randomUUID(),
-                timestamp: new Date(),
-                name: QueueJobs.IngestionJob as const,
-                payload: {
-                  data: {
-                    type: sortedBatchByEventBodyId[id].type,
-                    eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
-                  },
-                  authCheck,
-                },
-              },
-              {
-                delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
-              },
-            )
-          : Promise.reject("Failed to instantiate queue"),
-      ),
+      Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+        // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+        // That way we batch updates from the same invocation into a single file and reduce
+        // write operations on S3.
+        const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+        return s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+          data,
+        );
+      }),
     );
     results.forEach((result) => {
       if (result.status === "rejected") {
-        logger.error("Failed to add event to IngestionQueue", {
+        s3UploadErrored = true;
+        logger.error("Failed to upload event to S3", {
           error: result.reason,
         });
       }
     });
+  });
+
+  // This is a workaround to allow us to disable async ingestion processing for SDK CI testing
+  // TODO: remove this block after SDKs are ready for V3 async ingestion processing
+  if (env.LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED === "true") {
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
   }
 
-  // As part of the legacy processing we sent the entire batch to the worker.
-  if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
-    const queue = LegacyIngestionQueue.getInstance();
+  // Send each event individually to IngestionQueue for ClickHouse processing
+  if (env.LANGFUSE_CLICKHOUSE_INGESTION_ENABLED === "true") {
+    if (s3UploadErrored) {
+      throw new Error(
+        "Failed to upload events to blob storage, aborting event processing",
+      );
+    }
 
-    if (queue) {
-      let addToQueueFailed = false;
+    if (redis) {
+      const queue = IngestionQueue.getInstance();
+      await Promise.all(
+        Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+          queue
+            ? queue.add(
+                QueueJobs.IngestionJob,
+                {
+                  id: randomUUID(),
+                  timestamp: new Date(),
+                  name: QueueJobs.IngestionJob as const,
+                  payload: {
+                    data: {
+                      type: sortedBatchByEventBodyId[id].type,
+                      eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                    },
+                    authCheck,
+                  },
+                },
+                {
+                  delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+                },
+              )
+            : Promise.reject("Failed to instantiate queue"),
+        ),
+      );
+      if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED !== "true") {
+        // If postgres ingestion is disabled, we return early
+        return aggregateBatchResult(
+          [...validationErrors, ...authenticationErrors],
+          sortedBatch.map((event) => ({ id: event.id, result: event })),
+          authCheck.scope.projectId,
+        );
+      }
+    }
+  }
 
-      const queuePayload: LegacyIngestionEventType =
-        env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" && !s3UploadErrored
+  if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED === "true") {
+    // As part of the legacy processing we sent the entire batch to the worker.
+    if (redis) {
+      const queue = LegacyIngestionQueue.getInstance();
+
+      if (queue) {
+        let addToQueueFailed = false;
+
+        const queuePayload: LegacyIngestionEventType = !s3UploadErrored
           ? {
               data: Object.keys(sortedBatchByEventBodyId).map((id) => {
                 const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
@@ -255,44 +270,55 @@ export const processEventBatch = async (
             }
           : { data: sortedBatch, authCheck, useS3EventStore: false };
 
-      try {
-        await queue.add(QueueJobs.LegacyIngestionJob, {
-          payload: queuePayload,
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.LegacyIngestionJob as const,
-        });
-      } catch (e: unknown) {
-        logger.warn(
-          "Failed to add batch to queue, falling back to sync processing",
-          e,
-        );
-        addToQueueFailed = true;
-      }
+        try {
+          await queue.add(QueueJobs.LegacyIngestionJob, {
+            payload: queuePayload,
+            id: randomUUID(),
+            timestamp: new Date(),
+            name: QueueJobs.LegacyIngestionJob as const,
+          });
+        } catch (e: unknown) {
+          logger.warn(
+            "Failed to add batch to queue, falling back to sync processing",
+            e,
+          );
+          addToQueueFailed = true;
+        }
 
-      if (!addToQueueFailed) {
-        return aggregateBatchResult(
-          // we are not sending additional server errors to the client in case of early return
-          [...validationErrors, ...authenticationErrors],
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
+        if (!addToQueueFailed) {
+          return aggregateBatchResult(
+            // we are not sending additional server errors to the client in case of early return
+            [...validationErrors, ...authenticationErrors],
+            sortedBatch.map((event) => ({ id: event.id, result: event })),
+            authCheck.scope.projectId,
+          );
+        }
+      } else {
+        logger.warn(
+          "Ingestion queue not initialized, falling back to sync processing",
         );
       }
-    } else {
-      logger.error(
-        "Ingestion queue not initialized, falling back to sync processing",
-      );
     }
+
+    /*******************
+     * SYNC PROCESSING *
+     *******************/
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
   }
 
-  /*******************
-   * SYNC PROCESSING *
-   *******************/
-  const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
-
-  //  in case we did not return early, we return the result here
-  return aggregateBatchResult(
-    [...validationErrors, ...authenticationErrors, ...result.errors],
-    result.results,
+  throw new Error(
+    "Either Clickhouse or Postgres ingestion (or both) must be enabled",
   );
 };
 
@@ -336,6 +362,7 @@ const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
 export const aggregateBatchResult = (
   errors: Array<{ id: string; error: unknown }>,
   results: Array<{ id: string; result: unknown }>,
+  projectId?: string,
 ) => {
   const returnedErrors: {
     id: string;
@@ -382,7 +409,10 @@ export const aggregateBatchResult = (
 
   if (returnedErrors.length > 0) {
     traceException(errors);
-    logger.error("Error processing events", returnedErrors);
+    logger.error("Error processing events", {
+      errors: returnedErrors,
+      "langfuse.project.id": projectId,
+    });
   }
 
   results.forEach((result) => {

packages/shared/src/server/ingestion/types.ts

@@ -59,7 +59,7 @@ export const usage = MixedUsage.nullish()
 export const TraceBody = z.object({
   id: z.string().nullish(),
   timestamp: stringDateTime,
-  name: z.string().nullish(),
+  name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -17,7 +17,7 @@ type ValidateAndInflateScoreParams = {
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
 ): Promise<Score> {
-  const { body, projectId } = params;
+  const { body, projectId, scoreId } = params;
 
   if (body.configId) {
     const config = await prisma.scoreConfig.findFirst({
@@ -32,10 +32,22 @@ export async function validateAndInflateScore(
         "The configId you provided does not match a valid config in this project",
       );
 
-    validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+    // Override some fields in the score body with config fields
+    // We ignore the set fields in the body
+    const bodyWithConfigOverrides = {
+      ...body,
+      name: config.name,
+    };
+
+    validateConfigAgainstBody(
+      bodyWithConfigOverrides,
+      config as ValidatedScoreConfig,
+    );
 
     return inflateScoreBody({
-      ...params,
+      projectId,
+      scoreId,
+      body: bodyWithConfigOverrides,
       config: config as ValidatedScoreConfig,
     });
   }

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -3,6 +3,7 @@ import type { ZodSchema } from "zod";
 import { CallbackHandler } from "langfuse-langchain";
 
 import { ChatAnthropic } from "@langchain/anthropic";
+import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
   AIMessage,
@@ -15,7 +16,7 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
-import {
+import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
@@ -139,7 +140,11 @@ export async function fetchLLMCompletion(
     return new AIMessage(message.content);
   });
 
-  let chatModel: ChatOpenAI | ChatAnthropic | ChatBedrockConverse;
+  let chatModel:
+    | ChatOpenAI
+    | ChatAnthropic
+    | ChatBedrockConverse
+    | ChatVertexAI;
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
@@ -191,6 +196,21 @@ export async function fetchLLMCompletion(
       callbacks: finalCallbacks,
       maxRetries,
     });
+  } else if (modelParams.adapter === LLMAdapter.VertexAI) {
+    const credentials = GCPServiceAccountKeySchema.parse(JSON.parse(apiKey));
+
+    chatModel = new ChatVertexAI({
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxOutputTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks: finalCallbacks,
+      maxRetries,
+      authOptions: {
+        projectId: credentials.project_id,
+        credentials,
+      },
+    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;

packages/shared/src/server/llm/types.ts

@@ -16,6 +16,7 @@ export enum LLMAdapter {
   OpenAI = "openai",
   Azure = "azure",
   Bedrock = "bedrock",
+  VertexAI = "vertex-ai",
 }
 
 export enum ChatMessageRole {
@@ -114,10 +115,18 @@ export const anthropicModels = [
   "claude-instant-1.2",
 ] as const;
 
+export const vertexAIModels = [
+  "gemini-1.5-pro",
+  "gemini-1.5-flash",
+  "gemini-1.0-pro",
+] as const;
+
 export type AnthropicModel = (typeof anthropicModels)[number];
+export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
   [LLMAdapter.Anthropic]: anthropicModels,
   [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.VertexAI]: vertexAIModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
 } as const;

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -161,8 +161,8 @@ export class StringOptionsFilter implements Filter {
     return {
       query:
         this.operator === "any of"
-          ? `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`
-          : `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`,
+          ? `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} IN ({${varName}: Array(String)})`
+          : `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} NOT IN ({${varName}: Array(String)})`,
       params: { [varName]: this.values },
     };
   }

packages/shared/src/server/queries/clickhouse-sql/orderby-factory.ts

@@ -3,31 +3,53 @@ import { OrderByState } from "../../../interfaces/orderBy";
 import { UiColumnMapping } from "../../../tableDefinitions";
 import { logger } from "../../logger";
 
+type OrderByStateNotNull = Exclude<OrderByState, null>;
+
 export function orderByToClickhouseSql(
-  orderBy: OrderByState,
+  orderBy: OrderByState | OrderByState[] = [],
   tableColumns: UiColumnMapping[],
 ): string {
-  if (!orderBy) {
+  if (
+    !orderBy ||
+    (Array.isArray(orderBy) && orderBy.filter(Boolean).length === 0)
+  ) {
     return "";
   }
-  // Get column definition to map column to internal name, e.g. "t.id"
-  const col = tableColumns.find(
-    (c) => c.uiTableName === orderBy.column || c.uiTableId === orderBy.column,
-  );
 
-  if (!col) {
-    logger.warn("Invalid order by column", orderBy.column);
-    throw new Error("Invalid order by column: " + orderBy.column);
+  if (!Array.isArray(orderBy)) {
+    orderBy = [orderBy];
+  }
+  // Initialize an array to hold order by clauses
+  const orderByClauses: string[] = [];
+
+  // Loop through each orderBy entry
+  for (const ob of orderBy.filter((o): o is OrderByStateNotNull =>
+    Boolean(o),
+  )) {
+    // Get column definition to map column to internal name, e.g. "t.id"
+    const col = tableColumns.find(
+      (c) => c.uiTableName === ob.column || c.uiTableId === ob.column,
+    );
+
+    if (!col) {
+      logger.warn("Invalid order by column", ob.column);
+      throw new Error("Invalid order by column: " + ob.column);
+    }
+
+    // Assert that ob.order is either "asc" or "desc"
+    const orderByOrder = z.enum(["ASC", "DESC"]);
+    const order = orderByOrder.safeParse(ob.order);
+    if (!order.success) {
+      logger.warn("Invalid order", ob.order);
+      throw new Error("Invalid order: " + ob.order);
+    }
+
+    // Append the order by clause to the array
+    orderByClauses.push(
+      `${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`,
+    );
   }
 
-  // Assert that orderBy.order is either "asc" or "desc"
-  const orderByOrder = z.enum(["ASC", "DESC"]);
-  const order = orderByOrder.safeParse(orderBy.order);
-  if (!order.success) {
-    logger.warn("Invalid order", orderBy.order);
-    throw new Error("Invalid order: " + orderBy.order);
-  }
-
-  // Both column and order are safe, can use raw SQL
-  return `ORDER BY ${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`;
+  // Join all order by clauses with a comma and return
+  return `ORDER BY ${orderByClauses.join(", ")}`;
 }

packages/shared/src/server/queries/index.ts

@@ -20,3 +20,4 @@ export {
   NullFilter,
   type ClickhouseOperator,
 } from "./clickhouse-sql/clickhouse-filter";
+export { orderByToClickhouseSql } from "./clickhouse-sql/orderby-factory";

packages/shared/src/server/queues.ts

@@ -63,10 +63,18 @@ export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
 });
-export const TraceUpsertEventSchema = z.object({
+export const TraceQueueEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
+export const TracesQueueEventSchema = z.object({
+  projectId: z.string(),
+  traceIds: z.array(z.string()),
+});
+export const ProjectQueueEventSchema = z.object({
+  projectId: z.string(),
+  orgId: z.string(),
+});
 export const DatasetRunItemUpsertEventSchema = z.object({
   projectId: z.string(),
   datasetItemId: z.string(),
@@ -87,7 +95,9 @@ export const ExperimentCreateEventSchema = z.object({
 });
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
-export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
+export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
+export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
 >;
@@ -101,7 +111,7 @@ export type ExperimentCreateEventType = z.infer<
 export const EventBodySchema = z.union([
   z.object({
     name: z.literal(EventName.TraceUpsert),
-    payload: z.array(TraceUpsertEventSchema),
+    payload: z.array(TraceQueueEventSchema),
   }),
   z.object({
     name: z.literal(EventName.EvaluationExecution),
@@ -120,6 +130,8 @@ export type EventBodyType = z.infer<typeof EventBodySchema>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
   DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
@@ -131,6 +143,8 @@ export enum QueueName {
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
@@ -145,9 +159,21 @@ export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
     timestamp: Date;
     id: string;
-    payload: TraceUpsertEventType;
+    payload: TraceQueueEventType;
     name: QueueJobs.TraceUpsert;
   };
+  [QueueName.TraceDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: TracesQueueEventType | TraceQueueEventType;
+    name: QueueJobs.TraceDelete;
+  };
+  [QueueName.ProjectDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: ProjectQueueEventType;
+    name: QueueJobs.ProjectDelete;
+  };
   [QueueName.DatasetRunItemUpsert]: {
     timestamp: Date;
     id: string;

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -1,5 +1,5 @@
 import { Queue } from "bullmq";
-import { env } from "../..";
+import { env } from "../../env";
 import { logger } from "@azure/storage-blob";
 import { QueueName, QueueJobs } from "../queues";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";

packages/shared/src/server/redis/getQueue.ts

@@ -1,13 +1,15 @@
 import { Queue } from "bullmq";
 import { QueueName } from "../queues";
 import { BatchExportQueue } from "./batchExport";
-import { CloudUsageMeteringQueue } from "./CloudUsageMeteringQueue";
+import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
 import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
 import { IngestionQueue } from "./ingestionQueue";
 import { LegacyIngestionQueue } from "./legacyIngestion";
 import { TraceUpsertQueue } from "./traceUpsert";
+import { TraceDeleteQueue } from "./traceDelete";
+import { ProjectDeleteQueue } from "./projectDelete";
 
 export function getQueue(queueName: QueueName): Queue | null {
   switch (queueName) {
@@ -25,8 +27,12 @@ export function getQueue(queueName: QueueName): Queue | null {
       return ExperimentCreateQueue.getInstance();
     case QueueName.TraceUpsert:
       return TraceUpsertQueue.getInstance();
+    case QueueName.TraceDelete:
+      return TraceDeleteQueue.getInstance();
     case QueueName.IngestionQueue:
       return IngestionQueue.getInstance();
+    case QueueName.ProjectDelete:
+      return ProjectDeleteQueue.getInstance();
     default:
       const exhaustiveCheckDefault: never = queueName;
       throw new Error(`Queue ${queueName} not found`);

packages/shared/src/server/redis/projectDelete.ts

@@ -0,0 +1,45 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class ProjectDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ProjectDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ProjectDelete]
+  > | null {
+    if (ProjectDeleteQueue.instance) return ProjectDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ProjectDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ProjectDelete]>(
+          QueueName.ProjectDelete,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ProjectDeleteQueue.instance?.on("error", (err) => {
+      logger.error("ProjectDeleteQueue error", err);
+    });
+
+    return ProjectDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/traceDelete.ts

@@ -0,0 +1,44 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class TraceDeleteQueue {
+  private static instance: Queue<TQueueJobTypes[QueueName.TraceDelete]> | null =
+    null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.TraceDelete]
+  > | null {
+    if (TraceDeleteQueue.instance) return TraceDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    TraceDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.TraceDelete]>(
+          QueueName.TraceDelete,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    TraceDeleteQueue.instance?.on("error", (err) => {
+      logger.error("TraceDeleteQueue error", err);
+    });
+
+    return TraceDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -1,10 +1,4 @@
-import { randomUUID } from "crypto";
-import {
-  QueueJobs,
-  QueueName,
-  TQueueJobTypes,
-  TraceUpsertEventType,
-} from "../queues";
+import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";
@@ -32,7 +26,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
-              delay: 10_000, // 10 seconds
+              delay: 15_000, // 15 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/repositories/clickhouse.ts

@@ -40,37 +40,31 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // If event upload is enabled, we store all rows in S3 to have a backup
-    if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      await Promise.all(
-        opts.records.map((record) => {
-          // drop trailing s and pretend it's always a create.
-          // Only applicable to scores and traces.
-          let eventType = `${opts.table.slice(0, -1)}-create`;
-          if (opts.table === "observations") {
-            // @ts-ignore - If it's an observation we now that `type` is a string
-            eventType = `${record["type"].toLowerCase()}-create`;
-          }
-          s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
-            [
-              {
-                id: randomUUID(),
-                timestamp: new Date().toISOString(),
-                type: eventType,
-                body: opts.eventBodyMapper(record),
-              },
-            ],
-          );
-        }),
-      );
-    }
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    await Promise.all(
+      opts.records.map((record) => {
+        // drop trailing s and pretend it's always a create.
+        // Only applicable to scores and traces.
+        let eventType = `${opts.table.slice(0, -1)}-create`;
+        if (opts.table === "observations") {
+          // @ts-ignore - If it's an observation we now that `type` is a string
+          eventType = `${record["type"].toLowerCase()}-create`;
+        }
+        s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+          [
+            {
+              id: randomUUID(),
+              timestamp: new Date().toISOString(),
+              type: eventType,
+              body: opts.eventBodyMapper(record),
+            },
+          ],
+        );
+      }),
+    );
 
     const res = await clickhouseClient().insert({
       table: opts.table,

packages/shared/src/server/repositories/index.ts

@@ -7,3 +7,5 @@ export * from "./traces_converters";
 export * from "./scores_converters";
 export * from "./observations_converters";
 export * from "./clickhouse";
+export * from "./constants";
+export * from "./trace-sessions";

packages/shared/src/server/repositories/observations.ts

@@ -210,11 +210,13 @@ export const getObservationById = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const records = await getObservationByIdInternal(
     id,
     projectId,
     fetchWithInputOutput,
+    startTime,
   );
   const mapped = records.map(convertObservation);
 
@@ -311,6 +313,7 @@ const getObservationByIdInternal = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const query = `
   SELECT
@@ -345,11 +348,18 @@ const getObservationByIdInternal = async (
   FROM observations
   WHERE id = {id: String}
   AND project_id = {projectId: String}
+  ${startTime ? `AND start_time = {startTime: DateTime64(3)}` : ""}
   ORDER BY event_ts desc
   LIMIT 1 by id, project_id`;
   return await queryClickhouse<ObservationRecordReadType>({
     query,
-    params: { id, projectId },
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
   });
 };
 
@@ -389,7 +399,7 @@ export type ObservationsTableQueryResult = ObservationRecordReadType & {
 export const getObservationsTableCount = async (opts: ObservationTableQuery) =>
   getObservationsTableInternal<TableCount>({
     ...opts,
-    select: "count(*) as count",
+    select: "count",
   });
 
 export type ObservationsTableRow = Omit<
@@ -407,29 +417,7 @@ export const getObservationsTable = async (
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const traces = await getTracesByIds(
@@ -464,29 +452,7 @@ export const getObservationsTableWithModelData = async (
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const uniqueModels: string[] = Array.from(
@@ -543,17 +509,45 @@ export const getObservationsTableWithModelData = async (
 };
 
 const getObservationsTableInternal = async <T>(
-  opts: ObservationTableQuery & { select: string },
+  opts: ObservationTableQuery & { select: "count" | "rows" },
 ): Promise<Array<T>> => {
+  const select =
+    opts.select === "count"
+      ? "count(*) as count"
+      : `
+        o.id as id,
+        o.project_id as "project_id",
+        o.name as name,
+        o."model_parameters" as model_parameters,
+        o.start_time as "start_time",
+        o.end_time as "end_time",
+        o.trace_id as "trace_id",
+        o.completion_start_time as "completion_start_time",
+        o.provided_usage_details as "provided_usage_details",
+        o.usage_details as "usage_details",
+        o.provided_cost_details as "provided_cost_details",
+        o.cost_details as "cost_details",
+        o.level as level,
+        o.status_message as "status_message",
+        o.version as version,
+        o.parent_observation_id as "parent_observation_id",
+        o.created_at as "created_at",
+        o.updated_at as "updated_at",
+        o.provided_model_name as "provided_model_name",
+        o.total_cost as "total_cost",
+        internal_model_id as "internal_model_id",
+        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`;
+
   const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
     opts;
 
   const selectString = selectIOAndMetadata
     ? `
-    ${opts.select},
+    ${select},
     ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
   `
-    : opts.select;
+    : select;
 
   const scoresFilter = new FilterList([
     new StringFilter({
@@ -580,6 +574,10 @@ const getObservationsTableInternal = async <T>(
         .includes(f.column),
   );
 
+  const hasScoresFilter = filter.some(
+    (f) => f.column === "Scores" || f.column === "scores",
+  );
+
   const orderByTraces = opts.orderBy
     ? observationsTableTraceUiColumnDefinitions
         .map((c) => c.uiTableId)
@@ -650,63 +648,58 @@ const getObservationsTableInternal = async <T>(
       observation_id
   )`;
 
-  if (traceTableFilter.length > 0 || orderByTraces) {
-    // joins with traces are very expensive. We need to filter by time as well.
-    // We assume that a trace has to have been within the last 2 days to be relevant.
+  // if we have default ordering by time, we order by toDate(o.start_time) first and then by
+  // o.start_time. This way, clickhouse is able to read more efficiently directly from disk without ordering
+  const newDefaultOrder =
+    orderBy?.column === "startTime"
+      ? [{ column: "order_by_date", order: orderBy.order }, orderBy]
+      : [orderBy ?? null];
 
-    const query = `
+  const chOrderBy = orderByToClickhouseSql(newDefaultOrder, [
+    ...observationsTableUiColumnDefinitions,
+    {
+      uiTableName: "order_by_date",
+      uiTableId: "order_by_date",
+      clickhouseTableName: "observation",
+      clickhouseSelect: "toDate(o.start_time)",
+    },
+  ]);
+
+  // joins with traces are very expensive. We need to filter by time as well.
+  // We assume that a trace has to have been within the last 2 days to be relevant.
+
+  const query = `
       ${scoresCte}
       SELECT
        ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
+      FROM observations o 
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${hasScoresFilter ? `LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
         AND o.type = 'GENERATION'
-        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
+      ${chOrderBy}
+      ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-        ...(timeFilter
-          ? {
-              tracesTimestampFilter: convertDateToClickhouseDateTime(
-                timeFilter.value as Date,
-              ),
-            }
-          : {}),
-        ...search.params,
-      },
-    });
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedScoresFilter.params,
+      ...appliedObservationsFilter.params,
+      ...(timeFilter
+        ? {
+            tracesTimestampFilter: convertDateToClickhouseDateTime(
+              timeFilter.value as Date,
+            ),
+          }
+        : {}),
+      ...search.params,
+    },
+  });
 
-    return res;
-  } else {
-    // we query by T, which could also be {count: string}.
-    const query = `
-      ${scoresCte}
-      SELECT
-       ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
-      WHERE ${appliedObservationsFilter.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
-      ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
-
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-      },
-    });
-
-    return res;
-  }
+  return res;
 };
 
 export const getObservationsGroupedByModel = async (
@@ -907,6 +900,25 @@ export const deleteObservationsByTraceIds = async (
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteObservationsByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -1069,3 +1081,87 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     latency: Number(r.latency_ms) / 1000,
   }));
 };
+
+export const getObservationCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM observations
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
+export const getObservationCountOfProjectsSinceCreationDate = async ({
+  projectIds,
+  start,
+}: {
+  projectIds: string[];
+  start: Date;
+}) => {
+  const query = `
+    SELECT 
+      count(*) as count
+    FROM observations
+    WHERE project_id IN ({projectIds: Array(String)})
+    AND created_at >= {start: DateTime64(3)}
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectIds,
+      start: convertDateToClickhouseDateTime(start),
+    },
+  });
+
+  return Number(rows[0]?.count ?? 0);
+};
+
+export const getTraceIdsForObservations = async (
+  projectId: string,
+  observationIds: string[],
+) => {
+  const query = `
+    SELECT 
+      trace_id,
+      id
+    FROM observations
+    WHERE project_id = {projectId: String}
+    AND id IN ({observationIds: Array(String)})
+  `;
+
+  const rows = await queryClickhouse<{ id: string; trace_id: string }>({
+    query,
+    params: {
+      projectId,
+      observationIds,
+    },
+  });
+
+  return rows.map((row) => ({
+    id: row.id,
+    traceId: row.trace_id,
+  }));
+};

packages/shared/src/server/repositories/observations_converters.ts

@@ -25,10 +25,7 @@ export const convertObservationToView = (
       ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
-    timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
-      : null,
+
     promptName: record.prompt_name ?? null,
     promptVersion: record.prompt_version ?? null,
     modelId: record.internal_model_id ?? null,
@@ -108,8 +105,10 @@ export const convertObservation = (
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
     timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
+      ? parseClickhouseUTCDateTimeFormat(
+          record.completion_start_time,
+        ).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
   };
 };

packages/shared/src/server/repositories/scores.ts

@@ -5,8 +5,11 @@ import {
   queryClickhouse,
   upsertClickhouse,
 } from "./clickhouse";
-import { FilterList } from "../queries/clickhouse-sql/clickhouse-filter";
-import { FilterState } from "../../types";
+import {
+  Filter,
+  FilterList,
+} from "../queries/clickhouse-sql/clickhouse-filter";
+import { FilterCondition, FilterState, TimeFilter } from "../../types";
 import {
   createFilterFromFilterState,
   getProjectIdDefaultFilter,
@@ -93,6 +96,32 @@ export const getScoreById = async (
   return rows.map(convertToScore).shift();
 };
 
+export const getScoresByIds = async (
+  projectId: string,
+  scoreId: string[],
+  source?: ScoreSource,
+) => {
+  const query = `
+    SELECT *
+    FROM scores s
+    WHERE s.project_id = {projectId: String}
+    AND s.id IN ({scoreId: Array(String)})
+    ${source ? `AND s.source = {source: String}` : ""}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
+  `;
+
+  const rows = await queryClickhouse<ScoreRecordReadType>({
+    query,
+    params: {
+      projectId,
+      scoreId,
+      ...(source !== undefined ? { source } : {}),
+    },
+  });
+  return rows.map(convertToScore);
+};
+
 /**
  * Accepts a score in a Clickhouse-ready format.
  * id, project_id, name, and timestamp must always be provided.
@@ -476,6 +505,25 @@ export const deleteScoresByTraceIds = async (
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteScoresByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -489,10 +537,14 @@ export const getNumericScoreHistogram = async (
   );
   const chFilterRes = chFilter.apply();
 
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
   const query = `
     select s.value
     from scores s
+    ${traceFilter ? `LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
     WHERE s.project_id = {projectId: String}
+    ${traceFilter ? `AND t.project_id = {projectId: String}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     ORDER BY s.event_ts DESC
     LIMIT 1 BY s.id, s.project_id
@@ -549,3 +601,69 @@ export const getAggregatedScoresForPrompts = async (
     promptId: row.prompt_id,
   }));
 };
+
+export const getScoreCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM scores
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
+export const getDistinctScoreNames = async (
+  projectId: string,
+  cutoffCreatedAt: Date,
+  filter: FilterState,
+  isTimestampFilter: (filter: FilterCondition) => filter is TimeFilter,
+) => {
+  const scoreTimestampFilter = filter?.find(isTimestampFilter);
+
+  const query = `
+    SELECT DISTINCT
+      name
+    FROM scores s 
+    WHERE s.project_id = {projectId: String}
+    AND s.created_at <= {cutoffCreatedAt: DateTime64(3)}
+    ${scoreTimestampFilter ? `AND s.timestamp >= {filterTimestamp: DateTime64(3)}` : ""}
+  `;
+
+  const rows = await queryClickhouse<{ name: string }>({
+    query,
+    params: {
+      projectId,
+      cutoffCreatedAt: convertDateToClickhouseDateTime(cutoffCreatedAt),
+      ...(scoreTimestampFilter
+        ? {
+            filterTimestamp: convertDateToClickhouseDateTime(
+              scoreTimestampFilter.value,
+            ),
+          }
+        : {}),
+    },
+  });
+
+  return rows.map((row) => row.name);
+};

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,4 +1,4 @@
-import { ScoreSource, ScoreDataType } from "@prisma/client";
+import { ScoreSource, ScoreDataType, Score } from "@prisma/client";
 import { ScoreRecordReadType } from "./definitions";
 
 export type ScoreAggregation = {
@@ -11,7 +11,7 @@ export type ScoreAggregation = {
   comment: string | null;
 };
 
-export const convertToScore = (row: ScoreRecordReadType) => {
+export const convertToScore = (row: ScoreRecordReadType): Score => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),

packages/shared/src/server/repositories/trace-sessions.ts

@@ -0,0 +1,61 @@
+import z from "zod";
+import { prisma } from "../../db";
+import { singleFilter, stringOptionsFilter } from "../../interfaces/filters";
+
+export const getPublicSessionsFilter = async (
+  projectId: string,
+  filter: z.infer<typeof singleFilter>[],
+) => {
+  const sessionsBookmarkedFilter = filter?.find((f) => f.column === "⭐️");
+
+  // we are only fetching bookmarked sessions.
+  // They need to be manipulated in the UI and should not be as many.
+  const filteredSessions = sessionsBookmarkedFilter
+    ? await prisma.traceSession.findMany({
+        where: {
+          projectId: projectId,
+          bookmarked: true,
+        },
+        select: {
+          id: true,
+          createdAt: true,
+          bookmarked: true,
+          public: true,
+        },
+      })
+    : [];
+
+  const additionalBookmarkFilter: z.infer<typeof stringOptionsFilter>[] =
+    sessionsBookmarkedFilter &&
+    typeof sessionsBookmarkedFilter.value === "boolean" &&
+    ((sessionsBookmarkedFilter.value === true &&
+      sessionsBookmarkedFilter.operator === "=") ||
+      (sessionsBookmarkedFilter.value === false &&
+        sessionsBookmarkedFilter.operator === "<>"))
+      ? [
+          {
+            column: "id",
+            type: "stringOptions" as const,
+            operator: "any of" as const,
+            value: filteredSessions.map((s) => s.id),
+          },
+        ]
+      : sessionsBookmarkedFilter &&
+          typeof sessionsBookmarkedFilter.value === "boolean" &&
+          (sessionsBookmarkedFilter.value === false ||
+            (sessionsBookmarkedFilter.value === true &&
+              sessionsBookmarkedFilter.operator === "<>"))
+        ? [
+            {
+              column: "id",
+              type: "stringOptions" as const,
+              operator: "none of" as const,
+              value: filteredSessions.map((s) => s.id),
+            },
+          ]
+        : [];
+
+  return filter
+    ? [...filter.filter((f) => f.column !== "⭐️"), ...additionalBookmarkFilter]
+    : [...additionalBookmarkFilter];
+};

packages/shared/src/server/repositories/traces.ts

@@ -24,7 +24,6 @@ import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
-import { FetchTracesTableProps } from "../services/traces-ui-table-service";
 
 export const checkTraceExists = async (
   projectId: string,
@@ -151,6 +150,37 @@ export const hasAnyTrace = async (projectId: string) => {
   return rows.length > 0 && Number(rows[0].count) > 0;
 };
 
+export const getTraceCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM traces
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
 export const getTraceById = async (
   traceId: string,
   projectId: string,
@@ -395,7 +425,17 @@ export const getSessionsTable = async (props: {
   return rows;
 };
 
-const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
+export type FetchSessionsTableProps = {
+  select: string;
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
   const { select, projectId, filter, orderBy, limit, page } = props;
 
   const { tracesFilter, scoresFilter, observationsFilter } =
@@ -548,6 +588,25 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteTracesByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM traces
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -589,59 +648,95 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
   if (userIds.length === 0) {
     return [];
   }
+  // this query uses window functions on observations + traces to always get only the first row and thereby remove deduplicates
+  // we filter wherever possible by project id and user id
   const query = `
-    WITH observations_agg AS (
-      SELECT o.trace_id,
-             count(*) as obs_count,
-             sumMap(usage_details) as sum_usage_details,
-             sum(total_cost) as sum_total_cost,
-             anyLast(project_id) as project_id
-      FROM observations o FINAL
-      WHERE o.project_id = {projectId: String}
-      GROUP BY o.trace_id
-    ),
-    user_metric_data AS (
-      SELECT t.user_id,
-             max(t.timestamp) as max_timestamp,
-             min(t.timestamp) as min_timestamp,
-             count(*) as trace_count,
-             sum(o.obs_count) as total_observations,
-             sum(o.sum_total_cost) as session_total_cost,
-             sumMap(o.sum_usage_details)['input'] as session_input_usage,
-             sumMap(o.sum_usage_details)['output'] as session_output_usage,
-             sumMap(o.sum_usage_details)['total'] as session_total_usage
-      FROM traces t FINAL
-      LEFT JOIN observations_agg o
-      ON t.id = o.trace_id 
-      AND t.project_id = o.project_id
-      WHERE t.user_id IS NOT NULL
-      AND t.user_id != ''
-      AND t.user_id IN ({userIds: Array(String)})
-      AND t.project_id = {projectId: String}
-      GROUP BY t.user_id
+      WITH stats as (
+        SELECT
+            t.user_id as user_id,
+            count(distinct o.id) as obs_count,
+            sumMap(usage_details) as sum_usage_details,
+            sum(total_cost) as sum_total_cost,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            count(distinct t.id) as trace_count
+        FROM
+            (
+                SELECT
+                    o.project_id,
+                    o.trace_id,
+                    o.usage_details,
+                    o.total_cost,
+                    id,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY id
+                        ORDER BY
+                            event_ts DESC
+                    ) AS rn
+                FROM
+                    observations o
+                WHERE
+                    o.project_id = {projectId: String }
+                    AND o.trace_id in (
+                        SELECT
+                            distinct id
+                        from
+                            traces
+                        where
+                            user_id IN ({userIds: Array(String) })
+                            AND project_id = {projectId: String }
+                    )
+                    AND o.type = 'GENERATION'
+            ) as o
+            JOIN (
+                SELECT
+                    t.id,
+                    t.user_id,
+                    t.project_id,
+                    t.timestamp,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY id
+                        ORDER BY
+                            event_ts DESC
+                    ) AS rn
+                FROM
+                    traces t
+                WHERE
+                    t.user_id IN ({userIds: Array(String) })
+                    AND t.project_id = {projectId: String }
+            ) as t on t.id = o.trace_id
+            and t.project_id = o.project_id
+        WHERE
+            o.rn = 1
+            and t.rn = 1
+        group by
+            t.user_id
     )
-    SELECT user_id AS userId,
-           min_timestamp as firstTrace,
-           max_timestamp as lastTrace,
-           trace_count as totalTraces,
-           total_observations as totalObservations,
-           session_input_usage as totalPromptTokens,
-           session_output_usage as totalCompletionTokens,
-           session_total_usage as totalTokens,
-           session_total_cost as sumCalculatedTotalCost
-    FROM user_metric_data umd
+    SELECT
+        sum_usage_details [ 'input' ] as input_usage,
+        sum_usage_details [ 'output' ] as output_usage,
+        sum_usage_details [ 'total' ] as total_usage,
+        obs_count,
+        trace_count,
+        user_id,
+        sum_total_cost,
+        max_timestamp,
+        min_timestamp
+    FROM
+        stats
+
   `;
 
-  return queryClickhouse<{
-    userId: string;
-    firstTrace: Date | null;
-    lastTrace: Date | null;
-    totalPromptTokens: bigint;
-    totalCompletionTokens: bigint;
-    totalTokens: bigint;
-    totalObservations: bigint;
-    totalTraces: bigint;
-    sumCalculatedTotalCost: number;
+  const rows = await queryClickhouse<{
+    user_id: string;
+    max_timestamp: string;
+    min_timestamp: string;
+    input_usage: string;
+    output_usage: string;
+    total_usage: string;
+    obs_count: string;
+    trace_count: string;
+    sum_total_cost: string;
   }>({
     query,
     params: {
@@ -649,4 +744,38 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
       userIds,
     },
   });
+  return rows.map((row) => ({
+    userId: row.user_id,
+    maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
+    minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
+    inputUsage: Number(row.input_usage),
+    outputUsage: Number(row.output_usage),
+    totalUsage: Number(row.total_usage),
+    observationCount: Number(row.obs_count),
+    traceCount: Number(row.trace_count),
+    totalCost: Number(row.sum_total_cost),
+  }));
+};
+
+export const getTracesByIdsForAnyProject = async (traceIds: string[]) => {
+  const query = `
+      SELECT id, project_id
+      FROM traces
+      WHERE id IN ({traceIds: Array(String)})
+      ORDER BY event_ts DESC
+      LIMIT 1 by id, project_id;`;
+  const records = await queryClickhouse<{
+    id: string;
+    project_id: string;
+  }>({
+    query,
+    params: {
+      traceIds,
+    },
+  });
+
+  return records.map((record) => ({
+    id: record.id,
+    projectId: record.project_id,
+  }));
 };

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,10 +1,7 @@
-import { ObservationLevel, Trace } from "@prisma/client";
+import { Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
-import Decimal from "decimal.js";
-import { ScoreAggregate } from "../../features/scores";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import { TracesTableReturnType } from "../services/traces-ui-table-service";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -54,53 +51,3 @@ export const convertClickhouseToDomain = (
     externalId: null,
   };
 };
-
-export type TracesAllReturnType = {
-  id: string;
-  timestamp: Date;
-  name: string | null;
-  projectId: string;
-  userId: string | null;
-  release: string | null;
-  version: string | null;
-  public: boolean;
-  bookmarked: boolean;
-  sessionId: string | null;
-  tags: string[];
-};
-
-export const convertToDomain = (row: TracesTableReturnType) => {
-  return {
-    id: row.id,
-    projectId: row.project_id,
-    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
-    tags: row.tags,
-    bookmarked: row.bookmarked,
-    name: row.name ?? null,
-    release: row.release ?? null,
-    version: row.version ?? null,
-    userId: row.user_id ?? null,
-    sessionId: row.session_id ?? null,
-    latency: Number(row.latency),
-    usageDetails: row.usage_details,
-    costDetails: row.cost_details,
-    level: row.level,
-    observationCount: Number(row.observation_count),
-    scoresAvg: row.scores_avg,
-    public: row.public,
-  };
-};
-
-export type TracesMetricsReturnType = {
-  id: string;
-  promptTokens: bigint;
-  completionTokens: bigint;
-  totalTokens: bigint;
-  latency: number | null;
-  level: ObservationLevel;
-  observationCount: bigint;
-  calculatedTotalCost: Decimal | null;
-  calculatedInputCost: Decimal | null;
-  calculatedOutputCost: Decimal | null;
-  scores: ScoreAggregate;
-};

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -13,9 +13,17 @@ import {
 } from "../queries/clickhouse-sql/factory";
 import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
-import { convertToDomain } from "../repositories";
-import { queryClickhouse } from "../repositories/clickhouse";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "../repositories/clickhouse";
 import { TraceRecordReadType } from "../repositories/definitions";
+import {
+  OBSERVATIONS_TO_TRACE_INTERVAL,
+  SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
+} from "../repositories/constants";
+import Decimal from "decimal.js";
+import { ScoreAggregate } from "../../features/scores";
 
 export type TracesTableReturnType = Pick<
   TraceRecordReadType,
@@ -30,7 +38,83 @@ export type TracesTableReturnType = Pick<
   | "session_id"
   | "tags"
   | "public"
-> & {
+>;
+
+export type TracesAllUiReturnType = {
+  id: string;
+  timestamp: Date;
+  name: string | null;
+  projectId: string;
+  userId: string | null;
+  release: string | null;
+  version: string | null;
+  public: boolean;
+  bookmarked: boolean;
+  sessionId: string | null;
+  tags: string[];
+};
+
+export type TracesMetricsUiReturnType = {
+  id: string;
+  projectId: string;
+  promptTokens: bigint;
+  completionTokens: bigint;
+  totalTokens: bigint;
+  latency: number | null;
+  level: ObservationLevel;
+  observationCount: bigint;
+  calculatedTotalCost: Decimal | null;
+  calculatedInputCost: Decimal | null;
+  calculatedOutputCost: Decimal | null;
+  scores: ScoreAggregate;
+};
+
+export const convertToUiTableRows = (
+  row: TracesTableReturnType,
+): TracesAllUiReturnType => {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+    tags: row.tags,
+    bookmarked: row.bookmarked,
+    name: row.name ?? null,
+    release: row.release ?? null,
+    version: row.version ?? null,
+    userId: row.user_id ?? null,
+    sessionId: row.session_id ?? null,
+    public: row.public,
+  };
+};
+
+export const convertToUITableMetrics = (
+  row: TracesTableMetricsClickhouseReturnType,
+): Omit<TracesMetricsUiReturnType, "scores"> => {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    latency: Number(row.latency),
+    promptTokens: BigInt(row.usage_details?.input ?? 0),
+    completionTokens: BigInt(row.usage_details?.output ?? 0),
+    totalTokens: BigInt(row.usage_details?.total ?? 0),
+    observationCount: BigInt(row.observation_count ?? 0),
+    calculatedTotalCost: row.cost_details?.total
+      ? new Decimal(row.cost_details.total)
+      : null,
+    calculatedInputCost: row.cost_details?.input
+      ? new Decimal(row.cost_details.input)
+      : null,
+    calculatedOutputCost: row.cost_details?.output
+      ? new Decimal(row.cost_details.output)
+      : null,
+    level: row.level,
+  };
+};
+
+export type TracesTableMetricsClickhouseReturnType = {
+  id: string;
+  project_id: string;
+  timestamp: Date;
   level: ObservationLevel;
   observation_count: number | null;
   latency: string | null;
@@ -40,7 +124,7 @@ export type TracesTableReturnType = Pick<
 };
 
 export type FetchTracesTableProps = {
-  select: string;
+  select: "count" | "rows" | "metrics";
   projectId: string;
   filter: FilterState;
   searchQuery?: string;
@@ -58,7 +142,7 @@ export const getTracesTableCount = async (props: {
   page?: number;
 }) => {
   const countRows = await getTracesTableGeneric<{ count: string }>({
-    select: "count(*) as count",
+    select: "count",
     ...props,
   });
 
@@ -69,6 +153,23 @@ export const getTracesTableCount = async (props: {
   return converted.length > 0 ? converted[0].count : 0;
 };
 
+export const getTracesTableMetrics = async (props: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}): Promise<Array<Omit<TracesMetricsUiReturnType, "scores">>> => {
+  const countRows =
+    await getTracesTableGeneric<TracesTableMetricsClickhouseReturnType>({
+      select: "metrics",
+      ...props,
+    });
+
+  return countRows.map(convertToUITableMetrics);
+};
+
 export const getTracesTable = async (
   projectId: string,
   filter: FilterState,
@@ -78,24 +179,7 @@ export const getTracesTable = async (
   page?: number,
 ) => {
   const rows = await getTracesTableGeneric<TracesTableReturnType>({
-    select: `
-    t.id, 
-    t.project_id as project_id, 
-    t.timestamp, 
-    t.tags, 
-    t.bookmarked, 
-    t.name, 
-    t.release, 
-    t.version, 
-    t.user_id, 
-    t.session_id,
-    os.latency_milliseconds / 1000 as latency,
-    os.cost_details as cost_details,
-    os.usage_details as usage_details,
-    os.level as level,
-    os.observation_count as observation_count,
-    s.scores_avg as scores_avg,
-    t.public`,
+    select: "rows",
     projectId,
     filter,
     searchQuery,
@@ -104,13 +188,50 @@ export const getTracesTable = async (
     page,
   });
 
-  return rows.map(convertToDomain);
+  return rows.map(convertToUiTableRows);
 };
 
 const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
   const { select, projectId, filter, orderBy, limit, page, searchQuery } =
     props;
 
+  let sqlSelect: string;
+  switch (select) {
+    case "count":
+      sqlSelect = "count(*) as count";
+      break;
+    case "metrics":
+      sqlSelect = `
+        t.id as id,
+        t.project_id as project_id,
+        t.timestamp as timestamp,
+        os.latency_milliseconds / 1000 as latency,
+        os.cost_details as cost_details,
+        os.usage_details as usage_details,
+        os.level as level,
+        os.observation_count as observation_count,
+        s.scores_avg as scores_avg,
+        t.public as public`;
+      break;
+    case "rows":
+      sqlSelect = `
+        t.id as id,
+        t.project_id as project_id,
+        t.timestamp as timestamp,
+        t.tags as tags,
+        t.bookmarked as bookmarked,
+        t.name as name,
+        t.release as release,
+        t.version as version,
+        t.user_id as user_id,
+        t.session_id as session_id,
+        t.public as public`;
+      break;
+    default:
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
 
@@ -156,27 +277,20 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
       f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
   ) as DateTimeFilter | undefined;
 
-  timeStampFilter
-    ? scoresFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "scores",
-          field: "timestamp",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
+  const requiresScoresJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "scores") !== undefined ||
+    tracesTableUiColumnDefinitions.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "scores";
 
-  timeStampFilter
-    ? observationsFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "observations",
-          field: "start_time",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
+  const requiresObservationsJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "observations") !==
+      undefined ||
+    tracesTableUiColumnDefinitions.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "observations";
 
   const tracesFilterRes = tracesFilter.apply();
   const scoresFilterRes = scoresFilter.apply();
@@ -184,6 +298,36 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
 
   const search = clickhouseSearchCondition(searchQuery);
 
+  const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
+  const orderByCols = [
+    ...tracesTableUiColumnDefinitions,
+    {
+      clickhouseSelect: "toDate(t.timestamp)",
+      uiTableName: "timestamp_to_date",
+      uiTableId: "timestamp_to_date",
+      clickhouseTableName: "observations",
+    },
+  ];
+  const chOrderBy = orderByToClickhouseSql(
+    [
+      defaultOrder
+        ? {
+            column: "timestamp_to_date",
+            order: orderBy.order,
+          }
+        : null,
+      orderBy ?? null,
+    ],
+    orderByCols,
+  );
+
+  // complex query ahead:
+  // - we only join scores and observations if we really need them to speed up default views
+  // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
+  // - we filter the observations and scores as much as possible before joining them to traces.
+  // - we order by todate(timestamp), timestamp per default and do not use FINAL.
+  //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
+
   const query = `
     WITH observations_stats AS (
       SELECT
@@ -200,8 +344,10 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
           sumMap(cost_details) as cost_details,
           trace_id,
           project_id
-      FROM observations FINAL
-      WHERE ${observationFilterRes.query}
+      FROM observations o FINAL 
+      WHERE o.project_id = {projectId: String}
+      ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+      ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
       GROUP BY trace_id, project_id
     ),
     scores_avg AS (
@@ -214,21 +360,28 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
                 trace_id,
                 name,
                 avg(value) avg_value
-        FROM scores final
-        WHERE ${scoresFilterRes.query}
+        FROM scores s FINAL 
+        WHERE project_id = {projectId: String}
+        ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+        ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
         GROUP BY project_id,
                   trace_id,
                   name
       ) tmp
       GROUP BY project_id, trace_id
     )
-    SELECT ${select}
-    FROM traces t final
-    LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
-    LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
-    WHERE ${tracesFilterRes.query}
+    SELECT ${sqlSelect}
+    -- FINAL is used for non default ordering and count.
+    FROM traces t  ${["metrics", "rows"].includes(select) && defaultOrder ? "" : "FINAL"}
+    ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+    ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+    WHERE t.project_id = {projectId: String}
+    ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
     ${search.query}
-    ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
+    ${chOrderBy}
+    -- This is used for metrics and row queries. Count has only one result.
+    -- This is only used for default ordering. Otherwise, we use final.
+    ${["metrics", "rows"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
     ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
   `;
 
@@ -237,6 +390,8 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
     params: {
       limit: limit,
       offset: limit && page ? limit * page : 0,
+      traceTimestamp: timeStampFilter?.value.getTime(),
+      projectId: projectId,
       ...tracesFilterRes.params,
       ...observationFilterRes.params,
       ...scoresFilterRes.params,

packages/shared/src/server/test-utils/clickhouse-helpers.ts

@@ -0,0 +1,32 @@
+import { clickhouseClient } from "../clickhouse/client";
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../repositories/definitions";
+
+export const createTracesCh = async (trace: TraceRecordInsertType[]) => {
+  return await clickhouseClient().insert({
+    table: "traces",
+    format: "JSONEachRow",
+    values: trace,
+  });
+};
+
+export const createObservationsCh = async (
+  observations: ObservationRecordInsertType[],
+) => {
+  return await clickhouseClient().insert({
+    table: "observations",
+    format: "JSONEachRow",
+    values: observations,
+  });
+};
+
+export const createScoresCh = async (scores: ScoreRecordInsertType[]) => {
+  return await clickhouseClient().insert({
+    table: "scores",
+    format: "JSONEachRow",
+    values: scores,
+  });
+};

packages/shared/src/server/test-utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./tracing-factory";
+export * from "./clickhouse-helpers";
+export * from "./org-factory";

packages/shared/src/server/test-utils/org-factory.ts

@@ -0,0 +1,50 @@
+import { v4 } from "uuid";
+import { prisma } from "../../db";
+import { hashSecretKey, getDisplaySecretKey } from "../auth/apiKeys";
+
+export function createBasicAuthHeader(
+  username: string,
+  password: string,
+): string {
+  const base64Credentials = Buffer.from(`${username}:${password}`).toString(
+    "base64",
+  );
+  return `Basic ${base64Credentials}`;
+}
+
+export type CreateOrgProjectAndApiKeyOptions = {
+  projectId?: string;
+};
+export const createOrgProjectAndApiKey = async (
+  props?: CreateOrgProjectAndApiKeyOptions,
+) => {
+  const projectId = props?.projectId ?? v4();
+  const org = await prisma.organization.create({
+    data: {
+      id: v4(),
+      name: v4(),
+    },
+  });
+  await prisma.project.create({
+    data: {
+      id: projectId,
+      name: v4(),
+      orgId: org.id,
+    },
+  });
+  const publicKey = v4();
+  const secretKey = v4();
+
+  const auth = createBasicAuthHeader(publicKey, secretKey);
+  await prisma.apiKey.create({
+    data: {
+      id: v4(),
+      projectId: projectId,
+      publicKey: publicKey,
+      hashedSecretKey: await hashSecretKey(secretKey),
+      displaySecretKey: getDisplaySecretKey(secretKey),
+    },
+  });
+
+  return { projectId, publicKey, secretKey, auth };
+};

packages/shared/src/server/test-utils/tracing-factory.ts

@@ -1,9 +1,9 @@
-import {
-  type ScoreRecordInsertType,
-  type ObservationRecordInsertType,
-  type TraceRecordInsertType,
-} from "@langfuse/shared/src/server";
 import { v4 } from "uuid";
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../repositories/definitions";
 
 export const createTrace = (trace: Partial<TraceRecordInsertType>) => {
   return {

packages/shared/src/server/utils/transforms/transformStreamToCsv.ts

@@ -12,7 +12,7 @@ export function transformStreamToCsv(): Transform {
     transform(
       row: Record<string, any>,
       encoding: BufferEncoding,
-      callback: TransformCallback
+      callback: TransformCallback,
     ): void {
       if (isFirstChunk) {
         // Extract headers from the first object
@@ -23,7 +23,7 @@ export function transformStreamToCsv(): Transform {
 
       // Convert the object to a CSV line and push it
       const csvRow = headers.map((header) => {
-        const field = row[header];
+        const field = row[header] ?? "";
         let str = stringify(field);
 
         // escape and format fields that contain commas

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -68,7 +68,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "timeToFirstToken",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))",
+      "if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time))",
     // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
@@ -77,7 +77,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "latency",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "if(isNull(end_time), NULL, date_diff('seconds', start_time, end_time))",
+      "if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time))",
     // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
@@ -86,7 +86,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "tokensPerSecond",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "usage_details['input'] / date_diff('seconds', start_time, end_time)",
+      "(usage_details['input'] / date_diff('milliseconds', start_time, end_time))",
   },
   {
     uiTableName: "Input Cost ($)",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -71,28 +71,28 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
   {
     uiTableName: "Input Tokens",
     uiTableId: "inputTokens",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'input'), usage_details), usage_details['input'], NULL)",
   },
   {
     uiTableName: "Output Tokens",
     uiTableId: "outputTokens",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'output'), usage_details), usage_details['output'], NULL)",
   },
   {
     uiTableName: "Total Tokens",
     uiTableId: "totalTokens",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
   },
   {
     uiTableName: "Usage",
     uiTableId: "usage",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
   },
@@ -105,7 +105,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
   {
     uiTableName: "Latency (s)",
     uiTableId: "latency",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect: "latency_milliseconds / 1000",
     // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
     clickhouseTypeOverwrite: "Decimal64(3)",
@@ -113,19 +113,19 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
   {
     uiTableName: "Input Cost ($)",
     uiTableId: "inputCost",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect: "cost_details['input']",
   },
   {
     uiTableName: "Output Cost ($)",
     uiTableId: "outputCost",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect: "cost_details['output']",
   },
   {
     uiTableName: "Total Cost ($)",
     uiTableId: "totalCost",
-    clickhouseTableName: "traces",
+    clickhouseTableName: "observations",
     clickhouseSelect: "cost_details['total']",
   },
 ];

packages/shared/src/utils/zod.ts

@@ -26,7 +26,7 @@ export const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
     nestedLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ])
+  ]),
 );
 
 // Root schema that does not allow nulls at the root level
@@ -35,17 +35,17 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
     rootLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ])
+  ]),
 );
 
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1)
+    z.coerce.number().default(1),
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50)
+    z.coerce.number().lte(100).default(50),
   ),
 };
 
@@ -69,8 +69,11 @@ export const paginationMetaResponseZod = z.object({
   totalPages: z.number().int().nonnegative(),
 });
 
-export const noHtmlRegex = /<[^>]*>/;
-export const noHtmlCheck = (value: string) => !noHtmlRegex.test(value);
+export const htmlRegex = /<[^>]*>/;
+export const noHtmlCheck = (value: string) => !htmlRegex.test(value);
+
+const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
+export const noUrlCheck = (value: string) => !urlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);
 
@@ -83,7 +86,7 @@ export const NonEmptyString = z.string().min(1);
  */
 export const validateZodSchema = <T extends z.ZodTypeAny>(
   schema: T,
-  object: z.infer<T>
+  object: z.infer<T>,
 ): z.infer<T> => {
   return schema.parse(object);
 };

web/Dockerfile

@@ -85,8 +85,7 @@ ENV NEXT_MANUAL_SIG_HANDLE true
 # set the CI flag to true to get CI specific logs
 ENV CI true
 
-
-RUN NODE_OPTIONS='--max-old-space-size=4096' turbo run build --filter=web
+RUN NODE_OPTIONS='--max-old-space-size=4096' turbo run build --filter=web...
 
 # Production image, copy all the files and run next
 FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS runner

web/entrypoint.sh

@@ -14,6 +14,12 @@ if [ -z "$DATABASE_URL" ]; then
     fi
 fi
 
+# Check if CLICKHOUSE_URL is not set
+if [ -z "$CLICKHOUSE_URL" ]; then
+    echo "Error: CLICKHOUSE_URL is not configured. Migrating from V2? Check out migration guide at https://langfuse.com/docs/deployment/v3/migrate-v2-to-v3."
+    exit 1
+fi
+
 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
     export DIRECT_URL=$DATABASE_URL
@@ -52,4 +58,4 @@ if [ $status -ne 0 ]; then
 fi
 
 # Run the command passed to the docker image on start
-exec "$@"
+exec "$@"

web/jest.config.mjs

@@ -24,7 +24,7 @@ const serverTestConfig = {
 const asyncServerTestConfig = {
   displayName: "async-server",
   testPathIgnorePatterns: ["__e2e__"],
-  testMatch: ["/**/async/*.servertest.[jt]s?(x)"],
+  testMatch: ["/**/async/**/*.servertest.[jt]s?(x)"],
   testEnvironment: "jest-environment-node",
   setupFilesAfterEnv: ["<rootDir>/src/__tests__/after-teardown.ts"],
   globalTeardown: "<rootDir>/src/__tests__/teardown.ts",

web/next.config.mjs

@@ -28,6 +28,9 @@ const cspHeader = `
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
 `;
 
+// Match rules for Hugging Face
+const huggingFaceHosts = ["huggingface.co", ".*\\.hf\\.space$"];
+
 const reportToHeader = {
   key: "Report-To",
   value: JSON.stringify({
@@ -78,10 +81,6 @@ const nextConfig = {
       {
         source: "/:path*",
         headers: [
-          {
-            key: "x-frame-options",
-            value: "SAMEORIGIN",
-          },
           {
             key: "X-Content-Type-Options",
             value: "nosniff",
@@ -97,6 +96,21 @@ const nextConfig = {
           ...(env.SENTRY_CSP_REPORT_URI ? [reportToHeader] : []),
         ],
       },
+      {
+        source: "/:path*",
+        headers: [
+          {
+            key: "x-frame-options",
+            value: "SAMEORIGIN",
+          },
+        ],
+        // Disable x-frame-options on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
+      },
+      // CSP header
       {
         source: "/:path((?!api).*)*",
         headers: [
@@ -105,26 +119,31 @@ const nextConfig = {
             value: cspHeader.replace(/\n/g, ""),
           },
         ],
+        // Disable CSP on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
       },
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
       // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
       {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.92.0",
+  "version": "3.0.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -16,6 +16,7 @@
     "start": "dotenv -e ../.env -- sh -c 'NEXT_MANUAL_SIG_HANDLE=true next start'",
     "test": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects async-server",
     "test-sync": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects sync-server",
+    "test-client": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
     "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server",
@@ -115,13 +116,13 @@
     "kysely": "^0.27.4",
     "langchain": "^0.3.6",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.447.0",
+    "lucide-react": "^0.462.0",
     "next": "^14.2.15",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.176.0",
-    "posthog-node": "^3.6.3",
+    "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
     "prisma": "^5.22.0",
     "rate-limiter-flexible": "^5.0.3",
@@ -138,7 +139,7 @@
     "remark-gfm": "^4.0.0",
     "remark-math": "^6.0.0",
     "sonner": "^1.4.41",
-    "stripe": "^16.8.0",
+    "stripe": "^17.4.0",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.5.2",
     "tailwindcss-animate": "^1.0.7",

web/public/generated/api/openapi.yml

@@ -2389,7 +2389,7 @@ components:
           type: string
           description: >-
             The content of the comment. May include markdown. Currently limited
-            to 500 characters.
+            to 3000 characters.
         authorUserId:
           type: string
           nullable: true
@@ -4058,8 +4058,35 @@ components:
       title: MediaContentType
       type: string
       enum:
-        - >-
-          image/png","image/jpeg","image/jpg","image/webp","audio/mpeg","audio/mp3","audio/wav","text/plain","application/pdf
+        - image/png
+        - image/jpeg
+        - image/jpg
+        - image/webp
+        - image/gif
+        - image/svg+xml
+        - image/tiff
+        - image/bmp
+        - audio/mpeg
+        - audio/mp3
+        - audio/wav
+        - audio/ogg
+        - audio/oga
+        - audio/aac
+        - audio/mp4
+        - audio/flac
+        - video/mp4
+        - video/webm
+        - text/plain
+        - text/html
+        - text/css
+        - text/csv
+        - application/pdf
+        - application/msword
+        - application/vnd.ms-excel
+        - application/zip
+        - application/json
+        - application/xml
+        - application/octet-stream
       description: The MIME type of the media record
     DailyMetrics:
       title: DailyMetrics

web/public/generated/postman/collection.json

@@ -689,7 +689,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"traceId\": \"example\",\n    \"observationId\": \"example\",\n    \"contentType\": \"example\",\n    \"contentLength\": 0,\n    \"sha256Hash\": \"example\",\n    \"field\": \"example\"\n}",
+              "raw": "{\n    \"traceId\": \"example\",\n    \"observationId\": \"example\",\n    \"contentType\": \"image/png\",\n    \"contentLength\": 0,\n    \"sha256Hash\": \"example\",\n    \"field\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"

web/src/__e2e__/api.servertest.ts

@@ -1,7 +1,8 @@
 import { v4 } from "uuid";
 import { JobExecutionStatus, Prisma, prisma } from "@langfuse/shared/src/db";
 import {
-  clickhouseClient,
+  getObservationById,
+  getTraceById,
   OrgEnrichedApiKey,
   redis,
 } from "@langfuse/shared/src/server";
@@ -15,6 +16,7 @@ const generateAuth = (username: string, password: string) => {
 const workerAdminAuth = generateAuth("admin", "myworkerpassword");
 
 const userApiKeyAuth = generateAuth("pk-lf-1234567890", "sk-lf-1234567890");
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
 
 describe("Health endpoints", () => {
   it("web container returns healthy", async () => {
@@ -61,6 +63,7 @@ describe("Ingestion Pipeline", () => {
           timestamp: new Date().toISOString(),
           body: {
             name: "test trace",
+            timestamp: new Date().toISOString(),
             id: traceId,
             userId: "user-1", // triggers the eval
           },
@@ -73,6 +76,7 @@ describe("Ingestion Pipeline", () => {
             id: spanId,
             traceId: traceId,
             name: "test span",
+            startTime: new Date().toISOString(),
           },
         },
       ],
@@ -103,32 +107,20 @@ describe("Ingestion Pipeline", () => {
           },
         });
 
-        const traces = await clickhouseClient().query({
-          query: "SELECT * FROM traces",
-          format: "JSONEachRow",
-        });
-        console.log("traces", await traces.text());
         expect(traceResponse.status).toBe(200);
         expect(traceResponse.body).not.toBeNull();
         expect((await traceResponse.json()).id).toBe(traceId);
 
-        const trace = await prisma.trace.findUnique({
-          where: {
-            id: traceId,
-          },
-        });
+        const trace = await getTraceById(traceId, projectId);
         expect(trace).not.toBeNull();
         expect(trace?.name).toBe("test trace");
 
-        const observation = await prisma.observation.findUnique({
-          where: {
-            id: spanId,
-            traceId: traceId,
-          },
-        });
+        const observation = await getObservationById(spanId, projectId);
         expect(observation).not.toBeNull();
         expect(observation?.name).toBe("test span");
 
+        console.log("observationFounds", observation);
+
         expect(redis).not.toBeNull();
 
         const redisKeys = await redis?.keys(`api-key:*`);
@@ -153,6 +145,7 @@ describe("Ingestion Pipeline", () => {
             projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           },
         });
+        console.log("evalExecution", evalExecution);
 
         expect(evalExecution).not.toBeNull();
 
@@ -163,12 +156,12 @@ describe("Ingestion Pipeline", () => {
         // failure due to missing openai key in the pipeline. Expected
         expect(evalExecution.status).toBe(JobExecutionStatus.ERROR);
       },
-      50000,
+      60000,
       10000,
     );
 
     expect(response.status).toBe(207);
-  }, 60000);
+  }, 70000);
 
   it("rate limit ingestion", async () => {
     // update the org in the database and set the rate limit to 1 for ingestion

web/src/__e2e__/auth.spec.ts

@@ -22,7 +22,7 @@ test("Successfully sign up & able to go to homepage", async ({ page }) => {
   await page.goto("auth/sign-up");
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', randomEmailAddress());
-  await page.fill('input[type="password"]', "password2");
+  await page.fill('input[type="password"]', "Password2#!");
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
@@ -35,7 +35,7 @@ test("Successfully sign up & able to go to homepage with uppercase email", async
   await page.goto("auth/sign-up");
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', "A" + randomEmailAddress());
-  await page.fill('input[type="password"]', "password3");
+  await page.fill('input[type="password"]', "Password3#!");
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
@@ -50,7 +50,7 @@ test("Signup input validation", async ({ page }) => {
   await page.waitForTimeout(2000);
   await expect(page.getByText("Invalid email")).toBeVisible();
   await expect(
-    page.getByText("Password must be at least 8 characters long"),
+    page.getByText("Password must be at least 8 characters long."),
   ).toBeVisible();
   await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);

web/src/__tests__/aggregateScores.clienttest.ts

@@ -54,6 +54,33 @@ describe("aggregateScores", () => {
     });
   });
 
+  it("should correctly aggregate multiple numeric scores with the same key and value", () => {
+    const scores = [
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "test comment",
+      },
+      {
+        name: "test",
+        source: "API",
+        dataType: "NUMERIC",
+        value: 5,
+        comment: "another comment",
+      },
+    ] as APIScore[];
+    expect(aggregateScores(scores)).toEqual({
+      "test-API-NUMERIC": {
+        type: "NUMERIC",
+        values: [5, 5],
+        average: 5,
+        comment: undefined,
+      },
+    });
+  });
+
   it("should correctly aggregate scores with different keys", () => {
     const scores: APIScore[] = [
       {

web/src/__tests__/async/comments-api.servertest.ts

@@ -1,42 +1,40 @@
 /** @jest-environment node */
 
-import {
-  makeZodVerifiedAPICall,
-  pruneDatabase,
-} from "@/src/__tests__/test-utils";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
 import {
   GetCommentsV1Response,
   GetCommentV1Response,
   PostCommentsV1Response,
 } from "@/src/features/public-api/types/comments";
-import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 import { prisma } from "@langfuse/shared/src/db";
 import { z } from "zod";
+import {
+  createObservationsCh,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { createObservation, createTrace } from "@langfuse/shared/src/server";
 
 describe("Create and get comments", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
+  beforeAll(async () => {
+    const traces = [
+      createTrace({
+        name: "trace-name",
+        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        id: "1234",
+      }),
+    ];
+
+    await createTracesCh(traces);
+  });
 
   it("should create and get comment", async () => {
-    const traceResponse = await makeZodVerifiedAPICall(
-      PostTracesV1Response,
-      "POST",
-      "/api/public/traces",
-      {
-        name: "trace-name",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      },
-    );
-
-    const { id: traceId } = traceResponse.body;
-
     const commentResponse = await makeZodVerifiedAPICall(
       PostCommentsV1Response,
       "POST",
       "/api/public/comments",
       {
         content: "hello",
-        objectId: traceId,
+        objectId: "1234",
         objectType: "TRACE",
         projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         authorUserId: "user-1",
@@ -55,7 +53,7 @@ describe("Create and get comments", () => {
     expect(response.body).toMatchObject({
       id: commentId,
       projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      objectId: traceId,
+      objectId: "1234",
       objectType: "TRACE",
       content: "hello",
       authorUserId: "user-1",
@@ -80,36 +78,81 @@ describe("Create and get comments", () => {
       );
     } catch (error) {
       expect((error as Error).message).toBe(
-        `API call did not return 200, returned status 404, body {\"message\":\"No trace with id invalid-trace-id in project 7a88fb47-b4e2-43b8-a06c-a5ce950dc53a\",\"error\":\"LangfuseNotFoundError\"}`,
+        `API call did not return 200, returned status 404, body {\"message\":\"Reference object, TRACE: invalid-trace-id not found in Clickhouse. Skipping creating comment.\",\"error\":\"LangfuseNotFoundError\"}`,
+      );
+    }
+  });
+
+  it("should fail to create comment if content is empty", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        z.object({
+          message: z.string(),
+          error: z.array(z.object({})),
+        }),
+        "POST",
+        "/api/public/comments",
+        {
+          content: "",
+          objectId: "1234",
+          objectType: "TRACE",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"too_small\",\"minimum\":1,\"type\":\"string\",\"inclusive\":true,\"exact\":false,\"message\":\"String must contain at least 1 character(s)\",\"path\":[\"content\"]}]}`,
+      );
+    }
+  });
+
+  it("should fail to create comment if content is larger than 3000 characters", async () => {
+    try {
+      await makeZodVerifiedAPICall(
+        z.object({
+          message: z.string(),
+          error: z.array(z.object({})),
+        }),
+        "POST",
+        "/api/public/comments",
+        {
+          content: "a".repeat(3001),
+          objectId: "1234",
+          objectType: "TRACE",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        },
+      );
+    } catch (error) {
+      expect((error as Error).message).toBe(
+        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"too_big\",\"maximum\":3000,\"type\":\"string\",\"inclusive\":true,\"exact\":false,\"message\":\"String must contain at most 3000 character(s)\",\"path\":[\"content\"]}]}`,
       );
     }
   });
 });
 
 describe("GET /api/public/comments API Endpoint", () => {
-  beforeEach(async () => {
-    await pruneDatabase();
-
-    const trace = await prisma.trace.create({
-      data: {
-        id: "trace-2021-01-01",
-        createdAt: new Date("2021-01-01T00:00:00Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+  beforeAll(async () => {
+    const traces = [
+      createTrace({
         name: "trace-1",
-      },
+        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        id: "1234",
+      }),
+    ];
+
+    await createTracesCh(traces);
+
+    const observation = createObservation({
+      name: "generation-1",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      id: "5678",
+      type: "GENERATION",
+      trace_id: "1234",
     });
 
-    const generation = await prisma.observation.create({
-      data: {
-        id: "generation-2021-01-01",
-        createdAt: new Date("2021-01-01T00:00:00Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        traceId: trace.id,
-        name: "generation-1",
-        type: "GENERATION",
-      },
-    });
+    await createObservationsCh([observation]);
 
+    await prisma.comment.deleteMany();
     await prisma.comment.createMany({
       data: [
         {
@@ -117,7 +160,7 @@ describe("GET /api/public/comments API Endpoint", () => {
           createdAt: new Date("2021-01-01T00:00:00Z"),
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           content: "comment-1",
-          objectId: trace.id,
+          objectId: "1234",
           objectType: "TRACE",
           authorUserId: "user-1",
         },
@@ -126,7 +169,7 @@ describe("GET /api/public/comments API Endpoint", () => {
           createdAt: new Date("2021-02-01T00:00:00Z"),
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           content: "comment-2",
-          objectId: generation.id,
+          objectId: "5678",
           objectType: "OBSERVATION",
           authorUserId: "user-1",
         },
@@ -135,7 +178,7 @@ describe("GET /api/public/comments API Endpoint", () => {
           createdAt: new Date("2021-03-01T00:00:00Z"),
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           content: "comment-3",
-          objectId: trace.id,
+          objectId: "1234",
           objectType: "TRACE",
           authorUserId: "user-1",
         },
@@ -144,7 +187,7 @@ describe("GET /api/public/comments API Endpoint", () => {
           createdAt: new Date("2021-04-01T00:00:00Z"),
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           content: "comment-4",
-          objectId: trace.id,
+          objectId: "1234",
           objectType: "TRACE",
         },
         {
@@ -152,13 +195,12 @@ describe("GET /api/public/comments API Endpoint", () => {
           createdAt: new Date("2021-05-01T00:00:00Z"),
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           content: "comment-5",
-          objectId: trace.id,
+          objectId: "1234",
           objectType: "TRACE",
         },
       ],
     });
   });
-  afterEach(async () => await pruneDatabase());
 
   it("should return all comments", async () => {
     const comments = await makeZodVerifiedAPICall(
@@ -170,7 +212,7 @@ describe("GET /api/public/comments API Endpoint", () => {
   });
 
   it("should return comments for a specific objectId and objectType", async () => {
-    const objectId = "trace-2021-01-01";
+    const objectId = "1234";
     const objectType = "TRACE";
 
     const comments = await makeZodVerifiedAPICall(
@@ -190,7 +232,7 @@ describe("GET /api/public/comments API Endpoint", () => {
 
   it("should return comments linked to a specific object and by a specific author", async () => {
     const authorUserId = "user-1";
-    const objectId = "trace-2021-01-01";
+    const objectId = "1234";
     const objectType = "TRACE";
 
     const comments = await makeZodVerifiedAPICall(

web/src/__tests__/async/daily-metrics-api.servertest.ts

@@ -0,0 +1,72 @@
+import { createObservation, createTrace } from "@langfuse/shared/src/server";
+import {
+  createObservationsCh,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
+import { randomUUID } from "crypto";
+import { GetMetricsDailyV1Response } from "@/src/features/public-api/types/metrics";
+
+describe("/api/public/metrics/daily API Endpoint", () => {
+  it("should get correct results from daily metrics", async () => {
+    const traceName = randomUUID();
+    const createdTrace = createTrace({
+      name: traceName,
+      user_id: "user-1",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    });
+
+    const observations = [
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        input: "input",
+        output: "output",
+        provided_model_name: "model-1",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-2",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 100000,
+        input: "input-2",
+        output: "output-2",
+        provided_model_name: "model-2",
+      }),
+    ];
+
+    await createTracesCh([createdTrace]);
+    await createObservationsCh(observations);
+
+    const metrics = await makeZodVerifiedAPICall(
+      GetMetricsDailyV1Response,
+      "GET",
+      `/api/public/metrics/daily?traceName=${traceName}`,
+    );
+
+    expect(metrics.body.meta.totalItems).toBe(1);
+    expect(metrics.body.meta.totalPages).toBe(1);
+
+    const metric = metrics.body.data[0];
+    expect(metric.countTraces).toBe(1);
+    expect(metric.countObservations).toBe(2);
+    expect(metric.usage).toHaveLength(2);
+    expect(metric.totalCost).toBe(600);
+    for (const usage of metric.usage) {
+      expect(usage.model).toMatch(/model-\d/g);
+      expect(usage.inputUsage).toBe(1234);
+      expect(usage.outputUsage).toBe(5678);
+      expect(usage.totalUsage).toBe(6912);
+      expect(usage.countObservations).toBe(1);
+      expect(usage.countTraces).toBe(1);
+      expect(usage.totalCost).toBe(300);
+    }
+  });
+});

web/src/__tests__/async/dataset-service.servertest.ts

@@ -1,15 +1,15 @@
 import {
-  createObservations,
-  createScores,
-  createTraces,
-} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+  createObservationsCh,
+  createScoresCh,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
 import { v4 } from "uuid";
 import { prisma } from "@langfuse/shared/src/db";
 import {
   createObservation,
   createScore,
   createTrace,
-} from "@/src/__tests__/fixtures/tracing-factory";
+} from "@langfuse/shared/src/server";
 import {
   createDatasetRunsTable,
   fetchDatasetItems,
@@ -152,7 +152,7 @@ describe("Fetch datasets for UI presentation", () => {
       start_time: new Date().getTime() - 1000,
       end_time: new Date().getTime(),
     });
-    await createObservations([
+    await createObservationsCh([
       observation,
       observation2,
       observation3,
@@ -187,7 +187,7 @@ describe("Fetch datasets for UI presentation", () => {
       value: 1,
       comment: "some other comment for non run related score",
     });
-    await createScores([score, score2, score3]);
+    await createScoresCh([score, score2, score3]);
 
     const runs = await createDatasetRunsTable({
       projectId,
@@ -197,8 +197,6 @@ describe("Fetch datasets for UI presentation", () => {
       limit: 10,
     });
 
-    console.log("runs", JSON.stringify(runs));
-
     expect(runs).toHaveLength(2);
 
     const firstRun = runs.find((run) => run.run_id === datasetRunId);
@@ -246,6 +244,137 @@ describe("Fetch datasets for UI presentation", () => {
     expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
   });
 
+  it.only("should test that dataset runs can link to the same traces", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    const datasetRun2Id = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRun2Id,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const datasetRunItemId2 = v4();
+    const traceId = v4();
+    const scoreId = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    // linked to the second run
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRun2Id,
+        observationId: null,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const scoreName = v4();
+    const score = createScore({
+      id: scoreId,
+      observation_id: null,
+      trace_id: traceId,
+      project_id: projectId,
+      name: scoreName,
+    });
+
+    await createScoresCh([score]);
+
+    const runs = await createDatasetRunsTable({
+      projectId,
+      datasetId,
+      queryClickhouse: false,
+      page: 0,
+      limit: 10,
+    });
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.run_id === datasetRunId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+    expect(firstRun.run_id).toEqual(datasetRunId);
+
+    expect(firstRun.run_description).toBeNull();
+    expect(firstRun.run_metadata).toEqual({});
+
+    expect(firstRun.avgLatency).toBeGreaterThanOrEqual(0);
+    expect(firstRun.avgTotalCost.toString()).toStrictEqual("0");
+
+    const expectedObject = {
+      [`${scoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([100.5]),
+        average: 100.5,
+        comment: "comment",
+      },
+    };
+
+    expect(firstRun.scores).toEqual(expectedObject);
+
+    const secondRun = runs.find((run) => run.run_id === datasetRun2Id);
+
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("second run is not defined");
+    }
+
+    expect(secondRun.run_id).toEqual(datasetRun2Id);
+    expect(secondRun.run_description).toBeNull();
+    expect(secondRun.run_metadata).toEqual({});
+    expect(secondRun.avgLatency).toEqual(0);
+    expect(secondRun.avgTotalCost.toString()).toStrictEqual("0");
+
+    expect(firstRun.scores).toEqual(expectedObject);
+  });
+
   it("should fetch dataset run items for UI", async () => {
     const datasetId = v4();
 
@@ -315,7 +444,7 @@ describe("Fetch datasets for UI presentation", () => {
       project_id: projectId,
     });
 
-    await createTraces([trace1, trace2]);
+    await createTracesCh([trace1, trace2]);
 
     const observation = createObservation({
       id: observationId,
@@ -329,7 +458,7 @@ describe("Fetch datasets for UI presentation", () => {
       trace_id: traceId,
     });
 
-    await createObservations([observation]);
+    await createObservationsCh([observation]);
 
     const score = createScore({
       observation_id: observation2.id,
@@ -337,7 +466,7 @@ describe("Fetch datasets for UI presentation", () => {
       project_id: projectId,
     });
 
-    await createScores([score]);
+    await createScoresCh([score]);
 
     const runs = await getRunItemsByRunIdOrItemId(
       projectId,
@@ -543,7 +672,7 @@ describe("Fetch datasets for UI presentation", () => {
       end_time: new Date().getTime(),
     });
 
-    await createObservations([observation]);
+    await createObservationsCh([observation]);
 
     const trace1 = createTrace({
       id: traceId1,
@@ -554,7 +683,7 @@ describe("Fetch datasets for UI presentation", () => {
       project_id: projectId,
     });
 
-    await createTraces([trace1, trace2]);
+    await createTracesCh([trace1, trace2]);
 
     const input = {
       projectId: projectId,

web/src/__tests__/async/observation-api.servertest.ts

@@ -1,18 +1,22 @@
-import { createObservation as createObservationObject } from "@/src/__tests__/fixtures/tracing-factory";
-import { createObservations as createObservationsInClickhouse } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import {
+  createObservation as createObservationObject,
+  createTrace,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { createObservationsCh as createObservationsInClickhouse } from "@langfuse/shared/src/server";
 import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
 import { GetObservationV1Response } from "@/src/features/public-api/types/observations";
-import { v4 } from "uuid";
 import { v4 as uuidv4 } from "uuid";
 import { GetObservationsV1Response } from "@/src/features/public-api/types/observations";
+import { snakeCase } from "lodash";
 
 const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
 
 describe("/api/public/observations API Endpoint", () => {
   describe("GET /api/public/observations/:id", () => {
     it("should GET an observation", async () => {
-      const observationId = v4();
-      const traceId = v4();
+      const observationId = uuidv4();
+      const traceId = uuidv4();
 
       const observation = createObservationObject({
         id: observationId,
@@ -137,8 +141,6 @@ describe("/api/public/observations API Endpoint", () => {
       undefined,
     );
 
-    console.log(fetchedObservations.body);
-
     expect(fetchedObservations.status).toBe(200);
 
     expect(fetchedObservations.body.data.length).toBe(1);
@@ -148,8 +150,55 @@ describe("/api/public/observations API Endpoint", () => {
     expect(fetchedObservations.body.data[0]?.type).toEqual("GENERATION");
   });
 
+  it.each([
+    ["userId", uuidv4()],
+    ["traceId", uuidv4()],
+    ["name", uuidv4()],
+    ["version", uuidv4()],
+  ])(
+    "should fetch all observations filtered by a value (%s, %s)",
+    async (prop: string, value: string) => {
+      const traceId = uuidv4();
+
+      if (prop === "userId") {
+        const createdTrace = createTrace({
+          id: traceId,
+          [snakeCase(prop)]: value,
+          project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        });
+
+        await createTracesCh([createdTrace]);
+      }
+
+      const observation = createObservationObject({
+        id: uuidv4(),
+        trace_id: traceId,
+        start_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+        end_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        type: "GENERATION",
+        [snakeCase(prop)]: value,
+      });
+
+      await createObservationsInClickhouse([observation]);
+
+      const observations = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?${prop}=${value}`,
+      );
+
+      expect(observations.body.meta.totalItems).toBe(1);
+      expect(observations.body.data.length).toBe(1);
+      const obsResult = observations.body.data[0];
+      expect(obsResult.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+      if (prop === "userId") return;
+      expect((obsResult as any)[prop]).toBe(value);
+    },
+  );
+
   it("GET /observations with timestamp filters and pagination", async () => {
-    const traceId = v4();
+    const traceId = uuidv4();
     const obs1 = createObservationObject({
       id: "observation-2021-01-01",
       trace_id: traceId,

web/src/__tests__/async/repositories/observation-repository.servertest.ts

@@ -0,0 +1,168 @@
+import { createObservationsCh } from "@langfuse/shared/src/server";
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import {
+  getObservationById,
+  getObservationsViewForTrace,
+} from "@langfuse/shared/src/server";
+import Decimal from "decimal.js";
+import { v4 } from "uuid";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("Clickhouse Observations Repository Test", () => {
+  beforeEach(async () => {
+    await pruneDatabase();
+  });
+
+  it("should throw if no observations are found", async () => {
+    await expect(getObservationById(v4(), v4())).rejects.toThrow();
+  });
+
+  it("should return an observation if exists", async () => {
+    const observationId = v4();
+    const traceId = v4();
+
+    const observation = {
+      id: observationId,
+      trace_id: traceId,
+      project_id: projectId,
+      type: "sample_type",
+      metadata: {},
+      provided_usage_details: { input: 1234, output: 5678, total: 6912 },
+      provided_cost_details: { input: 100, output: 200, total: 300 },
+      usage_details: { input: 1234, output: 5678, total: 6912 },
+      cost_details: { input: 100, output: 200, total: 300 },
+      is_deleted: 0,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+      start_time: Date.now(),
+      event_ts: Date.now(),
+      name: "sample_name",
+      level: "sample_level",
+      status_message: "sample_status",
+      version: "1.0",
+      input: "sample_input",
+      output: "sample_output",
+      provided_model_name: "sample_model",
+      internal_model_id: "sample_internal_model_id",
+      model_parameters: '{"something":"sample_param"}',
+      total_cost: 300,
+      prompt_id: "sample_prompt_id",
+      prompt_name: "sample_prompt_name",
+      prompt_version: 1,
+      end_time: Date.now(),
+      completion_start_time: Date.now(),
+    };
+
+    await createObservationsCh([observation]);
+
+    const result = await getObservationById(observationId, projectId, true);
+    if (!result) {
+      throw new Error("Observation not found");
+    }
+    expect(result.id).toEqual(observation.id);
+    expect(result.traceId).toEqual(observation.trace_id);
+    expect(result.projectId).toEqual(observation.project_id);
+    expect(result.type).toEqual(observation.type);
+    expect(result.metadata).toEqual(observation.metadata);
+    expect(result.createdAt).toEqual(new Date(observation.created_at));
+    expect(result.updatedAt).toEqual(new Date(observation.updated_at));
+    expect(result.startTime).toEqual(new Date(observation.start_time));
+    expect(result.name).toEqual(observation.name);
+    expect(result.level).toEqual(observation.level);
+    expect(result.statusMessage).toEqual(observation.status_message);
+    expect(result.version).toEqual(observation.version);
+    expect(result.input).toEqual(observation.input);
+    expect(result.output).toEqual(observation.output);
+    expect(result.internalModelId).toEqual(observation.internal_model_id);
+    expect(result.modelParameters).toEqual({ something: "sample_param" });
+    expect(result.promptId).toEqual(observation.prompt_id);
+    expect(result.endTime).toEqual(new Date(observation.end_time));
+    expect(result.completionStartTime).toEqual(
+      new Date(observation.completion_start_time),
+    );
+    expect(result.totalCost).toEqual(new Decimal(observation.total_cost));
+    expect(result.promptTokens).toEqual(1234);
+    expect(result.completionTokens).toEqual(5678);
+    expect(result.totalTokens).toEqual(6912);
+  });
+  it("should return an observation view", async () => {
+    const observationId = v4();
+    const traceId = v4();
+
+    const observation = {
+      id: observationId,
+      trace_id: traceId,
+      project_id: projectId,
+      type: "sample_type",
+      metadata: {},
+      provided_usage_details: { input: 1234, output: 5678, total: 6912 },
+      provided_cost_details: { input: 100, output: 200, total: 300 },
+      usage_details: { input: 1234, output: 5678, total: 6912 },
+      cost_details: { input: 100, output: 200, total: 300 },
+      is_deleted: 0,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+      start_time: Date.now(),
+      event_ts: Date.now(),
+      name: "sample_name",
+      level: "sample_level",
+      status_message: "sample_status",
+      version: "1.0",
+      input: "sample_input",
+      output: "sample_output",
+      provided_model_name: "sample_model",
+      internal_model_id: "sample_internal_model_id",
+      model_parameters: '{"something":"sample_param"}',
+      total_cost: 300,
+      prompt_id: "sample_prompt_id",
+      prompt_name: "sample_prompt_name",
+      prompt_version: 1,
+      end_time: Date.now(),
+      completion_start_time: Date.now() + 1000,
+    };
+
+    await createObservationsCh([observation]);
+
+    const result = await getObservationsViewForTrace(traceId, projectId);
+    if (!result || result.length === 0) {
+      throw new Error("Observation not found");
+    }
+    const firstObservation = result[0];
+    expect(firstObservation.id).toEqual(observation.id);
+    expect(firstObservation.traceId).toEqual(observation.trace_id);
+    expect(firstObservation.projectId).toEqual(observation.project_id);
+    expect(firstObservation.type).toEqual(observation.type);
+    expect(firstObservation.metadata).toEqual(observation.metadata);
+    expect(firstObservation.createdAt).toEqual(
+      new Date(observation.created_at),
+    );
+    expect(firstObservation.updatedAt).toEqual(
+      new Date(observation.updated_at),
+    );
+    expect(firstObservation.startTime).toEqual(
+      new Date(observation.start_time),
+    );
+    expect(firstObservation.name).toEqual(observation.name);
+    expect(firstObservation.level).toEqual(observation.level);
+    expect(firstObservation.statusMessage).toEqual(observation.status_message);
+    expect(firstObservation.version).toEqual(observation.version);
+
+    expect(firstObservation.modelParameters).toEqual({
+      something: "sample_param",
+    });
+    expect(firstObservation.promptId).toEqual(observation.prompt_id);
+    expect(firstObservation.endTime).toEqual(new Date(observation.end_time));
+    expect(firstObservation.timeToFirstToken).toEqual(
+      new Date(observation.completion_start_time).getTime() -
+        new Date(observation.start_time).getTime(),
+    );
+    expect(firstObservation.timeToFirstToken).toBeGreaterThan(0);
+    expect(firstObservation.calculatedTotalCost).toEqual(
+      new Decimal(observation.total_cost),
+    );
+    expect(firstObservation.promptTokens).toEqual(1234);
+    expect(firstObservation.completionTokens).toEqual(5678);
+    expect(firstObservation.totalTokens).toEqual(6912);
+  });
+});

web/src/__tests__/async/repositories/score-repository.servertest.ts

@@ -1,4 +1,4 @@
-import { createScores } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { createScoresCh } from "@langfuse/shared/src/server";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
 import { getScoreById } from "@langfuse/shared/src/server";
 import { v4 } from "uuid";
@@ -33,7 +33,7 @@ describe("Clickhouse Scores Repository Test", () => {
       is_deleted: 0,
     };
 
-    await createScores([score]);
+    await createScoresCh([score]);
 
     const result = await getScoreById(projectId, scoreId);
     expect(result).not.toBeNull();

web/src/__tests__/async/repositories/trace-repository.servertest.ts

@@ -1,4 +1,4 @@
-import { createTraces } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { createTracesCh } from "@langfuse/shared/src/server";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
 import {
   getTraceById,
@@ -39,7 +39,7 @@ describe("Clickhouse Traces Repository Test", () => {
       is_deleted: 0,
     };
 
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const result = await getTraceById(
       traceId,
@@ -90,7 +90,7 @@ describe("Clickhouse Traces Repository Test", () => {
       is_deleted: 0,
     };
 
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const result = await getTraceById(traceId, projectId);
     expect(result).not.toBeNull();
@@ -154,7 +154,7 @@ describe("Clickhouse Traces Repository Test", () => {
       is_deleted: 0,
     };
 
-    await createTraces([trace1, trace2]);
+    await createTracesCh([trace1, trace2]);
 
     const results = await getTracesBySessionId(projectId, [sessionId]);
     expect(results).toHaveLength(2);

web/src/__tests__/async/scores-api.servertest.ts

@@ -2,16 +2,14 @@ import {
   createObservation,
   createScore,
   createTrace,
-} from "@/src/__tests__/fixtures/tracing-factory";
-import {
-  createObservations,
-  createScores,
-  createTraces,
-} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+} from "@langfuse/shared/src/server";
 import {
+  createObservationsCh,
+  createScoresCh,
+  createTracesCh,
   createOrgProjectAndApiKey,
-  makeZodVerifiedAPICall,
-} from "@/src/__tests__/test-utils";
+} from "@langfuse/shared/src/server";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
 import { GetScoreResponse, GetScoresResponse } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
@@ -41,7 +39,7 @@ describe("/api/public/scores API Endpoint", () => {
         is_deleted: 0,
       };
 
-      await createScores([score]);
+      await createScoresCh([score]);
 
       const getScore = await makeZodVerifiedAPICall(
         GetScoreResponse,
@@ -76,7 +74,7 @@ describe("/api/public/scores API Endpoint", () => {
       id: traceId,
       project_id: projectId,
     });
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const scoreId = v4();
 
@@ -90,7 +88,7 @@ describe("/api/public/scores API Endpoint", () => {
       comment: "comment",
       observation_id: null,
     });
-    await createScores([score]);
+    await createScoresCh([score]);
 
     const fetchedScore = await makeZodVerifiedAPICall(
       GetScoreResponse,
@@ -118,7 +116,7 @@ describe("/api/public/scores API Endpoint", () => {
       id: minimalTraceId,
       project_id: projectId,
     });
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const minimalScoreId = v4();
 
@@ -132,7 +130,7 @@ describe("/api/public/scores API Endpoint", () => {
       comment: null,
       observation_id: null,
     });
-    await createScores([score]);
+    await createScoresCh([score]);
 
     const fetchedScore = await makeZodVerifiedAPICall(
       GetScoreResponse,
@@ -189,7 +187,7 @@ describe("/api/public/scores API Endpoint", () => {
         tags: ["staging"],
       });
 
-      await createTraces([trace, trace_2, trace_3]);
+      await createTracesCh([trace, trace_2, trace_3]);
 
       const generation = createObservation({
         id: generationId,
@@ -197,7 +195,7 @@ describe("/api/public/scores API Endpoint", () => {
         type: "GENERATION",
       });
 
-      await createObservations([generation]);
+      await createObservationsCh([generation]);
 
       const config = await prisma.scoreConfig.create({
         data: {
@@ -266,7 +264,7 @@ describe("/api/public/scores API Endpoint", () => {
         comment: "comment",
       });
 
-      await createScores([score1, score2, score3, score4, score5]);
+      await createScoresCh([score1, score2, score3, score4, score5]);
     });
 
     it("get all scores", async () => {
@@ -422,7 +420,7 @@ describe("/api/public/scores API Endpoint", () => {
             queue_id: queueId,
           });
 
-          await createScores([score, score2]);
+          await createScoresCh([score, score2]);
         });
 
         it("get all scores for queueId", async () => {

web/src/__tests__/async/sessions-api.servertest.ts

@@ -1,9 +1,15 @@
 import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
 import { v4 } from "uuid";
 import { prisma } from "@langfuse/shared/src/db";
-import { createTraces } from "@/src/__tests__/server/repositories/clickhouse-helpers";
-import { createTrace } from "@/src/__tests__/fixtures/tracing-factory";
-import { GetSessionV1Response } from "@/src/features/public-api/types/sessions";
+import {
+  createTracesCh,
+  createOrgProjectAndApiKey,
+} from "@langfuse/shared/src/server";
+import { createTrace } from "@langfuse/shared/src/server";
+import {
+  GetSessionsV1Response,
+  GetSessionV1Response,
+} from "@/src/features/public-api/types/sessions";
 
 const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
 
@@ -24,7 +30,7 @@ describe("/api/public/sessions API Endpoint", () => {
         createTrace({ session_id: sessionId, project_id: projectId }),
       ];
 
-      await createTraces(traces);
+      await createTracesCh(traces);
 
       const getScore = await makeZodVerifiedAPICall(
         GetSessionV1Response,
@@ -44,4 +50,147 @@ describe("/api/public/sessions API Endpoint", () => {
       );
     });
   });
+  describe("GET /api/public/sessions API Endpoint", () => {
+    let projectId: string;
+    let auth: string;
+
+    beforeAll(async () => {
+      const { projectId: newProjectId, auth: newAuth } =
+        await createOrgProjectAndApiKey();
+      projectId = newProjectId;
+      auth = newAuth;
+
+      await prisma.traceSession.createMany({
+        data: [
+          {
+            id: "session-2021-01-01",
+            createdAt: new Date("2021-01-01T00:00:00Z"),
+            projectId: projectId,
+          },
+          {
+            id: "session-2021-02-01",
+            createdAt: new Date("2021-02-01T00:00:00Z"),
+            projectId: projectId,
+          },
+          {
+            id: "session-2021-03-01",
+            createdAt: new Date("2021-03-01T00:00:00Z"),
+            projectId: projectId,
+          },
+          {
+            id: "session-2021-04-01",
+            createdAt: new Date("2021-04-01T00:00:00Z"),
+            projectId: projectId,
+          },
+          {
+            id: "session-2021-05-01",
+            createdAt: new Date("2021-05-01T00:00:00Z"),
+            projectId: projectId,
+          },
+        ],
+      });
+    });
+
+    it("should return all sessions", async () => {
+      const sessions = await makeZodVerifiedAPICall(
+        GetSessionsV1Response,
+        "GET",
+        "/api/public/sessions",
+        undefined,
+        auth,
+      );
+      expect(sessions.body.data.length).toEqual(5);
+      expect(sessions.body.data.map((session) => session.id)).toEqual(
+        expect.arrayContaining([
+          "session-2021-01-01",
+          "session-2021-02-01",
+          "session-2021-03-01",
+          "session-2021-04-01",
+          "session-2021-05-01",
+        ]),
+      );
+    });
+
+    it("should return paginated sessions with limit and page number", async () => {
+      const limit = 2;
+      const page = 2;
+
+      const sessions = await makeZodVerifiedAPICall(
+        GetSessionsV1Response,
+        "GET",
+        `/api/public/sessions?limit=${limit}&page=${page}`,
+        undefined,
+        auth,
+      );
+
+      expect(sessions.body.data).toHaveLength(2);
+      expect(sessions.body.data.map((session) => session.id)).toEqual([
+        "session-2021-03-01",
+        "session-2021-02-01",
+      ]);
+      expect(sessions.body.meta.totalItems).toBe(5);
+      expect(sessions.body.meta.totalPages).toBe(3);
+      expect(sessions.body.meta.page).toBe(page);
+      expect(sessions.body.meta.limit).toBe(limit);
+    });
+
+    it("should return sessions within a specific date range", async () => {
+      const fromTimestamp = "2021-02-01T00:00:00Z";
+      const toTimestamp = "2021-04-01T00:00:00Z";
+
+      const sessions = await makeZodVerifiedAPICall(
+        GetSessionsV1Response,
+        "GET",
+        `/api/public/sessions?fromTimestamp=${fromTimestamp}&toTimestamp=${toTimestamp}`,
+        undefined,
+        auth,
+      );
+
+      expect(sessions.body.data).toHaveLength(2);
+      expect(sessions.body.data.map((session) => session.id)).toEqual([
+        "session-2021-03-01",
+        "session-2021-02-01",
+      ]);
+      expect(sessions.body.meta.totalItems).toBe(2);
+    });
+
+    it("should return sessions from a specific date onwards (including the date)", async () => {
+      const fromTimestamp = "2021-03-01T00:00:00Z";
+
+      const sessions = await makeZodVerifiedAPICall(
+        GetSessionsV1Response,
+        "GET",
+        `/api/public/sessions?fromTimestamp=${fromTimestamp}`,
+        undefined,
+        auth,
+      );
+
+      expect(sessions.body.data).toHaveLength(3);
+      expect(sessions.body.data.map((session) => session.id)).toEqual([
+        "session-2021-05-01",
+        "session-2021-04-01",
+        "session-2021-03-01",
+      ]);
+      expect(sessions.body.meta.totalItems).toBe(3);
+    });
+
+    it("should return sessions up to a specific date (excluding it)", async () => {
+      const toTimestamp = "2021-03-01T00:00:00Z";
+
+      const sessions = await makeZodVerifiedAPICall(
+        GetSessionsV1Response,
+        "GET",
+        `/api/public/sessions?toTimestamp=${toTimestamp}`,
+        undefined,
+        auth,
+      );
+
+      expect(sessions.body.data).toHaveLength(2);
+      expect(sessions.body.data.map((session) => session.id)).toEqual([
+        "session-2021-02-01",
+        "session-2021-01-01",
+      ]);
+      expect(sessions.body.meta.totalItems).toBe(2);
+    });
+  });
 });

web/src/__tests__/async/traces-api.servertest.ts

@@ -1,13 +1,15 @@
+import { createObservation, createTrace } from "@langfuse/shared/src/server";
 import {
-  createObservation,
-  createTrace,
-} from "@/src/__tests__/fixtures/tracing-factory";
-import {
-  createObservations,
-  createTraces,
-} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+  createObservationsCh,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
 import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
-import { GetTraceV1Response } from "@/src/features/public-api/types/traces";
+import {
+  GetTracesV1Response,
+  GetTraceV1Response,
+} from "@/src/features/public-api/types/traces";
+import { randomUUID } from "crypto";
+import { snakeCase } from "lodash";
 
 describe("/api/public/traces API Endpoint", () => {
   it("should create and get a trace via /traces", async () => {
@@ -41,8 +43,8 @@ describe("/api/public/traces API Endpoint", () => {
       }),
     ];
 
-    await createTraces([createdTrace]);
-    await createObservations(observations);
+    await createTracesCh([createdTrace]);
+    await createObservationsCh(observations);
 
     const trace = await makeZodVerifiedAPICall(
       GetTraceV1Response,
@@ -73,4 +75,187 @@ describe("/api/public/traces API Endpoint", () => {
       ]),
     );
   });
+
+  it("should fetch all traces", async () => {
+    const timestamp = new Date();
+    const createdTrace = createTrace({
+      name: "trace-name",
+      user_id: "user-1",
+      timestamp: timestamp.getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value", jsonKey: JSON.stringify({ foo: "bar" }) },
+      release: "1.0.0",
+      version: "2.0.0",
+    });
+
+    const observations = [
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name",
+        end_time: timestamp.getTime(),
+        start_time: timestamp.getTime() - 1000,
+        input: "input",
+        output: "output",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-2",
+        end_time: timestamp.getTime(),
+        start_time: timestamp.getTime() - 100000,
+        input: "input-2",
+        output: "output-2",
+      }),
+    ];
+
+    await createTracesCh([createdTrace]);
+    await createObservationsCh(observations);
+
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      "/api/public/traces",
+    );
+
+    expect(traces.body.meta.totalItems).toBeGreaterThanOrEqual(1);
+    expect(traces.body.data.length).toBeGreaterThanOrEqual(1);
+    const trace = traces.body.data.find((t) => t.id === createdTrace.id);
+    expect(trace).toBeTruthy();
+    if (!trace) {
+      return; // to satisfy TypeScript
+    }
+    expect(trace.name).toBe("trace-name");
+    expect(trace.release).toBe("1.0.0");
+    expect(trace.metadata.key).toBe("value");
+    expect(trace.metadata.jsonKey).toEqual({ foo: "bar" });
+    expect(trace.externalId).toBeNull();
+    expect(trace.version).toBe("2.0.0");
+    expect(trace.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(trace.latency).toBe(100);
+    expect(trace.observations.length).toBe(2);
+    expect(trace.scores.length).toBe(0);
+    expect(trace.timestamp).toBe(timestamp.toISOString());
+  });
+
+  it.each([
+    ["userId", randomUUID()],
+    ["sessionId", randomUUID()],
+    ["release", randomUUID()],
+    ["version", randomUUID()],
+    ["name", randomUUID()],
+  ])(
+    "should fetch all traces filtered by a value (%s, %s)",
+    async (prop: string, value: string) => {
+      const createdTrace = createTrace({
+        [snakeCase(prop)]: value,
+        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        metadata: { key: "value" },
+      });
+
+      await createTracesCh([createdTrace]);
+
+      const traces = await makeZodVerifiedAPICall(
+        GetTracesV1Response,
+        "GET",
+        `/api/public/traces?${prop}=${value}`,
+      );
+
+      expect(traces.body.meta.totalItems).toBe(1);
+      expect(traces.body.data.length).toBe(1);
+      const trace = traces.body.data[0];
+      expect(trace.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+      expect((trace as any)[prop]).toBe(value);
+    },
+  );
+
+  it("should fetch all traces filtered by a tag", async () => {
+    const tag = randomUUID();
+    const createdTrace = createTrace({
+      name: "trace-name",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+
+    await createTracesCh([createdTrace]);
+
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      `/api/public/traces?tags=${[tag]}`,
+    );
+
+    expect(traces.body.meta.totalItems).toBe(1);
+    expect(traces.body.data.length).toBe(1);
+    const trace = traces.body.data[0];
+    expect(trace.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+  });
+
+  it("should fetch all traces with pagination", async () => {
+    const tag = randomUUID();
+    const createdTrace1 = createTrace({
+      name: "trace-name",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+    const createdTrace2 = createTrace({
+      name: "trace-name",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+    const createdTrace3 = createTrace({
+      name: "trace-name",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+
+    await createTracesCh([createdTrace1, createdTrace2, createdTrace3]);
+
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      `/api/public/traces?tags=${[tag]}&limit=1&offset=1`,
+    );
+
+    expect(traces.body.meta.totalItems).toBe(3);
+    expect(traces.body.data.length).toBe(1);
+    expect(traces.body.meta.totalPages).toBe(3);
+    const trace = traces.body.data[0];
+    expect(trace.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+  });
+
+  it("should fetch all traces with custom order", async () => {
+    const tag = randomUUID();
+    const createdTrace1 = createTrace({
+      name: "trace-name1",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+    const createdTrace2 = createTrace({
+      name: "trace-name2",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      tags: [tag],
+    });
+
+    await createTracesCh([createdTrace1, createdTrace2]);
+
+    const traces = await makeZodVerifiedAPICall(
+      GetTracesV1Response,
+      "GET",
+      `/api/public/traces?tags=${[tag]}&orderBy=name.desc`,
+    );
+
+    expect(traces.body.meta.totalItems).toBe(2);
+    expect(traces.body.data.length).toBe(2);
+    const trace1 = traces.body.data[0];
+    expect(trace1.name).toBe("trace-name2");
+    const trace2 = traces.body.data[1];
+    expect(trace2.name).toBe("trace-name1");
+  });
 });

web/src/__tests__/async/traces-ui-table.servertest.ts

@@ -1,17 +1,14 @@
 import { v4 } from "uuid";
 import {
-  createObservations,
-  createTraces,
-} from "@/src/__tests__/server/repositories/clickhouse-helpers";
-import {
-  createObservation,
-  createTrace,
-} from "@/src/__tests__/fixtures/tracing-factory";
+  createObservationsCh,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { createObservation, createTrace } from "@langfuse/shared/src/server";
 import {
   getTracesTable,
+  type TracesAllUiReturnType,
   type ObservationRecordInsertType,
   type TraceRecordInsertType,
-  type TracesTableReturnType,
 } from "@langfuse/shared/src/server";
 import { type FilterState } from "@langfuse/shared";
 
@@ -21,7 +18,7 @@ describe("Traces table API test", () => {
     const trace_id = v4();
 
     const trace = createTrace({ id: trace_id, project_id });
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const tableRows = await getTracesTable(
       project_id,
@@ -43,12 +40,6 @@ describe("Traces table API test", () => {
     expect(tableRows[0].userId).toEqual(trace.user_id);
     expect(tableRows[0].sessionId).toEqual(trace.session_id);
     expect(tableRows[0].public).toEqual(trace.public);
-    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
-    expect(tableRows[0].usageDetails).toEqual({});
-    expect(tableRows[0].costDetails).toEqual({});
-    expect(tableRows[0].level).toBeDefined();
-    expect(tableRows[0].observationCount).toBeGreaterThanOrEqual(0);
-    expect(tableRows[0].scoresAvg).toEqual([]);
   });
 
   it("should get a correct trace with observations", async () => {
@@ -56,11 +47,11 @@ describe("Traces table API test", () => {
     const trace_id = v4();
 
     const trace = createTrace({ id: trace_id, project_id });
-    await createTraces([trace]);
+    await createTracesCh([trace]);
 
     const obs1 = createObservation({ trace_id, project_id });
     const obs2 = createObservation({ trace_id, project_id });
-    await createObservations([obs1, obs2]);
+    await createObservationsCh([obs1, obs2]);
 
     const tableRows = await getTracesTable(
       project_id,
@@ -82,29 +73,13 @@ describe("Traces table API test", () => {
     expect(tableRows[0].userId).toEqual(trace.user_id);
     expect(tableRows[0].sessionId).toEqual(trace.session_id);
     expect(tableRows[0].public).toEqual(trace.public);
-    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
-    expect(tableRows[0].usageDetails).toEqual({
-      input: (obs1.usage_details.input + obs2.usage_details.input).toString(),
-      output: (
-        obs1.usage_details.output + obs2.usage_details.output
-      ).toString(),
-      total: (obs1.usage_details.total + obs2.usage_details.total).toString(),
-    });
-    expect(tableRows[0].costDetails).toEqual({
-      input: obs1.cost_details.input + obs2.cost_details.input,
-      output: obs1.cost_details.output + obs2.cost_details.output,
-      total: obs1.cost_details.total + obs2.cost_details.total,
-    });
-    expect(tableRows[0].level).toBeDefined();
-    expect(tableRows[0].observationCount).toBeGreaterThanOrEqual(0);
-    expect(tableRows[0].scoresAvg).toEqual([]);
   });
 
   type TestCase = {
     traceInput: Partial<TraceRecordInsertType>;
     observationInput: Partial<ObservationRecordInsertType>[];
     filterstate: FilterState;
-    expected: Partial<TracesTableReturnType>[];
+    expected: Partial<TracesAllUiReturnType>[];
   };
 
   [
@@ -124,32 +99,6 @@ describe("Traces table API test", () => {
       ],
       expected: [],
     },
-    {
-      traceInput: {},
-      observationInput: [
-        {
-          cost_details: { total: 0.000001 },
-          usage_details: { total: 0 },
-        },
-        {
-          cost_details: { total: 0.000002 },
-          usage_details: { total: 0 },
-        },
-      ],
-      filterstate: [
-        {
-          column: "Total Cost ($)",
-          operator: ">" as const,
-          value: 0.000002,
-          type: "number" as const,
-        },
-      ],
-      expected: [
-        {
-          cost_details: { total: 0.000003 },
-        },
-      ],
-    },
     {
       traceInput: {},
       observationInput: [],
@@ -186,7 +135,7 @@ describe("Traces table API test", () => {
         project_id,
         ...testConfig.traceInput,
       });
-      await createTraces([trace]);
+      await createTracesCh([trace]);
 
       expect(testConfig.observationInput.length).not.toBeGreaterThan(2);
 
@@ -204,7 +153,7 @@ describe("Traces table API test", () => {
           ? testConfig.observationInput[1]
           : {}),
       });
-      await createObservations([obs1, obs2]);
+      await createObservationsCh([obs1, obs2]);
 
       const tableRows = await getTracesTable(
         project_id,
@@ -220,8 +169,8 @@ describe("Traces table API test", () => {
         if (expectedTrace.id !== undefined) {
           expect(tableRows[index].id).toEqual(expectedTrace.id);
         }
-        if (expectedTrace.project_id !== undefined) {
-          expect(tableRows[index].projectId).toEqual(expectedTrace.project_id);
+        if (expectedTrace.projectId !== undefined) {
+          expect(tableRows[index].projectId).toEqual(expectedTrace.projectId);
         }
         if (expectedTrace.tags !== undefined) {
           expect(tableRows[index].tags).toEqual(expectedTrace.tags);
@@ -238,39 +187,15 @@ describe("Traces table API test", () => {
         if (expectedTrace.version !== undefined) {
           expect(tableRows[index].version).toEqual(expectedTrace.version);
         }
-        if (expectedTrace.user_id !== undefined) {
-          expect(tableRows[index].userId).toEqual(expectedTrace.user_id);
+        if (expectedTrace.userId !== undefined) {
+          expect(tableRows[index].userId).toEqual(expectedTrace.userId);
         }
-        if (expectedTrace.session_id !== undefined) {
-          expect(tableRows[index].sessionId).toEqual(expectedTrace.session_id);
+        if (expectedTrace.sessionId !== undefined) {
+          expect(tableRows[index].sessionId).toEqual(expectedTrace.sessionId);
         }
         if (expectedTrace.public !== undefined) {
           expect(tableRows[index].public).toEqual(expectedTrace.public);
         }
-        if (expectedTrace.latency !== undefined) {
-          expect(tableRows[index].latency).toEqual(expectedTrace.latency);
-        }
-        if (expectedTrace.usage_details !== undefined) {
-          expect(tableRows[index].usageDetails).toEqual(
-            expectedTrace.usage_details,
-          );
-        }
-        if (expectedTrace.cost_details !== undefined) {
-          expect(tableRows[index].costDetails).toEqual(
-            expectedTrace.cost_details,
-          );
-        }
-        if (expectedTrace.level !== undefined) {
-          expect(tableRows[index].level).toEqual(expectedTrace.level);
-        }
-        if (expectedTrace.observation_count !== undefined) {
-          expect(tableRows[index].observationCount).toEqual(
-            expectedTrace.observation_count,
-          );
-        }
-        if (expectedTrace.scores_avg !== undefined) {
-          expect(tableRows[index].scoresAvg).toEqual(expectedTrace.scores_avg);
-        }
       });
     });
   });

web/src/__tests__/async/users-ui-table.servertest.ts

@@ -0,0 +1,68 @@
+import {
+  createObservation as createObservationObject,
+  createTrace,
+} from "@langfuse/shared/src/server";
+import {
+  createObservationsCh as createObservationsInClickhouse,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { v4 as uuidv4 } from "uuid";
+import { getUserMetrics } from "@langfuse/shared/src/server";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("getUserMetrics function", () => {
+  it("should return correct user metrics for a trace with two observations", async () => {
+    const userId = uuidv4();
+    const traceId = uuidv4();
+
+    const trace = createTrace({
+      id: traceId,
+      project_id: projectId,
+      user_id: userId,
+    });
+
+    await createTracesCh([trace]);
+
+    const observation1 = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: projectId,
+      usage_details: {
+        input: 100,
+        output: 200,
+        total: 300,
+      },
+      total_cost: 50,
+      type: "GENERATION",
+    });
+
+    const observation2 = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: projectId,
+      usage_details: {
+        input: 150,
+        output: 250,
+        total: 400,
+      },
+      total_cost: 75,
+      type: "GENERATION",
+    });
+
+    await createObservationsInClickhouse([observation1, observation2]);
+
+    const userMetrics = await getUserMetrics(projectId, [userId]);
+
+    expect(userMetrics.length).toBe(1);
+    expect(userMetrics[0]).toMatchObject({
+      userId: userId,
+      inputUsage: 250, // 100 + 150
+      outputUsage: 450, // 200 + 250
+      totalUsage: 700, // 300 + 400
+      observationCount: 2,
+      traceCount: 1,
+      totalCost: 125, // 50 + 75
+    });
+  });
+});

web/src/__tests__/generation-exports.servertest.ts

@@ -1,175 +0,0 @@
-/** @jest-environment node */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-
-import type { Session } from "next-auth";
-import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { BatchExportFileFormat, ModelUsageUnit } from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
-import { appRouter } from "@/src/server/api/root";
-import { createInnerTRPCContext } from "@/src/server/api/trpc";
-
-describe("observations.export RPC", () => {
-  const numberOfGenerations = 5;
-  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-
-  beforeAll(async () => {
-    // Disable S3 upload
-    process.env.S3_BUCKET_NAME = "";
-
-    await pruneDatabase();
-    const traceId = "trace-1";
-
-    await prisma.trace.create({
-      data: {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId,
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    });
-
-    for (let i = 1; i <= numberOfGenerations; i++) {
-      await prisma.observation.create({
-        data: {
-          type: "GENERATION",
-          id: `generation-${i}`,
-          name: `generation-${i}`,
-          model: "gpt-3.5-turbo",
-          totalCost: 1,
-          startTime: new Date("2021-01-01T00:00:00.000Z"),
-          endTime: new Date("2021-01-01T00:00:05.000Z"),
-          project: { connect: { id: projectId } },
-          traceId,
-          input: [
-            {
-              role: "system",
-              content: "Be a helpful assistant",
-            },
-            {
-              role: "user",
-              content: "How can i create a React component?",
-            },
-          ],
-          output: {
-            completion: `Creating a React component can be done in two ways.`,
-          },
-          metadata: {
-            user: `user-@langfuse.com`,
-          },
-          unit: ModelUsageUnit.Tokens,
-        },
-      });
-    }
-  });
-
-  afterAll(async () => await pruneDatabase());
-
-  const session: Session = {
-    expires: "1",
-    user: {
-      id: "clgb17vnp000008jjere5g15i",
-      name: "Demo User",
-      organizations: [
-        {
-          id: "seed-org-id",
-          role: "OWNER",
-          plan: "cloud:hobby",
-          cloudConfig: undefined,
-          projects: [
-            {
-              id: projectId,
-              role: "ADMIN",
-            },
-          ],
-        },
-      ],
-      featureFlags: {
-        templateFlag: true,
-      },
-      admin: true,
-    },
-    environment: {} as any,
-  };
-
-  const ctx = createInnerTRPCContext({ session });
-  const caller = appRouter.createCaller({ ...ctx, prisma });
-
-  it("should return a CSV file", async () => {
-    const result = await caller.generations.export({
-      fileFormat: BatchExportFileFormat.CSV,
-      orderBy: { column: "id", order: "ASC" },
-      filter: [
-        {
-          column: "Start Time",
-          type: "datetime",
-          operator: ">",
-          value: new Date("1990-01-01"),
-        },
-      ],
-      projectId,
-      searchQuery: null,
-    });
-
-    if (result.type !== "data")
-      throw new Error("No data returned. Is S3 accidentally enabled?");
-    const { data, fileName } = result;
-
-    const fileExtension = fileName.split(".").pop();
-    expect(fileName).toContain(`lf-export-${projectId}`);
-    expect(fileExtension).toBe("csv");
-    expect(data.split("\n").filter(Boolean).length).toBe(
-      numberOfGenerations + 1,
-    );
-  });
-
-  it("should return a JSON file", async () => {
-    const result = await caller.generations.export({
-      fileFormat: BatchExportFileFormat.JSON,
-      orderBy: { column: "id", order: "ASC" },
-      filter: [
-        {
-          column: "Start Time",
-          type: "datetime",
-          operator: ">",
-          value: new Date("1990-01-01"),
-        },
-      ],
-      projectId,
-      searchQuery: null,
-    });
-
-    if (result.type !== "data")
-      throw new Error("No data returned. Is S3 accidentally enabled?");
-    const { data, fileName } = result;
-
-    const fileExtension = fileName.split(".").pop();
-    expect(fileName).toContain(`lf-export-${projectId}`);
-    expect(fileExtension).toBe("json");
-
-    expect(JSON.parse(data).length).toBe(numberOfGenerations);
-  });
-
-  it("should throw on unsupported file formats", async () => {
-    const unsupportedFileFormat = "XLSX";
-
-    const call = caller.generations.export({
-      fileFormat: unsupportedFileFormat as unknown as BatchExportFileFormat,
-      orderBy: { column: "id", order: "ASC" },
-      filter: [
-        {
-          column: "Start Time",
-          type: "datetime",
-          operator: ">",
-          value: new Date("1990-01-01"),
-        },
-      ],
-      projectId,
-      searchQuery: null,
-    });
-
-    await expect(call).rejects.toThrow();
-  });
-});

web/src/__tests__/ingestion.servertest.ts

@@ -978,6 +978,39 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace.length).toBe(1);
   });
 
+  it("should fail for long trace name", async () => {
+    const traceId = v4();
+
+    const baseString =
+      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ";
+    const repeatCount = Math.ceil(1500 / baseString.length);
+    const name = baseString.repeat(repeatCount);
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name,
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+    expect("errors" in response.body).toBe(true);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(response.body.errors.length).toBe(1);
+    expect(response.body.errors[0].message).toBe("Invalid request data");
+  });
+
   it("should update all token counts if update does not contain model name", async () => {
     const traceId = v4();
     const generationId = v4();

web/src/__tests__/server/api/tables/prompts-ui.servertest.ts

@@ -1,4 +1,4 @@
-import { createObservations } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { createObservationsCh } from "@langfuse/shared/src/server";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
 import {
   getObservationMetricsForPrompts,
@@ -63,7 +63,7 @@ describe("UI Prompts Table", () => {
       id: v4(),
       prompt_name: null,
     };
-    await createObservations([
+    await createObservationsCh([
       observation,
       secondObservation,
       thirdObservation,
@@ -140,7 +140,7 @@ describe("UI Prompts Table", () => {
       prompt_version: 2,
       cost_details: { input: 234, output: 755 },
     };
-    await createObservations([
+    await createObservationsCh([
       observation,
       secondObservation,
       thirdObservation,
@@ -157,7 +157,7 @@ describe("UI Prompts Table", () => {
           firstObservation: expect.any(Date),
           lastObservation: expect.any(Date),
           medianInputUsage: 100,
-          medianLatencyMs: 0,
+          medianLatencyMs: expect.any(Number),
           medianOutputUsage: 5654,
           medianTotalCost: 0,
           promptId: "some-prompt-id",
@@ -168,7 +168,7 @@ describe("UI Prompts Table", () => {
           firstObservation: expect.any(Date),
           lastObservation: expect.any(Date),
           medianInputUsage: 100,
-          medianLatencyMs: 0,
+          medianLatencyMs: expect.any(Number),
           medianOutputUsage: 200,
           medianTotalCost: 0,
           promptId: "some-prompt-id",

web/src/__tests__/server/repositories/clickhouse-helpers.ts

@@ -1,42 +0,0 @@
-import {
-  clickhouseClient,
-  type ScoreRecordInsertType,
-  type ObservationRecordInsertType,
-  type TraceRecordInsertType,
-} from "@langfuse/shared/src/server";
-
-export const createTraces = async (trace: TraceRecordInsertType[]) => {
-  return await clickhouseClient().insert({
-    table: "traces",
-    format: "JSONEachRow",
-    values: trace,
-  });
-};
-
-export const createObservation = async (
-  observation: ObservationRecordInsertType,
-) => {
-  return await clickhouseClient().insert({
-    table: "observations",
-    format: "JSONEachRow",
-    values: [observation],
-  });
-};
-
-export const createObservations = async (
-  observations: ObservationRecordInsertType[],
-) => {
-  return await clickhouseClient().insert({
-    table: "observations",
-    format: "JSONEachRow",
-    values: observations,
-  });
-};
-
-export const createScores = async (scores: ScoreRecordInsertType[]) => {
-  return await clickhouseClient().insert({
-    table: "scores",
-    format: "JSONEachRow",
-    values: scores,
-  });
-};