chore: release v3.52.0

* feat: allow optional metadata param on org / project CRUD API

* chore: drop debugger statements

* chore: remove more debuggers

* chore: update tests

* chore: add metadata to fern spec

.env.prod.example

@@ -219,17 +219,6 @@ OTEL_SERVICE_NAME="langfuse"
 ### END Enterprise Edition Configuration
 
 
-
-### START Envs to be deprecated in Langfuse v3.0
-
-# Disable the expensive analytics queries and related features
-# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
-
-### END Envs to be deprecated in Langfuse v3.0
-
-
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
@@ -261,8 +250,9 @@ OTEL_SERVICE_NAME="langfuse"
 # NEXT_PUBLIC_DEMO_ORG_ID=
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
-# Crisp chat
-# NEXT_PUBLIC_CRISP_WEBSITE_ID=
+# Plain Chat
+# NEXT_PUBLIC_PLAIN_APP_ID=
+# PLAIN_AUTHENTICATION_SECRET=
 
 # Admin API
 # ADMIN_API_KEY=

.github/workflows/_deploy_ecs_service.yml

@@ -52,16 +52,11 @@ jobs:
             --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
             --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
             --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
-            --build-arg NEXT_PUBLIC_CRISP_WEBSITE_ID=${{ vars.NEXT_PUBLIC_CRISP_WEBSITE_ID }} \
+            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }} \
             --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
             --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
-            --build-arg NEXT_PUBLIC_DATADOG_APPLICATION_ID=${{ vars.NEXT_PUBLIC_DATADOG_APPLICATION_ID }} \
-            --build-arg NEXT_PUBLIC_DATADOG_CLIENT_TOKEN=${{ vars.NEXT_PUBLIC_DATADOG_CLIENT_TOKEN }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SITE=${{ vars.NEXT_PUBLIC_DATADOG_SITE }} \
-            --build-arg NEXT_PUBLIC_DATADOG_ENVIRONMENT=${{ vars.NEXT_PUBLIC_DATADOG_ENVIRONMENT }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SESSION_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_DATADOG_SESSION_SAMPLE_RATE }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SESSION_REPLAY_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_DATADOG_SESSION_REPLAY_SAMPLE_RATE }} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
       - name: Render AWS ECS Task Definition

.github/workflows/pipeline.yml

@@ -212,6 +212,9 @@ jobs:
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
+          echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100" >> .env
+          echo "ADMIN_API_KEY=admin-api-key" >> .env
+          echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
       - name: Run dev containers
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -338,14 +341,28 @@ jobs:
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed
       - name: Build
         run: pnpm run build

.gitignore

@@ -62,4 +62,6 @@ node_modules
 **/.next/*
 **/.turbo/*
 .yarn
-.turbo
+.turbo
+
+web/test-results/*

CONTRIBUTING.md

@@ -27,7 +27,7 @@ The maintainers are available on [Discord](https://langfuse.com/discord) in case
 
 ## Making a change
 
-_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone. Large changes that were not discussed in an issue may be rejected.
+_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone. Changes that were not discussed in an issue may be rejected.
 
 Once we've discussed your changes and you've got your code ready, make sure that tests are passing and open your pull request.
 

docker-compose.dev-azure.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default

docker-compose.dev.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default

docker-compose.yml

@@ -40,9 +40,18 @@ services:
       LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
       LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
       LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090}
       LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
       LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}
+      LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse}
+      LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/}
+      LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto}
+      LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090}
+      LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true}
       LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
       LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
       REDIS_HOST: ${REDIS_HOST:-redis}

ee/package.json

@@ -28,7 +28,7 @@
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
-    "next": "^14.2.25",
+    "next": "^14.2.26",
     "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },

fern/apis/client/definition/score.yml

@@ -21,6 +21,7 @@ types:
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       dataType:
         type: optional<ScoreDataType>
         docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
@@ -76,6 +77,7 @@ types:
       updatedAt: datetime
       authorUserId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       configId:
         type: optional<string>
         docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range

fern/apis/organizations/definition/api.yml

@@ -0,0 +1,32 @@
+name: langfuse-organizations
+docs: |
+  ## General Notes
+  
+  This admin API is only available on self-hosted instances, not on Langfuse Cloud.
+  An administrator must set the `ADMIN_API_KEY` environment variable to use this API.
+  
+  ## Authentication
+
+  Authenticate with the API by setting the Authorization header to `Bearer $ADMIN_API_KEY` where
+  `$ADMIN_API_KEY` is the API key set via the container environment.
+  
+  ## Exports
+
+  - OpenAPI spec: https://cloud.langfuse.com/generated/organizations-api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/organizations-postman/collection.json
+  
+  ## Looking for the Langfuse API?
+  
+  Are you looking for the full Langfuse API?
+  Check out our [API reference](https://api.reference.langfuse.com/#overview).
+
+error-discrimination:
+  strategy: status-code
+auth: bearer
+imports:
+  commons: commons.yml
+errors:
+  - commons.Error
+  - commons.UnauthorizedError
+  - commons.AccessDeniedError
+  - commons.MethodNotAllowedError

fern/apis/organizations/definition/commons.yml

@@ -0,0 +1,13 @@
+errors:
+  Error:
+    status-code: 400
+    type: string
+  UnauthorizedError:
+    status-code: 401
+    type: string
+  AccessDeniedError:
+    status-code: 403
+    type: string
+  MethodNotAllowedError:
+    status-code: 405
+    type: string

fern/apis/organizations/definition/organizations.yml

@@ -0,0 +1,170 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/admin
+  endpoints:
+    getAll:
+      docs: Get all organizations
+      method: GET
+      path: /organizations
+      response: Organizations
+      errors:
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    create:
+      docs: Create a new organization
+      method: POST
+      path: /organizations
+      request:
+        name: CreateOrganizationRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the organization
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    getById:
+      docs: Get organization by ID
+      method: GET
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    update:
+      docs: Update an organization
+      method: PUT
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      request:
+        name: UpdateOrganizationRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the organization
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    delete:
+      docs: Delete an organization
+      method: DELETE
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      response: DeleteOrganizationResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    # API Key endpoints
+    getApiKeys:
+      docs: Get all API keys for an organization
+      method: GET
+      path: /organizations/{organizationId}/apiKeys
+      path-parameters:
+        organizationId: string
+      response: ApiKeyList
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    createApiKey:
+      docs: Create a new API key for an organization
+      method: POST
+      path: /organizations/{organizationId}/apiKeys
+      path-parameters:
+        organizationId: string
+      request:
+        name: CreateApiKeyRequest
+        body:
+          properties:
+            note:
+              type: optional<string>
+              docs: Optional note for the API key
+      response: ApiKeyResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    deleteApiKey:
+      docs: Delete an API key for an organization
+      method: DELETE
+      path: /organizations/{organizationId}/apiKeys/{apiKeyId}
+      path-parameters:
+        organizationId: string
+        apiKeyId: string
+      response: DeleteOrganizationResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+types:
+  Organizations:
+    docs: List of organizations
+    properties:
+      organizations: list<Organization>
+
+  Organization:
+    docs: Organization object
+    properties:
+      id: string
+      name: string
+      createdAt: datetime
+      metadata:
+        type: map<string, unknown>
+        docs: Metadata for the organization
+
+  DeleteOrganizationResponse:
+    docs: Response for successful organization deletion
+    properties:
+      success: boolean
+
+  ApiKeyList:
+    docs: List of API keys for an organization
+    properties:
+      apiKeys: list<ApiKeySummary>
+
+  ApiKeySummary:
+    docs: Summary of an API key
+    properties:
+      id: string
+      createdAt: datetime
+      expiresAt: optional<datetime>
+      lastUsedAt: optional<datetime>
+      note: optional<string>
+      publicKey: string
+      displaySecretKey: string
+
+  ApiKeyResponse:
+    docs: Response for API key creation
+    properties:
+      id: string
+      createdAt: datetime
+      publicKey: string
+      secretKey: string
+      displaySecretKey: string
+      note: optional<string>

fern/apis/organizations/generators.yml

@@ -0,0 +1,17 @@
+default-group: local
+groups:
+  local:
+    generators:
+      - name: fernapi/fern-openapi
+        version: 0.1.7
+        output:
+          location: local-file-system
+          path: ../../../web/public/generated/organizations-api
+        config:
+          namespaceExport: Langfuse
+          allowCustomFetcher: true
+      - name: fernapi/fern-postman
+        version: 0.0.45
+        output:
+          location: local-file-system
+          path: ../../../web/public/generated/organizations-postman

fern/apis/server/definition/commons.yml

@@ -252,6 +252,7 @@ types:
       updatedAt: datetime
       authorUserId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       configId:
         type: optional<string>
         docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range

fern/apis/server/definition/ingestion.yml

@@ -271,6 +271,7 @@ types:
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       dataType:
         type: optional<commons.ScoreDataType>
         docs: When set, must match the score value's type. If not set, will be inferred from the score value or config

fern/apis/server/definition/memberships.yml

@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    getOrganizationMemberships:
+      docs: Get all memberships for the organization associated with the API key (requires organization-scoped API key)
+      method: GET
+      path: /organizations/memberships
+      response: MembershipsResponse
+
+    updateOrganizationMembership:
+      docs: Create or update a membership for the organization associated with the API key (requires organization-scoped API key)
+      method: PUT
+      path: /organizations/memberships
+      request: MembershipRequest
+      response: MembershipResponse
+
+    getProjectMemberships:
+      docs: Get all memberships for a specific project (requires organization-scoped API key)
+      method: GET
+      path: /projects/{projectId}/memberships
+      path-parameters:
+        projectId: string
+      response: MembershipsResponse
+
+    updateProjectMembership:
+      docs: Create or update a membership for a specific project (requires organization-scoped API key). The user must already be a member of the organization.
+      method: PUT
+      path: /projects/{projectId}/memberships
+      path-parameters:
+        projectId: string
+      request: MembershipRequest
+      response: MembershipResponse
+
+types:
+  MembershipRole:
+    enum:
+      - OWNER
+      - ADMIN
+      - MEMBER
+      - VIEWER
+
+  MembershipRequest:
+    properties:
+      userId: string
+      role: MembershipRole
+
+  MembershipResponse:
+    properties:
+      userId: string
+      role: MembershipRole
+      email: string
+      name: string
+
+  MembershipsResponse:
+    properties:
+      memberships: list<MembershipResponse>

fern/apis/server/definition/projects.yml

@@ -11,6 +11,85 @@ service:
       method: GET
       path: /projects
       response: Projects
+    
+    create:
+      docs: Create a new project (requires organization-scoped API key)
+      method: POST
+      path: /projects
+      request:
+        name: CreateProjectRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the project
+            retention:
+              type: integer
+              docs: Number of days to retain data. Must be 0 or at least 7 days. Requires data-retention entitlement for non-zero values. Optional.
+      response: Project
+    
+    update:
+      docs: Update a project by ID (requires organization-scoped API key).
+      method: PUT
+      path: /projects/{projectId}
+      path-parameters:
+        projectId: string
+      request:
+        name: UpdateProjectRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the project
+            retention:
+              type: integer
+              docs: Number of days to retain data. Must be 0 or at least 7 days. Requires data-retention entitlement for non-zero values. Optional.
+      response: Project
+    
+    delete:
+      docs: Delete a project by ID (requires organization-scoped API key). Project deletion is processed asynchronously.
+      method: DELETE
+      path: /projects/{projectId}
+      path-parameters:
+        projectId: string
+      response:
+        status-code: 202
+        type: ProjectDeletionResponse
+        
+    # API Key endpoints
+    getApiKeys:
+      docs: Get all API keys for a project (requires organization-scoped API key)
+      method: GET
+      path: /projects/{projectId}/apiKeys
+      path-parameters:
+        projectId: string
+      response: ApiKeyList
+
+    createApiKey:
+      docs: Create a new API key for a project (requires organization-scoped API key)
+      method: POST
+      path: /projects/{projectId}/apiKeys
+      path-parameters:
+        projectId: string
+      request:
+        name: CreateApiKeyRequest
+        body:
+          properties:
+            note:
+              type: optional<string>
+              docs: Optional note for the API key
+      response: ApiKeyResponse
+
+    deleteApiKey:
+      docs: Delete an API key for a project (requires organization-scoped API key)
+      method: DELETE
+      path: /projects/{projectId}/apiKeys/{apiKeyId}
+      path-parameters:
+        projectId: string
+        apiKeyId: string
+      response: ApiKeyDeletionResponse
 
 types:
   Projects:
@@ -21,3 +100,45 @@ types:
     properties:
       id: string
       name: string
+      metadata:
+        type: map<string, unknown>
+        docs: Metadata for the project
+      retentionDays:
+        type: integer
+        docs: Number of days to retain data. Null or 0 means no retention. Omitted if no retention is configured.
+
+  ProjectDeletionResponse:
+    properties:
+      success: boolean
+      message: string
+      
+  ApiKeyList:
+    docs: List of API keys for a project
+    properties:
+      apiKeys: list<ApiKeySummary>
+
+  ApiKeySummary:
+    docs: Summary of an API key
+    properties:
+      id: string
+      createdAt: datetime
+      expiresAt: optional<datetime>
+      lastUsedAt: optional<datetime>
+      note: optional<string>
+      publicKey: string
+      displaySecretKey: string
+
+  ApiKeyResponse:
+    docs: Response for API key creation
+    properties:
+      id: string
+      createdAt: datetime
+      publicKey: string
+      secretKey: string
+      displaySecretKey: string
+      note: optional<string>
+      
+  ApiKeyDeletionResponse:
+    docs: Response for API key deletion
+    properties:
+      success: boolean

fern/apis/server/definition/scim.yml

@@ -0,0 +1,199 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/public/scim
+  endpoints:
+    getServiceProviderConfig:
+      docs: Get SCIM Service Provider Configuration (requires organization-scoped API key)
+      method: GET
+      path: /ServiceProviderConfig
+      response: ServiceProviderConfig
+
+    getResourceTypes:
+      docs: Get SCIM Resource Types (requires organization-scoped API key)
+      method: GET
+      path: /ResourceTypes
+      response: ResourceTypesResponse
+
+    getSchemas:
+      docs: Get SCIM Schemas (requires organization-scoped API key)
+      method: GET
+      path: /Schemas
+      response: SchemasResponse
+
+    listUsers:
+      docs: List users in the organization (requires organization-scoped API key)
+      method: GET
+      path: /Users
+      request:
+        name: ListUsersRequest
+        query-parameters:
+          filter:
+            docs: Filter expression (e.g. userName eq "value")
+            type: optional<string>
+          startIndex:
+            docs: 1-based index of the first result to return (default 1)
+            type: optional<integer>
+          count:
+            docs: Maximum number of results to return (default 100)
+            type: optional<integer>
+      response: ScimUsersListResponse
+
+    createUser:
+      docs: Create a new user in the organization (requires organization-scoped API key)
+      method: POST
+      path: /Users
+      request:
+        name: CreateUserRequest
+        body:
+          properties:
+            userName:
+              docs: User's email address (required)
+              type: string
+            name:
+              docs: User's name information
+              type: ScimName
+            emails:
+              docs: User's email addresses
+              type: optional<list<ScimEmail>>
+            active:
+              docs: Whether the user is active
+              type: optional<boolean>
+            password:
+              docs: Initial password for the user
+              type: optional<string>
+      response: ScimUser
+
+    getUser:
+      docs: Get a specific user by ID (requires organization-scoped API key)
+      method: GET
+      path: /Users/{userId}
+      path-parameters:
+        userId: string
+      response: ScimUser
+
+    deleteUser:
+      docs: Remove a user from the organization (requires organization-scoped API key). Note that this only removes the user from the organization but does not delete the user entity itself.
+      method: DELETE
+      path: /Users/{userId}
+      path-parameters:
+        userId: string
+      response: EmptyResponse
+
+types:
+  ServiceProviderConfig:
+    properties:
+      schemas: list<string>
+      documentationUri: string
+      patch: ScimFeatureSupport
+      bulk: BulkConfig
+      filter: FilterConfig
+      changePassword: ScimFeatureSupport
+      sort: ScimFeatureSupport
+      etag: ScimFeatureSupport
+      authenticationSchemes: list<AuthenticationScheme>
+      meta: ResourceMeta
+
+  ScimFeatureSupport:
+    properties:
+      supported: boolean
+
+  BulkConfig:
+    properties:
+      supported: boolean
+      maxOperations: integer
+      maxPayloadSize: integer
+
+  FilterConfig:
+    properties:
+      supported: boolean
+      maxResults: integer
+
+  ResourceMeta:
+    properties:
+      resourceType: string
+      location: string
+
+  AuthenticationScheme:
+    properties:
+      name: string
+      description: string
+      specUri: string
+      type: string
+      primary: boolean
+
+  ResourceTypesResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      Resources: list<ResourceType>
+
+  ResourceType:
+    properties:
+      schemas: optional<list<string>>
+      id: string
+      name: string
+      endpoint: string
+      description: string
+      schema: string
+      schemaExtensions: list<SchemaExtension>
+      meta: ResourceMeta
+
+  SchemaExtension:
+    properties:
+      schema: string
+      required: boolean
+
+  SchemasResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      Resources: list<SchemaResource>
+
+  SchemaResource:
+    properties:
+      id: string
+      name: string
+      description: string
+      attributes: list<unknown>
+      meta: ResourceMeta
+
+  ScimUsersListResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      startIndex: integer
+      itemsPerPage: integer
+      Resources: list<ScimUser>
+
+  ScimUser:
+    properties:
+      schemas: list<string>
+      id: string
+      userName: string
+      name: ScimName
+      emails: list<ScimEmail>
+      meta: UserMeta
+
+  UserMeta:
+    properties:
+      resourceType: string
+      created: optional<string>
+      lastModified: optional<string>
+
+  ScimName:
+    properties:
+      formatted: optional<string>
+
+  ScimEmail:
+    properties:
+      primary: boolean
+      value: string
+      type: string
+
+  EmptyResponse:
+    docs: Empty response for 204 No Content responses
+    properties: {}

fern/apis/server/definition/score.yml

@@ -95,6 +95,7 @@ types:
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
       observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       environment:
         type: optional<string>
         docs: The environment of the score. Can be any lowercase alphanumeric string with hyphens and underscores that does not start with 'langfuse'.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.45.2",
+  "version": "3.52.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/clickhouse/migrations/clustered/0010_add_metadata_column_on_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS metadata;

packages/shared/clickhouse/migrations/clustered/0010_add_metadata_column_on_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default ADD COLUMN metadata Map(LowCardinality(String), String) AFTER comment;

packages/shared/clickhouse/migrations/unclustered/0010_add_metadata_column_on_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores DROP COLUMN IF EXISTS metadata;

packages/shared/clickhouse/migrations/unclustered/0010_add_metadata_column_on_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ADD COLUMN metadata Map(LowCardinality(String), String) AFTER comment;

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 64",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 65",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -61,6 +61,7 @@
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
+    "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.12",
     "@langchain/aws": "^0.1.3",
     "@langchain/core": "^0.3.37",

packages/shared/prisma/generated/types.ts

@@ -4,6 +4,11 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
+export const ApiKeyScope = {
+    ORGANIZATION: "ORGANIZATION",
+    PROJECT: "PROJECT"
+} as const;
+export type ApiKeyScope = (typeof ApiKeyScope)[keyof typeof ApiKeyScope];
 export const Role = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
@@ -80,12 +85,33 @@ export const JobExecutionStatus = {
     CANCELLED: "CANCELLED"
 } as const;
 export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
+export const BlobStorageIntegrationFileType = {
+    JSON: "JSON",
+    CSV: "CSV",
+    JSONL: "JSONL"
+} as const;
+export type BlobStorageIntegrationFileType = (typeof BlobStorageIntegrationFileType)[keyof typeof BlobStorageIntegrationFileType];
 export const BlobStorageIntegrationType = {
     S3: "S3",
     S3_COMPATIBLE: "S3_COMPATIBLE",
     AZURE_BLOB_STORAGE: "AZURE_BLOB_STORAGE"
 } as const;
 export type BlobStorageIntegrationType = (typeof BlobStorageIntegrationType)[keyof typeof BlobStorageIntegrationType];
+export const DashboardWidgetViews = {
+    TRACES: "TRACES",
+    OBSERVATIONS: "OBSERVATIONS",
+    SCORES_NUMERIC: "SCORES_NUMERIC",
+    SCORES_CATEGORICAL: "SCORES_CATEGORICAL"
+} as const;
+export type DashboardWidgetViews = (typeof DashboardWidgetViews)[keyof typeof DashboardWidgetViews];
+export const DashboardWidgetChartType = {
+    LINE_TIME_SERIES: "LINE_TIME_SERIES",
+    BAR_TIME_SERIES: "BAR_TIME_SERIES",
+    HORIZONTAL_BAR: "HORIZONTAL_BAR",
+    VERTICAL_BAR: "VERTICAL_BAR",
+    PIE: "PIE"
+} as const;
+export type DashboardWidgetChartType = (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export type Account = {
     id: string;
     user_id: string;
@@ -137,7 +163,9 @@ export type ApiKey = {
     display_secret_key: string;
     last_used_at: Timestamp | null;
     expires_at: Timestamp | null;
-    project_id: string;
+    project_id: string | null;
+    organization_id: string | null;
+    scope: Generated<ApiKeyScope>;
 };
 export type AuditLog = {
     id: string;
@@ -208,6 +236,7 @@ export type BlobStorageIntegration = {
     last_sync_at: Timestamp | null;
     enabled: boolean;
     export_frequency: string;
+    file_type: Generated<BlobStorageIntegrationFileType>;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -227,6 +256,33 @@ export type CronJobs = {
     job_started_at: Timestamp | null;
     state: string | null;
 };
+export type Dashboard = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    created_by: string | null;
+    updated_by: string | null;
+    project_id: string | null;
+    name: string;
+    description: string;
+    definition: unknown;
+};
+export type DashboardWidget = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    created_by: string | null;
+    updated_by: string | null;
+    project_id: string | null;
+    name: string;
+    description: string;
+    view: DashboardWidgetViews;
+    dimensions: unknown;
+    metrics: unknown;
+    filters: unknown;
+    chart_type: DashboardWidgetChartType;
+    chart_config: unknown;
+};
 export type Dataset = {
     id: string;
     project_id: string;
@@ -401,6 +457,24 @@ export type LlmApiKeys = {
     config: unknown | null;
     project_id: string;
 };
+export type LlmSchema = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    schema: unknown;
+};
+export type LlmTool = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    parameters: unknown;
+};
 export type Media = {
     id: string;
     sha_256_hash: string;
@@ -457,6 +531,7 @@ export type Organization = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     cloud_config: unknown | null;
+    metadata: unknown | null;
 };
 export type OrganizationMembership = {
     id: string;
@@ -490,6 +565,7 @@ export type Project = {
     deleted_at: Timestamp | null;
     name: string;
     retention_days: number | null;
+    metadata: unknown | null;
 };
 export type ProjectMembership = {
     org_membership_id: string;
@@ -525,6 +601,13 @@ export type PromptDependency = {
     child_label: string | null;
     child_version: number | null;
 };
+export type PromptProtectedLabels = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    label: string;
+};
 export type ScoreConfig = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -598,6 +681,8 @@ export type DB = {
     blob_storage_integrations: BlobStorageIntegration;
     comments: Comment;
     cron_jobs: CronJobs;
+    dashboard_widgets: DashboardWidget;
+    dashboards: Dashboard;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
@@ -606,6 +691,8 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    llm_schemas: LlmSchema;
+    llm_tools: LlmTool;
     media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
@@ -618,6 +705,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompt_dependencies: PromptDependency;
+    prompt_protected_labels: PromptProtectedLabels;
     prompts: Prompt;
     score_configs: ScoreConfig;
     scores: LegacyPrismaScore;

packages/shared/prisma/migrations/20250326180640_add_llm_tools_and_schemas_tables/migration.sql

@@ -0,0 +1,37 @@
+-- CreateTable
+CREATE TABLE "llm_schemas" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "schema" JSON NOT NULL,
+
+    CONSTRAINT "llm_schemas_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "llm_tools" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "parameters" JSON NOT NULL,
+
+    CONSTRAINT "llm_tools_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_schemas_project_id_name_key" ON "llm_schemas"("project_id", "name");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_tools_project_id_name_key" ON "llm_tools"("project_id", "name");
+
+-- AddForeignKey
+ALTER TABLE "llm_schemas" ADD CONSTRAINT "llm_schemas_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "llm_tools" ADD CONSTRAINT "llm_tools_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250401122159_add_prompt_protected_labels_table/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "prompt_protected_labels" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "label" TEXT NOT NULL,
+
+    CONSTRAINT "prompt_protected_labels_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "prompt_protected_labels_project_id_label_key" ON "prompt_protected_labels"("project_id", "label");
+
+-- AddForeignKey
+ALTER TABLE "prompt_protected_labels" ADD CONSTRAINT "prompt_protected_labels_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250402142320_add_blobstorage_integration_file_type/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "BlobStorageIntegrationFileType" AS ENUM ('JSON', 'CSV', 'JSONL');
+
+-- AlterTable
+ALTER TABLE "blob_storage_integrations" ADD COLUMN     "file_type" "BlobStorageIntegrationFileType" NOT NULL DEFAULT 'CSV';

packages/shared/prisma/migrations/20250403153555_membership_invitations_no_duplicates/migration.sql

@@ -0,0 +1,17 @@
+-- Delete duplicate membership invitations, keeping only the newest one for each email and org_id combination
+WITH ranked_invitations AS (
+  SELECT 
+    id,
+    email,
+    org_id,
+    ROW_NUMBER() OVER (PARTITION BY email, org_id ORDER BY updated_at DESC) as rn
+  FROM membership_invitations
+)
+DELETE FROM membership_invitations
+WHERE id IN (
+  SELECT id FROM ranked_invitations WHERE rn > 1
+);
+
+
+-- CreateIndex
+CREATE UNIQUE INDEX "membership_invitations_email_org_id_key" ON "membership_invitations"("email", "org_id");

packages/shared/prisma/migrations/20250409154352_add_dashboard_data_model/migration.sql

@@ -0,0 +1,58 @@
+-- CreateEnum
+CREATE TYPE "DashboardWidgetViews" AS ENUM ('TRACES', 'OBSERVATIONS', 'SCORES_NUMERIC', 'SCORES_CATEGORICAL');
+
+-- CreateEnum
+CREATE TYPE "DashboardWidgetChartType" AS ENUM ('LINE_TIME_SERIES', 'BAR_TIME_SERIES', 'HORIZONTAL_BAR', 'VERTICAL_BAR', 'PIE');
+
+-- CreateTable
+CREATE TABLE "dashboards" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_by" TEXT,
+    "project_id" TEXT,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "definition" JSONB NOT NULL,
+
+    CONSTRAINT "dashboards_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "dashboard_widgets" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_by" TEXT,
+    "project_id" TEXT,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "view" "DashboardWidgetViews" NOT NULL,
+    "dimensions" JSONB NOT NULL,
+    "metrics" JSONB NOT NULL,
+    "filters" JSONB NOT NULL,
+    "chart_type" "DashboardWidgetChartType" NOT NULL,
+    "chart_config" JSONB NOT NULL,
+
+    CONSTRAINT "dashboard_widgets_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250410145712_add_organization_scoped_api_keys/migration.sql

@@ -0,0 +1,13 @@
+-- CreateEnum
+CREATE TYPE "ApiKeyScope" AS ENUM ('ORGANIZATION', 'PROJECT');
+
+-- AlterTable
+ALTER TABLE "api_keys" ADD COLUMN     "organization_id" TEXT,
+ADD COLUMN     "scope" "ApiKeyScope" NOT NULL DEFAULT 'PROJECT',
+ALTER COLUMN "project_id" DROP NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "api_keys_organization_id_idx" ON "api_keys"("organization_id");
+
+-- AddForeignKey
+ALTER TABLE "api_keys" ADD CONSTRAINT "api_keys_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250420120553_add_organization_and_project_metadata/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "organizations" ADD COLUMN     "metadata" JSONB;
+
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN     "metadata" JSONB;

packages/shared/prisma/schema.prisma

@@ -82,6 +82,10 @@ model User {
   featureFlags            String[]                 @default([]) @map("feature_flags")
   annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
   annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
+  dashboardWidgetsCreated DashboardWidget[]        @relation("CreatedByUser")
+  dashboardWidgetsUpdated DashboardWidget[]        @relation("UpdatedByUser")
+  dashboardCreated        Dashboard[]              @relation("CreatedByUser")
+  dashboardUpdated        Dashboard[]              @relation("UpdatedByUser")
 
   @@map("users")
 }
@@ -101,9 +105,11 @@ model Organization {
   createdAt               DateTime                 @default(now()) @map("created_at")
   updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
   cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+  metadata                Json?
   organizationMemberships OrganizationMembership[]
   projects                Project[]
   MembershipInvitation    MembershipInvitation[]
+  ApiKey                  ApiKey[]
 
   @@map("organizations")
 }
@@ -116,6 +122,7 @@ model Project {
   deletedAt              DateTime?                 @map("deleted_at")
   name                   String
   retentionDays          Int?                      @map("retention_days")
+  metadata               Json?
   projectMembers         ProjectMembership[]
   organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
   apiKeys                ApiKey[]
@@ -142,24 +149,38 @@ model Project {
   LegacyObservation      LegacyPrismaObservation[]
   LegacyScore            LegacyPrismaScore[]
   PromptDependency       PromptDependency[]
+  LlmSchema              LlmSchema[]
+  LlmTool                LlmTool[]
+  PromptProtectedLabels  PromptProtectedLabels[]
+  Dashboard              Dashboard[]
+  DashboardWidget        DashboardWidget[]
 
   @@index([orgId])
   @@map("projects")
 }
 
-model ApiKey {
-  id                  String    @id @unique @default(cuid())
-  createdAt           DateTime  @default(now()) @map("created_at")
-  note                String?
-  publicKey           String    @unique @map("public_key")
-  hashedSecretKey     String    @unique @map("hashed_secret_key")
-  fastHashedSecretKey String?   @unique @map("fast_hashed_secret_key")
-  displaySecretKey    String    @map("display_secret_key")
-  lastUsedAt          DateTime? @map("last_used_at")
-  expiresAt           DateTime? @map("expires_at")
-  projectId           String    @map("project_id")
-  project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+enum ApiKeyScope {
+  ORGANIZATION
+  PROJECT
+}
 
+model ApiKey {
+  id                  String        @id @unique @default(cuid())
+  createdAt           DateTime      @default(now()) @map("created_at")
+  note                String?
+  publicKey           String        @unique @map("public_key")
+  hashedSecretKey     String        @unique @map("hashed_secret_key")
+  fastHashedSecretKey String?       @unique @map("fast_hashed_secret_key")
+  displaySecretKey    String        @map("display_secret_key")
+  lastUsedAt          DateTime?     @map("last_used_at")
+  expiresAt           DateTime?     @map("expires_at")
+  projectId           String?       @map("project_id")
+  project             Project?      @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  orgId               String?       @map("organization_id")
+  organization        Organization? @relation(fields: [orgId], references: [id], onDelete: Cascade)
+  scope               ApiKeyScope   @default(PROJECT) @map("scope")
+
+  @@index(orgId)
   @@index(projectId)
   @@index(publicKey)
   @@index(hashedSecretKey)
@@ -254,6 +275,7 @@ model MembershipInvitation {
   createdAt       DateTime     @default(now()) @map("created_at")
   updatedAt       DateTime     @default(now()) @updatedAt @map("updated_at")
 
+  @@unique([email, orgId]) // do not include projectId as this leads to issues when processing the invites, needs new logic
   @@index([projectId])
   @@index([orgId])
   @@index([email])
@@ -681,6 +703,18 @@ model PromptDependency {
   @@map("prompt_dependencies")
 }
 
+model PromptProtectedLabels {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  projectId String   @map("project_id")
+  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  label     String
+
+  @@unique([projectId, label])
+  @@map("prompt_protected_labels")
+}
+
 // Update ObservationView below when making changes to this model!
 model Model {
   id        String   @id @default(cuid())
@@ -859,6 +893,7 @@ model SsoConfig {
   authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
 
   authConfig Json? @map("auth_config")
+
   // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
   // secrets like clientSecret are encrypted on the application level
   @@map("sso_configs")
@@ -891,10 +926,11 @@ model BlobStorageIntegration {
   forcePathStyle  Boolean                    @map("force_path_style")
 
   // Integration Config
-  nextSyncAt      DateTime? @map("next_sync_at")
-  lastSyncAt      DateTime? @map("last_sync_at")
+  nextSyncAt      DateTime?                      @map("next_sync_at")
+  lastSyncAt      DateTime?                      @map("last_sync_at")
   enabled         Boolean
-  exportFrequency String    @map("export_frequency")
+  exportFrequency String                         @map("export_frequency")
+  fileType        BlobStorageIntegrationFileType @default(CSV) @map("file_type")
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
@@ -902,6 +938,14 @@ model BlobStorageIntegration {
   @@map("blob_storage_integrations")
 }
 
+enum BlobStorageIntegrationFileType {
+  JSON
+  CSV
+  JSONL
+
+  @@map("BlobStorageIntegrationFileType")
+}
+
 enum BlobStorageIntegrationType {
   S3
   S3_COMPATIBLE
@@ -1009,3 +1053,100 @@ model BillingMeterBackup {
   @@index([stripeCustomerId, meterId, startTime, endTime])
   @@map("billing_meter_backups")
 }
+
+model LlmSchema {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  schema      Json   @map("schema") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_schemas")
+}
+
+model LlmTool {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  parameters  Json   @map("parameters") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_tools")
+}
+
+model Dashboard {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  createdBy     String? @map("created_by")
+  createdByUser User?   @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+  updatedBy     String? @map("updated_by")
+  updatedByUser User?   @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+  // Optional to account for Langfuse-managed dashboards
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+
+  definition Json @map("definition")
+
+  @@map("dashboards")
+}
+
+enum DashboardWidgetViews {
+  TRACES
+  OBSERVATIONS
+  SCORES_NUMERIC
+  SCORES_CATEGORICAL
+}
+
+enum DashboardWidgetChartType {
+  LINE_TIME_SERIES
+  BAR_TIME_SERIES
+  HORIZONTAL_BAR
+  VERTICAL_BAR
+  PIE
+}
+
+model DashboardWidget {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  createdBy     String? @map("created_by")
+  createdByUser User?   @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+  updatedBy     String? @map("updated_by")
+  updatedByUser User?   @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+  // Optional to account for Langfuse-managed dashboards
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+
+  view       DashboardWidgetViews @map("view")
+  dimensions Json                 @map("dimensions")
+  metrics    Json                 @map("metrics")
+  filters    Json                 @map("filters")
+
+  chartType   DashboardWidgetChartType @map("chart_type")
+  chartConfig Json                     @map("chart_config")
+
+  @@map("dashboard_widgets")
+}

packages/shared/prisma/seed.ts

@@ -14,6 +14,7 @@ import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey, logger } from "../src/server";
 import { encrypt } from "../src/encryption";
 import { redis } from "../src/server/redis/redis";
+import { randomUUID } from "crypto";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -180,6 +181,7 @@ async function main() {
         publicKey: seedApiKey.public,
         hashedSecretKey: await hashSecretKey(seedApiKey.secret),
         displaySecretKey: getDisplaySecretKey(seedApiKey.secret),
+        scope: "PROJECT",
         project: {
           connect: {
             id: project1.id,
@@ -241,6 +243,7 @@ async function main() {
           publicKey: secondKey.public,
           hashedSecretKey: await hashSecretKey(secondKey.secret),
           displaySecretKey: getDisplaySecretKey(secondKey.secret),
+          scope: "PROJECT",
           project: {
             connect: {
               id: project2.id,
@@ -374,6 +377,48 @@ async function main() {
     });
 
     await createDatasets(project1, project2, observations);
+
+    await createDashboardsAndWidgets([project1, project2]);
+
+    await prisma.llmSchema.createMany({
+      data: [
+        {
+          projectId: project1.id,
+          name: "get_weather",
+          description: "Fetches weather in Celsius for a given location",
+          schema: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+              unit: {
+                type: "string",
+                enum: ["celsius", "fahrenheit"],
+              },
+            },
+            required: ["location", "unit"],
+          },
+        },
+        {
+          projectId: project1.id,
+          name: "calculator",
+          description: "Performs basic arithmetic calculations",
+          schema: {
+            type: "object",
+            properties: {
+              expression: {
+                type: "string",
+                description:
+                  "The mathematical expression to evaluate, e.g. '2 + 2'",
+              },
+            },
+            required: ["expression"],
+          },
+        },
+      ],
+    });
   }
 }
 
@@ -391,6 +436,85 @@ main()
     process.exit(1);
   });
 
+async function createDashboardsAndWidgets(projects: Project[]) {
+  logger.info("Creating dashboards and widgets");
+
+  // Process each project
+  for (const project of projects) {
+    const widget = await prisma.dashboardWidget.upsert({
+      where: { id: "cabc" },
+      create: {
+        id: "cabc",
+        projectId: project.id,
+        name: "Trace Counts",
+        description: "Trace Counts by Name Over Time",
+        view: "TRACES",
+        dimensions: [{ field: "name" }],
+        metrics: [{ measure: "count", agg: "count" }],
+        filters: [],
+        chartType: "BAR_TIME_SERIES",
+        chartConfig: {
+          type: "BAR_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    const widget2 = await prisma.dashboardWidget.upsert({
+      where: { id: "cdef" },
+      create: {
+        id: "cdef",
+        projectId: project.id,
+        name: "Observation Latencies by Model",
+        description: "p95 Observation Latencies by Model Name",
+        view: "OBSERVATIONS",
+        dimensions: [{ field: "providedModelName" }],
+        metrics: [{ measure: "count", agg: "sum" }],
+        filters: [],
+        chartType: "LINE_TIME_SERIES",
+        chartConfig: {
+          type: "LINE_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    // Create a dashboard with multiple widgets
+    await prisma.dashboard.upsert({
+      where: { id: "seed-dashboard" },
+      create: {
+        id: "seed-dashboard",
+        projectId: project.id,
+        name: "Performance Overview",
+        description: "Dashboard with various performance metrics",
+        definition: {
+          widgets: [
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget.id,
+              x: 0,
+              y: 0,
+              x_size: 6,
+              y_size: 2,
+            },
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget2.id,
+              x: 6,
+              y: 0,
+              x_size: 6,
+              y_size: 2,
+            },
+          ],
+        },
+      },
+      update: {},
+    });
+  }
+}
+
 export async function createDatasets(
   project1: {
     id: string;
@@ -730,6 +854,7 @@ function createObjects(
               source: "API",
               projectId,
               dataType: ScoreDataType.NUMERIC,
+              metadata: {},
             },
           ]
         : []),
@@ -745,6 +870,7 @@ function createObjects(
               dataType: ScoreDataType.CATEGORICAL,
               stringValue:
                 Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+              metadata: {},
             },
           ]
         : []),

packages/shared/scripts/load-seed.ts

@@ -58,6 +58,7 @@ const prepareProjectsAndApiKeys = async (
           publicKey: `pk-${Math.random().toString(36).substr(2, 9)}`,
           hashedSecretKey: sk,
           displaySecretKey: getDisplaySecretKey(sk),
+          scope: "PROJECT",
           project: {
             connect: {
               id: projectId,

packages/shared/scripts/prepareClickhouse.ts

@@ -159,6 +159,7 @@ export const prepareClickhouse = async (
       randUniform(0, 100) as value,
       'API' as source,
       'comment' as comment,
+      map('key', 'value') AS metadata,
       toString(rand() % 100) as author_user_id,
       toString(rand() % 100) as config_id,
       if (rand() < 0.33, 'NUMERIC', if (rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) as data_type,

packages/shared/src/domain/index.ts

@@ -0,0 +1,3 @@
+export * from "./observations";
+export * from "./traces";
+export * from "./scores";

packages/shared/src/domain/observations.ts

@@ -0,0 +1,76 @@
+import z from "zod";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const ObservationType = {
+  SPAN: "SPAN",
+  EVENT: "EVENT",
+  GENERATION: "GENERATION",
+} as const;
+export const ObservationTypeDomain = z.enum(["SPAN", "EVENT", "GENERATION"]);
+export type ObservationType = z.infer<typeof ObservationTypeDomain>;
+
+export const ObservationLevel = {
+  DEBUG: "DEBUG",
+  DEFAULT: "DEFAULT",
+  WARNING: "WARNING",
+  ERROR: "ERROR",
+} as const;
+export const ObservationLevelDomain = z.enum([
+  "DEBUG",
+  "DEFAULT",
+  "WARNING",
+  "ERROR",
+]);
+export type ObservationLevelType = z.infer<typeof ObservationLevelDomain>;
+
+export const ObservationDomain = z.object({
+  id: z.string(),
+  name: z.string(),
+  timestamp: z.date(),
+  environment: z.string(),
+  tags: z.array(z.string()),
+});
+export type ObservationDomain = z.infer<typeof ObservationDomain>;
+
+export const ObservationSchema = z.object({
+  id: z.string(),
+  traceId: z.string().nullable(),
+  projectId: z.string(),
+  environment: z.string(),
+  type: ObservationTypeDomain,
+  startTime: z.date(),
+  endTime: z.date().nullable(),
+  name: z.string().nullable(),
+  metadata: MetadataDomain,
+  parentObservationId: z.string().nullable(),
+  level: ObservationLevelDomain,
+  statusMessage: z.string().nullable(),
+  version: z.string().nullable(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  model: z.string().nullable(),
+  internalModelId: z.string().nullable(),
+  modelParameters: jsonSchema.nullable(),
+  input: jsonSchema.nullable(),
+  output: jsonSchema.nullable(),
+  completionStartTime: z.date().nullable(),
+  promptId: z.string().nullable(),
+  promptName: z.string().nullable(),
+  promptVersion: z.number().nullable(),
+  latency: z.number().nullable(),
+  timeToFirstToken: z.number().nullable(),
+  usageDetails: z.record(z.string(), z.number()),
+  costDetails: z.record(z.string(), z.number()),
+  providedCostDetails: z.record(z.string(), z.number()),
+  // aggregated data from cost_details
+  inputCost: z.number().nullable(),
+  outputCost: z.number().nullable(),
+  totalCost: z.number().nullable(),
+  // aggregated data from usage_details
+  inputUsage: z.number(),
+  outputUsage: z.number(),
+  totalUsage: z.number(),
+});
+
+export type Observation = z.infer<typeof ObservationSchema>;

packages/shared/src/domain/scores.ts

@@ -0,0 +1,35 @@
+import { ScoreDataType } from "@prisma/client";
+import z from "zod";
+import { MetadataDomain } from "./traces";
+
+export const ScoreSource = {
+  ANNOTATION: "ANNOTATION",
+  API: "API",
+  EVAL: "EVAL",
+} as const;
+
+export const ScoreSourceDomain = z.enum(["ANNOTATION", "API", "EVAL"]);
+export type ScoreSourceType = z.infer<typeof ScoreSourceDomain>;
+
+export const ScoreSchema = z.object({
+  id: z.string(),
+  timestamp: z.date(),
+  projectId: z.string(),
+  environment: z.string(),
+  name: z.string(),
+  value: z.number().nullable(),
+  source: ScoreSourceDomain,
+  authorUserId: z.string().nullable(),
+  comment: z.string().nullable(),
+  metadata: MetadataDomain,
+  traceId: z.string(),
+  observationId: z.string().nullable(),
+  configId: z.string().nullable(),
+  stringValue: z.string().nullable(),
+  queueId: z.string().nullable(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  dataType: z.nativeEnum(ScoreDataType),
+});
+
+export type ScoreDomain = z.infer<typeof ScoreSchema>;

packages/shared/src/domain/traces.ts

@@ -0,0 +1,32 @@
+import z from "zod";
+import { jsonSchema, jsonSchemaNullable } from "../utils/zod";
+
+export const MetadataDomain = z.record(
+  z.string(),
+  jsonSchemaNullable.or(z.undefined()),
+);
+
+export type MetadataDomain = z.infer<typeof MetadataDomain>;
+
+// to be used across the application in frontend and backend.
+export const TraceDomain = z.object({
+  id: z.string(),
+  name: z.string().nullable(),
+  timestamp: z.date(),
+  environment: z.string(),
+  tags: z.array(z.string()),
+  bookmarked: z.boolean(),
+  public: z.boolean(),
+  release: z.string().nullable(),
+  version: z.string().nullable(),
+  input: jsonSchema.nullable(),
+  output: jsonSchema.nullable(),
+  metadata: MetadataDomain,
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  sessionId: z.string().nullable(),
+  userId: z.string().nullable(),
+  projectId: z.string(),
+});
+
+export type TraceDomain = z.infer<typeof TraceDomain>;

packages/shared/src/env.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
+  NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
@@ -63,6 +64,8 @@ const EnvSchema = z.object({
     .enum(["true", "false"])
     .default("false"),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
   LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z

packages/shared/src/errors/ConflictError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class LangfuseConflictError extends BaseError {
+  constructor(description = "Conflict") {
+    super("LangfuseConflictError", 409, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -6,3 +6,4 @@ export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
 export { InternalServerError } from "./InternalServerError";
+export { LangfuseConflictError } from "./ConflictError";

packages/shared/src/features/batchExport/types.ts

@@ -15,6 +15,7 @@ export enum BatchExportStatus {
 export enum BatchExportFileFormat {
   JSON = "JSON",
   CSV = "CSV",
+  JSONL = "JSONL",
 }
 
 export enum BatchExportTableName {
@@ -35,6 +36,11 @@ export const exportOptions: Record<
 > = {
   CSV: { label: "CSV", extension: "csv", fileType: "text/csv" },
   JSON: { label: "JSON", extension: "json", fileType: "application/json" },
+  JSONL: {
+    label: "JSONL",
+    extension: "jsonl",
+    fileType: "application/x-ndjson",
+  },
 } as const;
 
 export const BatchExportQuerySchema = z.object({

packages/shared/src/features/scores/scoreTypes.ts

@@ -1,13 +1,14 @@
 import { z } from "zod";
 
 import { isPresent, stringDateTime } from "../../utils/typeChecks";
-import { Score } from "../../server";
 import {
+  jsonSchema,
   NonEmptyString,
   paginationMetaResponseZod,
   publicApiPaginationZod,
 } from "../../utils/zod";
 import { Category as ConfigCategory } from "./scoreConfigTypes";
+import { ScoreDomain } from "../../domain";
 
 /**
  * Types to use across codebase
@@ -50,6 +51,7 @@ const ScoreBase = z.object({
   source: z.enum(ScoreSource),
   authorUserId: z.string().nullish(),
   comment: z.string().nullish(),
+  metadata: jsonSchema.nullish(),
   traceId: z.string(),
   observationId: z.string().nullish(),
   configId: z.string().nullish(),
@@ -64,6 +66,7 @@ const BaseScoreBody = z.object({
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
+  metadata: jsonSchema.nullish(),
   environment: z.string().default("default"),
 });
 
@@ -155,26 +158,45 @@ export const ScorePropsAgainstConfig = z.union([
  * Transformations
  */
 
+type ValidatedAPIScore<IncludeHasMetadata extends boolean> = APIScore & {
+  hasMetadata: IncludeHasMetadata extends true ? boolean : never;
+};
+
+type InputScore = ScoreDomain & { hasMetadata?: boolean };
+
 /**
  * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
  * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
  * @param scores
  * @returns list of validated scores
  */
-export const filterAndValidateDbScoreList = (
-  scores: Score[],
-  onParseError?: (error: z.ZodError) => void,
-): APIScore[] =>
-  scores.reduce((acc, ts) => {
+export const filterAndValidateDbScoreList = <
+  IncludeHasMetadata extends boolean,
+>({
+  scores,
+  includeHasMetadata = false as IncludeHasMetadata,
+  onParseError,
+}: {
+  scores: InputScore[];
+  includeHasMetadata?: IncludeHasMetadata;
+  // eslint-disable-next-line no-unused-vars
+  onParseError?: (error: z.ZodError) => void;
+}): ValidatedAPIScore<IncludeHasMetadata>[] => {
+  return scores.reduce((acc, ts) => {
     const result = APIScoreSchema.safeParse(ts);
     if (result.success) {
-      acc.push(result.data);
+      const score = { ...result.data };
+      if (includeHasMetadata) {
+        Object.assign(score, { hasMetadata: ts.hasMetadata ?? false });
+      }
+      acc.push(score as ValidatedAPIScore<IncludeHasMetadata>);
     } else {
       console.error("Score parsing error: ", result.error);
       onParseError?.(result.error);
     }
     return acc;
-  }, [] as APIScore[]);
+  }, [] as ValidatedAPIScore<IncludeHasMetadata>[]);
+};
 
 /**
  * Use this function when pulling a single score from the database before using in the application to ensure type safety.
@@ -183,7 +205,7 @@ export const filterAndValidateDbScoreList = (
  * @returns validated score
  * @throws error if score fails validation
  */
-export const validateDbScore = (score: Score): APIScore =>
+export const validateDbScore = (score: ScoreDomain): APIScore =>
   APIScoreSchema.parse(score);
 
 /**
@@ -272,6 +294,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
+  // eslint-disable-next-line no-unused-vars
   onParseError?: (error: z.ZodError) => void,
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
@@ -280,7 +303,7 @@ export const legacyFilterAndValidateV1GetScoreList = (
       if (result.success) {
         acc.push(result.data);
       } else {
-        console.error("Score parsing error: ", result.error);
+        console.error(`Score parsing error ${JSON.stringify(result.error)}`);
         onParseError?.(result.error);
       }
       return acc;

packages/shared/src/features/scores/types.ts

@@ -1,18 +1,22 @@
 import { ScoreDataType } from "@prisma/client";
-import { ScoreSourceType } from "../../server";
+import { ScoreSourceType, type MetadataDomain } from "../../domain";
 
-export type CategoricalAggregate = {
+export type BaseAggregate = {
+  comment?: string | null;
+  id?: string | null;
+  hasMetadata?: boolean | null;
+};
+
+export type CategoricalAggregate = BaseAggregate & {
   type: "CATEGORICAL";
   values: string[];
   valueCounts: { value: string; count: number }[];
-  comment?: string | null;
 };
 
-export type NumericAggregate = {
+export type NumericAggregate = BaseAggregate & {
   type: "NUMERIC";
   values: number[];
   average: number;
-  comment?: string | null;
 };
 
 export type ScoreAggregate = Record<
@@ -21,11 +25,13 @@ export type ScoreAggregate = Record<
 >;
 
 export type ScoreSimplified = {
+  id: string;
   name: string;
   dataType: ScoreDataType;
   source: ScoreSourceType;
   value?: number | null;
   comment?: string | null;
+  metadata?: MetadataDomain | null;
   stringValue?: string | null;
 };
 

packages/shared/src/index.ts

@@ -46,7 +46,13 @@ export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
 export * from "./server/repositories/types";
 
+// metadata conversion
+export * from "./server/utils/metadata_conversion";
+
 // errors
 export * from "./errors/index";
 
 export * from "./utils/environment";
+
+// domain
+export * from "./domain";

packages/shared/src/observationsTable.ts

@@ -1,8 +1,8 @@
+import { ObservationLevelType } from "./domain";
 import {
   type OptionsDefinition,
   type ColumnDefinition,
 } from "./tableDefinitions";
-import { ObservationLevelType } from "./server";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -154,8 +154,8 @@ export const observationsTableCols: ColumnDefinition[] = [
     nullable: true,
   },
   {
-    name: "Usage",
-    id: "usage",
+    name: "Tokens",
+    id: "tokens",
     type: "number",
     internal: 'o."total_tokens"',
   },

packages/shared/src/server/auth/apiKeys.ts

@@ -1,4 +1,4 @@
-import { PrismaClient } from "@prisma/client";
+import { PrismaClient, ApiKeyScope } from "@prisma/client";
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
@@ -38,7 +38,8 @@ export function createShaHash(privateKey: string, salt: string): string {
 
 export async function createAndAddApiKeysToDb(p: {
   prisma: PrismaClient;
-  projectId: string;
+  entityId: string;
+  scope: ApiKeyScope;
   note?: string;
   predefinedKeys?: {
     secretKey: string;
@@ -59,14 +60,18 @@ export async function createAndAddApiKeysToDb(p: {
 
   const hashFromProvidedKey = createShaHash(sk, salt);
 
+  const entity =
+    p.scope === "PROJECT" ? { projectId: p.entityId } : { orgId: p.entityId };
+
   const apiKey = await p.prisma.apiKey.create({
     data: {
-      projectId: p.projectId,
+      ...entity,
       publicKey: pk,
       hashedSecretKey: hashedSk,
       displaySecretKey: displaySk,
       fastHashedSecretKey: hashFromProvidedKey,
       note: p.note,
+      scope: p.scope,
     },
   });
 

packages/shared/src/server/auth/types.ts

@@ -1,8 +1,9 @@
 import z from "zod";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
+import { ApiKeyScope } from "../../";
 
-export const OrgEnrichedApiKey = z.object({
+const ApiKeyBaseSchema = z.object({
   id: z.string(),
   note: z.string().nullable(),
   publicKey: z.string(),
@@ -10,7 +11,6 @@ export const OrgEnrichedApiKey = z.object({
   createdAt: z.string().datetime().nullable(),
   lastUsedAt: z.string().datetime().nullable(),
   expiresAt: z.string().datetime().nullable(),
-  projectId: z.string(),
   fastHashedSecretKey: z.string(),
   hashedSecretKey: z.string(),
   orgId: z.string(),
@@ -18,6 +18,17 @@ export const OrgEnrichedApiKey = z.object({
   rateLimitOverrides: CloudConfigRateLimit.nullish(),
 });
 
+export const OrgEnrichedApiKey = z.discriminatedUnion("scope", [
+  ApiKeyBaseSchema.extend({
+    scope: z.literal(ApiKeyScope.ORGANIZATION),
+    projectId: z.null(),
+  }),
+  ApiKeyBaseSchema.extend({
+    scope: z.literal(ApiKeyScope.PROJECT),
+    projectId: z.string(),
+  }),
+]);
+
 export const API_KEY_NON_EXISTENT = "api-key-non-existent";
 
 export const CachedApiKey = z.union([
@@ -38,8 +49,8 @@ export type AuthHeaderValidVerificationResult = {
 };
 
 export type ApiAccessScope = {
-  projectId: string;
-  accessLevel: "all" | "scores";
+  projectId: string | null;
+  accessLevel: "organization" | "project" | "scores";
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;

packages/shared/src/server/clickhouse/client.ts

@@ -18,6 +18,15 @@ export const clickhouseClient = (
     propagation.inject(context.active(), headers);
   }
 
+  const cloudOptions: Record<string, unknown> = {};
+  if (
+    ["STAGING", "EU", "US"].includes(
+      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ?? "",
+    )
+  ) {
+    cloudOptions.input_format_json_throw_on_bad_escape_sequence = 0;
+  }
+
   return createClient({
     ...params.opts,
     url: env.CLICKHOUSE_URL,
@@ -26,10 +35,11 @@ export const clickhouseClient = (
     database: env.CLICKHOUSE_DB,
     http_headers: headers,
     clickhouse_settings: {
+      ...cloudOptions,
+      ...(params.opts?.clickhouse_settings ?? {}),
       log_comment: JSON.stringify(params.tags ?? {}),
       async_insert: 1,
       wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
-      input_format_json_throw_on_bad_escape_sequence: 0,
     },
   });
 };

packages/shared/src/server/index.ts

@@ -52,6 +52,7 @@ export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
 export * from "./services/datasets-ui-table-service";
+export * from "./services/DashboardService";
 
 // test utils
 export * from "./test-utils";

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -98,13 +98,20 @@ export const processEventBatch = async (
   recordDistribution("langfuse.ingestion.event_distribution", input.length);
 
   currentSpan?.setAttribute("langfuse.ingestion.batch_size", input.length);
-  currentSpan?.setAttribute("langfuse.project.id", authCheck.scope.projectId);
+  currentSpan?.setAttribute(
+    "langfuse.project.id",
+    authCheck.scope.projectId ?? "",
+  );
   currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
   currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
    **************/
+  if (!authCheck.scope.projectId) {
+    throw new UnauthorizedError("Missing project ID");
+  }
+
   const validationErrors: { id: string; error: unknown }[] = [];
   const authenticationErrors: { id: string; error: unknown }[] = [];
 
@@ -245,7 +252,13 @@ export const processEventBatch = async (
                   eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
                   fileKey: sortedBatchByEventBodyId[id].key,
                 },
-                authCheck,
+                authCheck: authCheck as {
+                  validKey: true;
+                  scope: {
+                    projectId: string;
+                    accessLevel: "project" | "scores";
+                  };
+                },
               },
             },
             { delay: getDelay(delay) },
@@ -272,11 +285,11 @@ const isAuthorized = (
   if (event.type === eventTypes.SCORE_CREATE) {
     return (
       authScope.scope.accessLevel === "scores" ||
-      authScope.scope.accessLevel === "all"
+      authScope.scope.accessLevel === "project"
     );
   }
 
-  return authScope.scope.accessLevel === "all";
+  return authScope.scope.accessLevel === "project";
 };
 
 /**

packages/shared/src/server/ingestion/types.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 
 import { NonEmptyString, jsonSchema } from "../../utils/zod";
 import { ModelUsageUnit } from "../../constants";
-import { type ScoreSourceType } from "../repositories";
+import { ScoreSourceType } from "../../domain";
 
 export const idSchema = z
   .string()
@@ -320,6 +320,7 @@ const BaseScoreBody = z.object({
   environment: EnvironmentName,
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
+  metadata: jsonSchema.nullish(),
   source: z
     .enum(["API", "EVAL", "ANNOTATION"])
     .default("API" as ScoreSourceType),

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -1,11 +1,11 @@
 import {
   ScoreBodyWithoutConfig,
+  ScoreDomain,
   ScorePropsAgainstConfig,
   validateDbScoreConfigSafe,
   ValidatedScoreConfig,
 } from "../../../src";
 import { prisma, ScoreDataType } from "../../db";
-import { Score } from "../repositories";
 
 import { InvalidRequestError, LangfuseNotFoundError } from "../../errors";
 
@@ -17,7 +17,7 @@ type ValidateAndInflateScoreParams = {
 
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
-): Promise<Score> {
+): Promise<ScoreDomain> {
   const { body, projectId, scoreId } = params;
 
   if (body.configId) {
@@ -85,7 +85,7 @@ function mapStringValueToNumericValue(
 
 function inflateScoreBody(
   params: ValidateAndInflateScoreParams & { config?: ValidatedScoreConfig },
-): Score {
+): ScoreDomain {
   const { body, projectId, scoreId, config } = params;
 
   const relevantDataType = config?.dataType ?? body.dataType;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,4 +1,4 @@
-import type { ZodSchema } from "zod";
+import { type ZodSchema } from "zod";
 
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatVertexAI } from "@langchain/google-vertexai";
@@ -9,6 +9,7 @@ import {
   BaseMessage,
   HumanMessage,
   SystemMessage,
+  ToolMessage,
 } from "@langchain/core/messages";
 import {
   BytesOutputParser,
@@ -25,8 +26,13 @@ import { logger } from "../logger";
 import {
   ChatMessage,
   ChatMessageRole,
+  ChatMessageType,
   LLMAdapter,
+  LLMJSONSchema,
+  LLMToolDefinition,
   ModelParams,
+  ToolCallResponse,
+  ToolCallResponseSchema,
   TraceParams,
 } from "./types";
 import { CallbackHandler } from "langfuse-langchain";
@@ -37,7 +43,7 @@ type ProcessTracedEvents = () => Promise<void>;
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
-  structuredOutputSchema?: ZodSchema;
+  structuredOutputSchema?: ZodSchema | LLMJSONSchema;
   callbacks?: BaseCallbackHandler[];
   baseURL?: string;
   apiKey: string;
@@ -50,6 +56,7 @@ type LLMCompletionParams = {
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
   streaming: boolean;
+  tools?: LLMToolDefinition[];
 };
 
 export async function fetchLLMCompletion(
@@ -65,7 +72,10 @@ export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
   },
-): Promise<{ completion: string; processTracedEvents: ProcessTracedEvents }>;
+): Promise<{
+  completion: string;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
@@ -73,19 +83,34 @@ export async function fetchLLMCompletion(
     structuredOutputSchema: ZodSchema;
   },
 ): Promise<{
-  completion: unknown;
+  completion: Record<string, unknown>;
+  processTracedEvents: ProcessTracedEvents;
+}>;
+
+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    tools: LLMToolDefinition[];
+    streaming: false;
+  },
+): Promise<{
+  completion: ToolCallResponse;
   processTracedEvents: ProcessTracedEvents;
 }>;
 
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams,
 ): Promise<{
-  completion: string | IterableReadableStream<Uint8Array> | unknown;
+  completion:
+    | string
+    | IterableReadableStream<Uint8Array>
+    | Record<string, unknown>
+    | ToolCallResponse;
   processTracedEvents: ProcessTracedEvents;
 }> {
   // the apiKey must never be printed to the console
   const {
     messages,
+    tools,
     modelParams,
     streaming,
     callbacks,
@@ -140,11 +165,25 @@ export async function fetchLLMCompletion(
       )
         return new SystemMessage(message.content);
 
-      return new AIMessage(message.content);
+      if (message.type === ChatMessageType.ToolResult)
+        return new ToolMessage({
+          content: message.content,
+          tool_call_id: message.toolCallId,
+        });
+
+      return new AIMessage({
+        content: message.content,
+        tool_calls:
+          message.type === ChatMessageType.AssistantToolCall
+            ? (message.toolCalls as any)
+            : undefined,
+      });
     });
   }
 
-  finalMessages = finalMessages.filter((m) => m.content.length > 0);
+  finalMessages = finalMessages.filter(
+    (m) => m.content.length > 0 || "tool_calls" in m,
+  );
 
   let chatModel:
     | ChatOpenAI
@@ -256,7 +295,9 @@ export async function fetchLLMCompletion(
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;
-    throw new Error("This model provider is not supported.");
+    throw new Error(
+      `This model provider is not supported: ${_exhaustiveCheck}`,
+    );
   }
 
   const runConfig = {
@@ -320,6 +361,25 @@ export async function fetchLLMCompletion(
       };
     }
 
+    if (tools && tools.length > 0) {
+      const langchainTools = tools.map((tool) => ({
+        type: "function",
+        function: tool,
+      }));
+
+      const result = await chatModel
+        .bindTools(langchainTools)
+        .invoke(finalMessages, runConfig);
+
+      const parsed = ToolCallResponseSchema.safeParse(result);
+      if (!parsed.success) throw Error("Failed to parse LLM tool call result");
+
+      return {
+        completion: parsed.data,
+        processTracedEvents,
+      };
+    }
+
     if (streaming) {
       return {
         completion: await chatModel
@@ -339,6 +399,7 @@ export async function fetchLLMCompletion(
     if (throwOnError) {
       throw error;
     }
-    return { completion: null, processTracedEvents };
+
+    return { completion: "", processTracedEvents };
   }
 }

packages/shared/src/server/llm/types.ts

@@ -7,15 +7,193 @@ import { AuthHeaderValidVerificationResult } from "../auth/types";
 /* eslint-disable no-unused-vars */
 // disable lint as this is exported and used in web/worker
 
-export type PromptVariable = { name: string; value: string; isUsed: boolean };
+export const LLMJSONSchema = z.record(z.string(), z.unknown());
+export type LLMJSONSchema = z.infer<typeof LLMJSONSchema>;
 
-export type ChatMessage = {
-  role: ChatMessageRole | string; // Users may ingest any string as role via API/SDK
-  content: string;
-};
+export const JSONSchemaFormSchema = z
+  .string()
+  .transform((value, ctx) => {
+    try {
+      const parsed = JSON.parse(value);
+      return parsed;
+    } catch {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "Parameters must be valid JSON",
+      });
+      return z.NEVER;
+    }
+  })
+  .pipe(
+    z
+      .object({
+        type: z.literal("object"),
+        properties: z.record(z.any()),
+        required: z.array(z.string()).optional(),
+        additionalProperties: z.boolean().optional(),
+      })
+      .passthrough()
+      .transform((data) => JSON.stringify(data, null, 2)),
+  );
 
+export const LLMToolDefinitionSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  parameters: LLMJSONSchema,
+});
+export type LLMToolDefinition = z.infer<typeof LLMToolDefinitionSchema>;
+
+const AnthropicMessageContentWithToolUse = z.union([
+  z.object({
+    type: z.literal("text"),
+    text: z.string(),
+  }),
+  z.object({
+    type: z.literal("tool_use"),
+    id: z.string(),
+    name: z.string(),
+    input: z.unknown(),
+  }),
+]);
+
+export const LLMToolCallSchema = z.object({
+  name: z.string(),
+  id: z.string(),
+  args: z.record(z.string(), z.unknown()),
+});
+export type LLMToolCall = z.infer<typeof LLMToolCallSchema>;
+
+export const OpenAIToolCallSchema = z.object({
+  id: z.string(),
+  function: z.object({
+    name: z.string(),
+    arguments: z.union([
+      z.record(z.string(), z.unknown()),
+      z
+        .string()
+        .transform((v) => JSON.parse(v))
+        .pipe(z.record(z.string(), z.unknown())),
+    ]),
+  }),
+  type: z.literal("function"),
+});
+export type OpenAIToolCallSchema = z.infer<typeof OpenAIToolCallSchema>;
+
+export const OpenAIToolSchema = z.object({
+  type: z.literal("function"),
+  function: LLMToolDefinitionSchema,
+});
+export type OpenAIToolSchema = z.infer<typeof OpenAIToolSchema>;
+
+export const OpenAIResponseFormatSchema = z.object({
+  type: z.literal("json_schema"),
+  json_schema: z.object({
+    name: z.string(),
+    schema: LLMJSONSchema,
+  }),
+});
+
+export const ToolCallResponseSchema = z.object({
+  content: z.union([z.string(), z.array(AnthropicMessageContentWithToolUse)]),
+  tool_calls: z.array(LLMToolCallSchema),
+});
+export type ToolCallResponse = z.infer<typeof ToolCallResponseSchema>;
+export enum ChatMessageRole {
+  System = "system",
+  Developer = "developer",
+  User = "user",
+  Assistant = "assistant",
+  Tool = "tool",
+}
+
+export enum ChatMessageType {
+  System = "system",
+  Developer = "developer",
+  User = "user",
+  AssistantText = "assistant-text",
+  AssistantToolCall = "assistant-tool-call",
+  ToolResult = "tool-result",
+  PublicAPICreated = "public-api-created",
+}
+
+export const SystemMessageSchema = z.object({
+  type: z.literal(ChatMessageType.System),
+  role: z.literal(ChatMessageRole.System),
+  content: z.string(),
+});
+export type SystemMessage = z.infer<typeof SystemMessageSchema>;
+
+export const DeveloperMessageSchema = z.object({
+  type: z.literal(ChatMessageType.Developer),
+  role: z.literal(ChatMessageRole.Developer),
+  content: z.string(),
+});
+export type DeveloperMessage = z.infer<typeof DeveloperMessageSchema>;
+
+export const UserMessageSchema = z.object({
+  type: z.literal(ChatMessageType.User),
+  role: z.literal(ChatMessageRole.User),
+  content: z.string(),
+});
+export type UserMessage = z.infer<typeof UserMessageSchema>;
+
+export const AssistantTextMessageSchema = z.object({
+  type: z.literal(ChatMessageType.AssistantText),
+  role: z.literal(ChatMessageRole.Assistant),
+  content: z.string(),
+});
+export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
+
+export const AssistantToolCallMessageSchema = z.object({
+  type: z.literal(ChatMessageType.AssistantToolCall),
+  role: z.literal(ChatMessageRole.Assistant),
+  content: z.string(),
+  toolCalls: z.array(LLMToolCallSchema),
+});
+export type AssistantToolCallMessage = z.infer<
+  typeof AssistantToolCallMessageSchema
+>;
+
+export const ToolResultMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ToolResult),
+  role: z.literal(ChatMessageRole.Tool),
+  content: z.string(),
+  toolCallId: z.string(),
+});
+export type ToolResultMessage = z.infer<typeof ToolResultMessageSchema>;
+
+export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
+export const ChatMessageSchema = z.union([
+  SystemMessageSchema,
+  DeveloperMessageSchema,
+  UserMessageSchema,
+  AssistantTextMessageSchema,
+  AssistantToolCallMessageSchema,
+  ToolResultMessageSchema,
+  z
+    .object({
+      role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+      content: z.string(),
+    })
+    .transform((msg) => {
+      return {
+        ...msg,
+        type: ChatMessageType.PublicAPICreated as const,
+      };
+    }),
+]);
+
+export type ChatMessage = z.infer<typeof ChatMessageSchema>;
 export type ChatMessageWithId = ChatMessage & { id: string };
 
+export const PromptChatMessageSchema = z.object({
+  role: z.string(),
+  content: z.string(),
+});
+export const PromptChatMessageListSchema = z.array(PromptChatMessageSchema);
+
+export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
@@ -26,30 +204,15 @@ export enum LLMAdapter {
   GoogleAIStudio = "google-ai-studio",
 }
 
-export enum ChatMessageRole {
-  System = "system",
-  User = "user",
-  Assistant = "assistant",
-  Developer = "developer",
-}
-
 export const SYSTEM_ROLES: string[] = [
   ChatMessageRole.System,
   ChatMessageRole.Developer,
 ];
 
-export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
-
-const ChatMessageSchema = z.object({
-  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
-  content: z.string(),
-});
-
-export const ChatMessageListSchema = z.array(ChatMessageSchema);
 export const TextPromptSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
-  ChatMessageListSchema,
+  PromptChatMessageListSchema,
   TextPromptSchema,
 ]);
 export type PromptContent = z.infer<typeof PromptContentSchema>;
@@ -90,7 +253,16 @@ export const ExperimentMetadataSchema = z
 export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
 
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
+// WARNING: The first entry in the array is chosen as the default model to add LLM API keys. Make sure it supports top_p, max_tokens and temperature.
 export const openAIModels = [
+  "gpt-4.1",
+  "gpt-4.1-2025-04-14",
+  "gpt-4.1-mini-2025-04-14",
+  "gpt-4.1-nano-2025-04-14",
+  "o3",
+  "o3-2025-04-16",
+  "o4-mini",
+  "o4-mini-2025-04-16",
   "gpt-4o",
   "gpt-4o-2024-08-06",
   "gpt-4o-2024-05-13",
@@ -121,6 +293,7 @@ export const openAIModels = [
 export type OpenAIModel = (typeof openAIModels)[number];
 
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
+// WARNING: The first entry in the array is chosen as the default model to add LLM API keys. Make sure it supports top_p, max_tokens and temperature.
 export const anthropicModels = [
   "claude-3-7-sonnet-20250219",
   "claude-3-5-sonnet-20241022",
@@ -134,7 +307,9 @@ export const anthropicModels = [
   "claude-instant-1.2",
 ] as const;
 
+// WARNING: The first entry in the array is chosen as the default model to add LLM API keys. Make sure it supports top_p, max_tokens and temperature.
 export const vertexAIModels = [
+  "gemini-2.5-pro-exp-03-25",
   "gemini-2.0-pro-exp-02-05",
   "gemini-2.0-flash-001",
   "gemini-2.0-flash-lite-preview-02-05",
@@ -144,7 +319,9 @@ export const vertexAIModels = [
   "gemini-1.0-pro",
 ] as const;
 
+// WARNING: The first entry in the array is chosen as the default model to add LLM API keys. Make sure it supports top_p, max_tokens and temperature.
 export const googleAIStudioModels = [
+  "gemini-2.5-pro-exp-03-25",
   "gemini-2.0-flash",
   "gemini-2.0-flash-lite-preview-02-05",
   "gemini-2.0-flash-thinking-exp-01-21",

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -391,6 +391,10 @@ export class FilterList {
     return this.filters.some(predicate);
   }
 
+  forEach(callback: (filter: Filter) => void) {
+    this.filters.forEach(callback);
+  }
+
   length() {
     return this.filters.length;
   }

packages/shared/src/server/queries/createGenerationsQuery.ts

@@ -1,21 +1,15 @@
-import { ObservationView } from "../repositories";
+import { Observation } from "../../domain";
 
 type AdditionalObservationFields = {
   traceName: string | null;
   traceTags: Array<string>;
-  usageDetails: Record<string, number>;
-  costDetails: Record<string, number>;
+  traceTimestamp: Date | null;
 };
 
-export type FullObservation = AdditionalObservationFields & ObservationView;
+export type FullObservation = AdditionalObservationFields & Observation;
 
 export type FullObservations = Array<FullObservation>;
 
 export type FullObservationsWithScores = Array<
   FullObservation & { scores?: Record<string, string[] | number[]> | null }
 >;
-
-export type IOAndMetadataOmittedObservations = Array<
-  Omit<ObservationView, "input" | "output" | "metadata"> &
-    AdditionalObservationFields
->;

packages/shared/src/server/queries/index.ts

@@ -2,7 +2,6 @@ export { createSessionsAllQuery } from "./createSessionsAllQuery";
 export {
   type FullObservations,
   type FullObservationsWithScores,
-  type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
 export {
   FilterList,

packages/shared/src/server/queues.ts

@@ -17,7 +17,6 @@ export const IngestionEvent = z.object({
     validKey: z.literal(true),
     scope: z.object({
       projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
     }),
   }),
 });

packages/shared/src/server/redis/batchExport.ts

@@ -25,7 +25,7 @@ export class BatchExportQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 8,
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/redis/projectDelete.ts

@@ -26,7 +26,7 @@ export class ProjectDeleteQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,
-              attempts: 5,
+              attempts: 10,
               delay: 60_000, // 1 minute
               backoff: {
                 type: "exponential",

packages/shared/src/server/repositories/dashboards.ts

@@ -10,7 +10,6 @@ import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
   SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
-  TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
 
 export type DateTrunc = "year" | "month" | "week" | "day" | "hour" | "minute";
@@ -35,103 +34,6 @@ const convertEnvFilterToClickhouseFilter = (filter: FilterState) => {
   ]);
 };
 
-export const getTotalTraces = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  ).apply();
-
-  const query = `
-    SELECT 
-      count(id) as count 
-    FROM traces t FINAL 
-    WHERE project_id = {projectId: String}
-    AND ${chFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-  `;
-
-  const result = await queryClickhouse<{ count: number }>({
-    query,
-    params: {
-      projectId,
-      ...chFilter.params,
-      ...environmentFilter.params,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "totalTraces",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  if (result.length === 0) {
-    return undefined;
-  }
-
-  return [{ countTraceId: result[0].count }];
-};
-
-export const getObservationsCostGroupedByName = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-
-  const query = `
-    SELECT 
-      provided_model_name as name,
-      sumMap(cost_details)['total'] as sum_cost_details,
-      sumMap(usage_details)['total'] as sum_usage_details
-    FROM observations o FINAL ${hasTraceFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-    WHERE project_id = {projectId: String}
-    ${appliedFilter.query ? `AND ${appliedFilter.query}` : ""}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    GROUP BY provided_model_name
-    ORDER BY sumMap(cost_details)['total'] DESC
-    LIMIT 50
-    `;
-
-  const result = await queryClickhouse<{
-    name: string;
-    sum_cost_details: number;
-    sum_usage_details: number;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "observationCostGroupedByName",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result;
-};
-
 export const getScoreAggregate = async (
   projectId: string,
   filter: FilterState,
@@ -199,138 +101,6 @@ export const getScoreAggregate = async (
   return result;
 };
 
-export const groupTracesByTime = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  ).apply();
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "timestamp",
-  );
-
-  const query = `
-    SELECT 
-      ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
-      count(*) as count
-    FROM traces t FINAL
-    WHERE project_id = {projectId: String}
-    AND ${chFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    GROUP BY timestamp
-    ${orderByQuery}
-    `;
-  const result = await queryClickhouse<{
-    timestamp: string;
-    count: string;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...chFilter.params,
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "tracesByTime",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
-    countTraceId: Number(row.count),
-  }));
-};
-
-export const getTotalObservationUsageByTimeByModel = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-  const timeFilter = tracesFilter
-    ? (chFilter.find(
-        (f) =>
-          f.clickhouseTable === "observations" &&
-          f.field.includes("start_time") &&
-          (f.operator === ">=" || f.operator === ">"),
-      ) as DateTimeFilter | undefined)
-    : undefined;
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "start_time",
-  );
-
-  const query = `
-    SELECT 
-      ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
-      sumMap(usage_details)['total'] as units,
-      sumMap(cost_details)['total'] as cost, 
-      provided_model_name
-    FROM observations o FINAL
-    ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-    WHERE project_id = {projectId: String}
-    AND ${appliedFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
-    GROUP BY start_time, provided_model_name
-    ${orderByQuery}
-    `;
-
-  const result = await queryClickhouse<{
-    start_time: string;
-    units: string;
-    cost: string;
-    provided_model_name: string;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...orderByParams,
-      ...(timeFilter
-        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
-        : {}),
-    },
-    tags: {
-      feature: "dashboard",
-      type: "observationUsageByTime",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    startTime: parseClickhouseUTCDateTimeFormat(row.start_time),
-    units: Number(row.units),
-    cost: Number(row.cost),
-    model: row.provided_model_name,
-  }));
-};
-
 export const getObservationCostByTypeByTime = async (
   projectId: string,
   filter: FilterState,
@@ -527,604 +297,6 @@ export const getObservationUsageByTypeByTime = async (
   });
 };
 
-export const getDistinctModels = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-  const timeFilter = tracesFilter
-    ? (chFilter.find(
-        (f) =>
-          f.clickhouseTable === "observations" &&
-          f.field.includes("start_time") &&
-          (f.operator === ">=" || f.operator === ">"),
-      ) as DateTimeFilter | undefined)
-    : undefined;
-
-  // No need for final as duplicates are caught by distinct anyway.
-  const query = `
-    SELECT distinct(provided_model_name) as model, count(*) as count
-    FROM observations o
-    ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-    WHERE project_id = {projectId: String}
-    AND ${appliedFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
-    GROUP BY provided_model_name
-    ORDER BY count(*) DESC
-    LIMIT 1000
-    `;
-
-  const result = await queryClickhouse<{ model: string }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...(timeFilter
-        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
-        : {}),
-    },
-    tags: {
-      feature: "dashboard",
-      type: "distinctModels",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result;
-};
-
-export const getScoresAggregateOverTime = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-  // TODO: Validate whether we can filter traces on timestamp here.
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "timestamp",
-  );
-
-  const query = `
-  SELECT 
-    ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
-    name,
-    data_type,
-    source,
-    AVG(value) as avg_value
-  FROM scores FINAL
-  ${traceFilter ? "JOIN traces t ON scores.trace_id = t.id AND scores.project_id = t.project_id" : ""}
-  WHERE project_id = {projectId: String}
-  ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-  AND ${appliedFilter.query}
-  AND data_type IN ('NUMERIC', 'BOOLEAN')
-  GROUP BY 
-    timestamp,
-    name,
-    data_type,
-    source
-  ${orderByQuery};
-`;
-
-  const result = await queryClickhouse<{
-    timestamp: string;
-    name: string;
-    data_type: string;
-    source: string;
-    avg_value: number;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "scoresAggregateOverTime",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
-    scoreName: row.name,
-    scoreDataType: row.data_type,
-    scoreSource: row.source,
-    avgValue: Number(row.avg_value),
-  }));
-};
-
-export const getModelUsageByUser = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const timeFilter = chFilter.find(
-    (f) =>
-      f.clickhouseTable === "observations" &&
-      f.field.includes("start_time") &&
-      (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  const query = `
-    SELECT 
-      sumMap(usage_details)['total'] as sum_usage_details,
-      sumMap(cost_details)['total'] as sum_cost_details,
-      user_id
-    FROM observations o FINAL
-    JOIN traces t FINAL
-    ON o.trace_id = t.id AND o.project_id = t.project_id
-    WHERE project_id = {projectId: String}
-    AND t.user_id IS NOT NULL
-    AND ${appliedFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
-    GROUP BY user_id
-    ORDER BY sum_cost_details DESC
-    `;
-
-  const result = await queryClickhouse<{
-    sum_usage_details: string;
-    sum_cost_details: number;
-    user_id: string;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...(timeFilter
-        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
-        : {}),
-    },
-    tags: {
-      feature: "dashboard",
-      type: "modelUsageByUser",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    sumUsageDetails: Number(row.sum_usage_details),
-    sumCostDetails: Number(row.sum_cost_details),
-    userId: row.user_id,
-  }));
-};
-
-export const getObservationLatencies = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  // Skipping FINAL here, as the quantiles are approximate to begin with.
-  const query = `
-    SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles,
-      name
-    FROM observations o
-    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-    WHERE project_id = {projectId: String}
-    AND ${appliedFilter.query}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    GROUP BY name
-    ORDER BY quantiles[2] DESC
-    `;
-
-  const result = await queryClickhouse<{ quantiles: string[]; name: string }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "observationLatencies",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    p50: Number(row.quantiles[0]) / 1000,
-    p90: Number(row.quantiles[1]) / 1000,
-    p95: Number(row.quantiles[2]) / 1000,
-    p99: Number(row.quantiles[3]) / 1000,
-    name: row.name,
-  }));
-};
-
-export const getTracesLatencies = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const timestampFilter = chFilter.find(
-    (f) =>
-      f.clickhouseTable === "traces" &&
-      f.field === 't."timestamp"' &&
-      (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  // Skipping FINAL here, as the quantiles are approximate to begin with.
-  const query = `
-    WITH trace_latencies as (
-      select o.trace_id,
-             t.name,
-             o.project_id,
-             date_diff('millisecond', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
-      FROM traces t 
-      JOIN observations o
-      ON o.trace_id = t.id AND o.project_id = t.project_id
-      WHERE project_id = {projectId: String}
-      AND ${appliedFilter.query}
-      ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-      ${timestampFilter ? `AND o.start_time > {dateTimeFilterObservations: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-      GROUP BY o.project_id, o.trace_id, t.name
-    )
-
-    SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(duration) as quantiles,
-      name
-    FROM trace_latencies
-    GROUP BY name
-    ORDER BY quantiles[2] DESC
-  `;
-
-  const result = await queryClickhouse<{ quantiles: string[]; name: string }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...(timestampFilter
-        ? { dateTimeFilterObservations: timestampFilter.value }
-        : {}),
-    },
-    tags: {
-      feature: "dashboard",
-      type: "tracesLatencies",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    p50: Number(row.quantiles[0]) / 1000,
-    p90: Number(row.quantiles[1]) / 1000,
-    p95: Number(row.quantiles[2]) / 1000,
-    p99: Number(row.quantiles[3]) / 1000,
-    name: row.name,
-  }));
-};
-
-export const getModelLatenciesOverTime = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-
-  const appliedFilter = chFilter.apply();
-
-  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "start_time_bucket",
-  );
-
-  // Skipping FINAL here, as the quantiles are approximate to begin with.
-  const query = `
-  SELECT 
-    ${selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket")},
-    provided_model_name,
-    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles
-  FROM observations o
-  ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-  WHERE project_id = {projectId: String}
-  ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-  AND ${appliedFilter.query}
-  GROUP BY provided_model_name, start_time_bucket
-  ${orderByQuery};
-`;
-
-  const result = await queryClickhouse<{
-    start_time_bucket: string;
-    provided_model_name: string;
-    quantiles: string[];
-  }>({
-    query,
-    params: {
-      projectId,
-      ...appliedFilter.params,
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "modelLatenciesOverTime",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    p50: Number(row.quantiles[0]) / 1000,
-    p75: Number(row.quantiles[1]) / 1000,
-    p90: Number(row.quantiles[2]) / 1000,
-    p95: Number(row.quantiles[3]) / 1000,
-    p99: Number(row.quantiles[4]) / 1000,
-    model: row.provided_model_name,
-    start_time: parseClickhouseUTCDateTimeFormat(row.start_time_bucket),
-  }));
-};
-
-export const getNumericScoreTimeSeries = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-  const chFilterRes = chFilter.apply();
-
-  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "score_timestamp",
-  );
-
-  const query = `
-    SELECT
-    ${selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp")},
-    s.name as score_name,
-    AVG(s.value) as avg_value
-    FROM scores s final
-    ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
-    WHERE s.project_id = {projectId: String}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
-    GROUP BY score_name, score_timestamp
-    ${orderByQuery}
-  `;
-
-  const result = await queryClickhouse<{
-    score_timestamp: string;
-    score_name: string;
-    avg_value: number;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...(chFilterRes ? chFilterRes.params : {}),
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "numericScoreTimeSeries",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.score_timestamp),
-    scoreName: row.score_name,
-    avgValue: Number(row.avg_value),
-  }));
-};
-
-export const getCategoricalScoreTimeSeries = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-  const chFilterRes = chFilter.apply();
-
-  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "score_timestamp",
-  );
-
-  const query = `
-    SELECT
-    ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp") + ", " : ""}
-    s.name as score_name,
-    s.data_type as score_data_type,
-    s.source as score_source,
-    s.string_value as score_value,
-    count(s.string_value) as count
-    FROM scores s final
-    ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
-    WHERE s.project_id = {projectId: String}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
-    GROUP BY score_name, score_data_type, score_source, score_value ${bucketSizeInSeconds ? ", score_timestamp" : ""}
-      ${orderByQuery}
-  `;
-
-  const result = await queryClickhouse<{
-    score_timestamp?: string;
-    score_name: string;
-    score_data_type: string;
-    score_source: string;
-    score_value: string;
-    count: number;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...(chFilterRes ? chFilterRes.params : {}),
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "categoricalScoreTimeSeries",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    scoreTimestamp: row.score_timestamp
-      ? parseClickhouseUTCDateTimeFormat(row.score_timestamp)
-      : undefined,
-    scoreName: row.score_name,
-    scoreDataType: row.score_data_type,
-    scoreSource: row.score_source,
-    scoreValue: row.score_value,
-    count: Number(row.count),
-  }));
-};
-
-export const getObservationsStatusTimeSeries = async (
-  projectId: string,
-  filter: FilterState,
-) => {
-  const { envFilter, remainingFilters } =
-    extractEnvironmentFilterFromFilters(filter);
-  const environmentFilter = new FilterList(
-    convertEnvFilterToClickhouseFilter(envFilter),
-  ).apply();
-  const chFilter = new FilterList(
-    createFilterFromFilterState(remainingFilters, dashboardColumnDefinitions),
-  );
-  const chFilterRes = chFilter.apply();
-
-  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-
-  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
-    filter,
-    "start_time_bucket",
-  );
-
-  const query = `
-    SELECT 
-      ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket") + ", " : ""}
-      count(*) as observation_count,
-      level as level
-    FROM observations o
-    ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
-    WHERE project_id = {projectId: String}
-    ${environmentFilter.query ? `AND ${environmentFilter.query}` : ""}
-    AND o.level IS NOT NULL
-    AND ${chFilterRes?.query}
-    GROUP BY level ${bucketSizeInSeconds ? ", start_time_bucket" : ""}
-    ${orderByQuery}
-  `;
-
-  const result = await queryClickhouse<{
-    start_time_bucket?: string;
-    observation_count: string;
-    level: string;
-  }>({
-    query,
-    params: {
-      projectId,
-      ...(chFilterRes ? chFilterRes.params : {}),
-      ...environmentFilter.params,
-      ...orderByParams,
-    },
-    tags: {
-      feature: "dashboard",
-      type: "observationStatusTimeSeries",
-      kind: "analytic",
-      projectId,
-    },
-  });
-
-  return result.map((row) => ({
-    start_time_bucket: row.start_time_bucket
-      ? parseClickhouseUTCDateTimeFormat(row.start_time_bucket)
-      : undefined,
-    count: Number(row.observation_count),
-    level: row.level,
-  }));
-};
-
 export const orderByTimeSeries = (
   filter: FilterState,
   col: string,

packages/shared/src/server/repositories/definitions.ts

@@ -132,6 +132,7 @@ export const scoreRecordBaseSchema = z.object({
   value: z.number().nullish(),
   source: z.string(),
   comment: z.string().nullish(),
+  metadata: z.record(z.string()),
   author_user_id: z.string().nullish(),
   config_id: z.string().nullish(),
   data_type: z.enum(["NUMERIC", "CATEGORICAL", "BOOLEAN"]).nullish(),
@@ -350,6 +351,7 @@ export const convertPostgresScoreToInsert = (
     value: score.value,
     source: score.source,
     comment: score.comment,
+    metadata: score.metadata ?? {},
     author_user_id: score.author_user_id,
     config_id: score.config_id,
     data_type: score.data_type,

packages/shared/src/server/repositories/observations.ts

@@ -26,17 +26,27 @@ import { TableCount } from "./types";
 import { OrderByState } from "../../interfaces/orderBy";
 import { getTracesByIds } from "./traces";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import {
-  convertObservationToView,
-  convertObservation,
-} from "./observations_converters";
+import { convertObservation } from "./observations_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
 import { env } from "../../env";
+import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 
+/**
+ * Checks if observation exists in clickhouse.
+ *
+ * @param {string} projectId - Project ID for the observation
+ * @param {string} id - ID of the observation
+ * @param {Date} startTime - Timestamp for time-based filtering, uses event payload or job timestamp
+ * @returns {Promise<boolean>} - True if observation exists
+ *
+ * Notes:
+ * • Filters with two days lookback window subject to startTime
+ * • Used for validating observation references before eval job creation
+ */
 export const checkObservationExists = async (
   projectId: string,
   id: string,
@@ -101,7 +111,7 @@ export const upsertObservation = async (
   });
 };
 
-export const getObservationsViewForTrace = async (
+export const getObservationsForTrace = async (
   traceId: string,
   projectId: string,
   timestamp?: Date,
@@ -161,7 +171,7 @@ export const getObservationsViewForTrace = async (
     },
   });
 
-  return records.map(convertObservationToView);
+  return records.map(convertObservation);
 };
 
 export const getObservationForTraceIdByName = async (
@@ -227,7 +237,7 @@ export const getObservationForTraceIdByName = async (
     },
   });
 
-  return records.map(convertObservationToView);
+  return records.map(convertObservation);
 };
 
 export const getObservationById = async (
@@ -306,33 +316,6 @@ export const getObservationsById = async (
   return records.map(convertObservation);
 };
 
-export const getObservationViewById = async (
-  id: string,
-  projectId: string,
-  fetchWithInputOutput: boolean = false,
-) => {
-  const records = await getObservationByIdInternal(
-    id,
-    projectId,
-    fetchWithInputOutput,
-  );
-  const mapped = records.map(convertObservationToView);
-
-  if (mapped.length === 0) {
-    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
-  }
-
-  if (mapped.length > 1) {
-    logger.error(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-    throw new InternalServerError(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-  }
-  return mapped.shift();
-};
-
 const getObservationByIdInternal = async (
   id: string,
   projectId: string,
@@ -402,6 +385,7 @@ export type ObservationTableQuery = {
   limit?: number;
   offset?: number;
   selectIOAndMetadata?: boolean;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
 };
 
 export type ObservationsTableQueryResult = ObservationRecordReadType & {
@@ -467,13 +451,14 @@ export const getObservationsTableWithModelData = async (
     const trace = traces.find((t) => t.id === o.trace_id);
     const model = models.find((m) => m.id === o.internal_model_id);
     return {
-      ...convertObservationToView(o),
+      ...convertObservation(o),
       latency: o.latency ? Number(o.latency) / 1000 : null,
       timeToFirstToken: o.time_to_first_token
         ? Number(o.time_to_first_token) / 1000
         : null,
       traceName: trace?.name ?? null,
       traceTags: trace?.tags ?? [],
+      traceTimestamp: trace?.timestamp ?? null,
       userId: trace?.userId ?? null,
       modelId: model?.id ?? null,
       inputPrice:
@@ -525,8 +510,15 @@ const getObservationsTableInternal = async <T>(
         if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time)) as latency,
         if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time)) as "time_to_first_token"`;
 
-  const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
-    opts;
+  const {
+    projectId,
+    filter,
+    selectIOAndMetadata,
+    limit,
+    offset,
+    orderBy,
+    clickhouseConfigs,
+  } = opts;
 
   const selectString = selectIOAndMetadata
     ? `
@@ -689,6 +681,7 @@ const getObservationsTableInternal = async <T>(
       type: "observation",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return res;
@@ -1005,18 +998,18 @@ export const deleteObservationsByProjectId = async (projectId: string) => {
 
 export const deleteObservationsOlderThanDays = async (
   projectId: string,
-  days: number,
+  beforeDate: Date,
 ) => {
   const query = `
     DELETE FROM observations
     WHERE project_id = {projectId: String}
-    AND start_time < now() - INTERVAL {numDays: Int} DAYS;
+    AND start_time < {cutoffDate: DateTime64(3)};
   `;
   await commandClickhouse({
     query: query,
     params: {
       projectId,
-      numDays: days,
+      cutoffDate: convertDateToClickhouseDateTime(beforeDate),
     },
     clickhouseConfigs: {
       request_timeout: 120_000, // 2 minutes
@@ -1428,6 +1421,13 @@ export const getGenerationsForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");

packages/shared/src/server/repositories/observations_converters.ts

@@ -1,54 +1,18 @@
-import { Prisma } from "@prisma/client";
-import Decimal from "decimal.js";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
 import { parseJsonPrioritised } from "../../utils/json";
 import {
   Observation,
-  ObservationView,
-  ObservationType,
   ObservationLevelType,
-} from "./types";
-
-export const convertObservationToView = (
-  record: ObservationRecordReadType,
-): Omit<ObservationView, "inputPrice" | "outputPrice" | "totalPrice"> & {
-  usageDetails: Record<string, number>;
-  costDetails: Record<string, number>;
-} => {
-  // these cost are not used from the view. They are in the select statement but not in the
-  // Prisma file. We will not clean this up but keep it as it is for now.
-  // eslint-disable-next-line no-unused-vars
-  const { inputCost, outputCost, totalCost, internalModelId, ...rest } =
-    convertObservation(record ?? undefined);
-  return {
-    ...rest,
-    latency: record.end_time
-      ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
-      : null,
-
-    promptName: record.prompt_name ?? null,
-    promptVersion: record.prompt_version ?? null,
-    modelId: record.internal_model_id ?? null,
-  };
-};
+  ObservationType,
+} from "../../domain";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 
 export const convertObservation = (
   record: ObservationRecordReadType,
-): Omit<Observation, "internalModel"> & {
-  promptName: string | null;
-  promptVersion: number | null;
-  latency: number | null;
-  timeToFirstToken: number | null;
-  usageDetails: Record<string, number>;
-  costDetails: Record<string, number>;
-} => {
-  const reducedUsageDetails = reduceUsageOrCostDetails(record.usage_details);
+): Observation => {
   const reducedCostDetails = reduceUsageOrCostDetails(record.cost_details);
-  const reducedProvidedCostDetails = reduceUsageOrCostDetails(
-    record.provided_cost_details,
-  );
+  const reducedUsageDetails = reduceUsageOrCostDetails(record.usage_details);
 
   return {
     id: record.id,
@@ -62,25 +26,16 @@ export const convertObservation = (
       ? parseClickhouseUTCDateTimeFormat(record.end_time)
       : null,
     name: record.name ?? null,
-    metadata:
-      record.metadata &&
-      Object.fromEntries(
-        Object.entries(record.metadata ?? {}).map(([key, val]) => [
-          key,
-          val && parseJsonPrioritised(val),
-        ]),
-      ),
+    metadata: parseMetadataCHRecordToDomain(record.metadata),
     level: record.level as ObservationLevelType,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: (record.input
-      ? parseJsonPrioritised(record.input)
-      : null) as Prisma.JsonValue | null,
-    output: (record.output
-      ? parseJsonPrioritised(record.output)
-      : null) as Prisma.JsonValue | null,
+    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
+    output: record.output
+      ? (parseJsonPrioritised(record.output) ?? null)
+      : null,
     modelParameters: record.model_parameters
-      ? JSON.parse(record.model_parameters)
+      ? (JSON.parse(record.model_parameters) ?? null)
       : null,
     completionStartTime: record.completion_start_time
       ? parseClickhouseUTCDateTimeFormat(record.completion_start_time)
@@ -88,29 +43,6 @@ export const convertObservation = (
     promptId: record.prompt_id ?? null,
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
-    promptTokens: reducedUsageDetails.input ?? 0,
-    completionTokens: reducedUsageDetails.output ?? 0,
-    totalTokens: reducedUsageDetails.total ?? 0,
-    calculatedInputCost:
-      reducedCostDetails.input != null
-        ? new Decimal(reducedCostDetails.input)
-        : null,
-    calculatedOutputCost:
-      reducedCostDetails.output != null
-        ? new Decimal(reducedCostDetails.output)
-        : null,
-    calculatedTotalCost: record.cost_details?.total
-      ? new Decimal(record.cost_details.total)
-      : null,
-    inputCost:
-      reducedProvidedCostDetails.input != null
-        ? new Decimal(reducedProvidedCostDetails.input)
-        : null,
-    outputCost:
-      reducedProvidedCostDetails.output != null
-        ? new Decimal(reducedProvidedCostDetails.output)
-        : null,
-    totalCost: record.total_cost ? new Decimal(record.total_cost) : null,
     usageDetails: Object.fromEntries(
       Object.entries(record.usage_details ?? {}).map(([key, value]) => [
         key,
@@ -123,9 +55,14 @@ export const convertObservation = (
         Number(value),
       ]),
     ),
+    providedCostDetails: Object.fromEntries(
+      Object.entries(record.provided_cost_details ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
     model: record.provided_model_name ?? null,
     internalModelId: record.internal_model_id ?? null,
-    unit: "TOKENS", // to be removed.
     promptName: record.prompt_name ?? null,
     promptVersion: record.prompt_version ?? null,
     latency: record.end_time
@@ -139,6 +76,12 @@ export const convertObservation = (
           parseClickhouseUTCDateTimeFormat(record.start_time).getTime()) /
         1000
       : null,
+    inputCost: reducedCostDetails.input,
+    outputCost: reducedCostDetails.output,
+    totalCost: reducedCostDetails.total,
+    inputUsage: reducedUsageDetails.input ?? 0,
+    outputUsage: reducedUsageDetails.output ?? 0,
+    totalUsage: reducedUsageDetails.total ?? 0,
   };
 };
 
@@ -153,13 +96,13 @@ export const reduceUsageOrCostDetails = (
     input: Object.entries(details ?? {})
       .filter(([usageType]) => usageType.startsWith("input"))
       .reduce(
-        (acc, [_, value]) => (acc ?? 0) + Number(value),
+        (acc, [, value]) => (acc ?? 0) + Number(value),
         null as number | null, // default to null if no input usage is found
       ),
     output: Object.entries(details ?? {})
       .filter(([usageType]) => usageType.startsWith("output"))
       .reduce(
-        (acc, [_, value]) => (acc ?? 0) + Number(value),
+        (acc, [, value]) => (acc ?? 0) + Number(value),
         null as number | null, // default to null if no output usage is found
       ),
     total: Number(details?.total ?? 0),

packages/shared/src/server/repositories/scores.ts

@@ -1,5 +1,5 @@
 import { ScoreDataType } from "@prisma/client";
-import { Score, ScoreSourceType } from "./types";
+import { ScoreDomain, ScoreSourceType } from "../../domain/scores";
 import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
@@ -27,6 +27,8 @@ import { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./constants";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { ScoreRecordReadType } from "./definitions";
 import { env } from "../../env";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -71,7 +73,7 @@ export const searchExistingAnnotationScore = async (
       projectId,
     },
   });
-  return rows.map(convertToScore).shift();
+  return rows.map((row) => convertToScore(row)).shift();
 };
 
 export const getScoreById = async (
@@ -160,19 +162,50 @@ export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   });
 };
 
-export type GetScoresForTracesProps = {
+export type GetScoresForTracesProps<
+  ExcludeMetadata extends boolean,
+  IncludeHasMetadata extends boolean,
+> = {
   projectId: string;
   traceIds: string[];
   timestamp?: Date;
   limit?: number;
   offset?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions;
+  excludeMetadata?: ExcludeMetadata;
+  includeHasMetadata?: IncludeHasMetadata;
 };
 
-export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
-  const { projectId, traceIds, timestamp, limit, offset } = props;
+// Used in multiple places, including the public API, hence the non-default exclusion of metadata via excludeMetadata flag
+export const getScoresForTraces = async <
+  ExcludeMetadata extends boolean,
+  IncludeHasMetadata extends boolean,
+>(
+  props: GetScoresForTracesProps<ExcludeMetadata, IncludeHasMetadata>,
+) => {
+  const {
+    projectId,
+    traceIds,
+    timestamp,
+    limit,
+    offset,
+    clickhouseConfigs,
+    excludeMetadata = false,
+    includeHasMetadata = false,
+  } = props;
+
+  const select = [
+    !excludeMetadata ? "*" : "* EXCEPT (metadata)",
+    includeHasMetadata
+      ? "length(mapKeys(s.metadata)) > 0 AS has_metadata"
+      : null,
+  ]
+    .filter((s) => s != null)
+    .join(", ");
+
   const query = `
-      select 
-        *
+      select
+        ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
       AND s.trace_id IN ({traceIds: Array(String)}) 
@@ -182,7 +215,15 @@ export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<ScoreRecordReadType>({
+  const rows = await queryClickhouse<
+    ScoreRecordReadType & {
+      metadata: ExcludeMetadata extends true
+        ? never
+        : ScoreRecordReadType["metadata"];
+      // has_metadata is 0 or 1 from ClickHouse, later converted to a boolean
+      has_metadata: IncludeHasMetadata extends true ? 0 | 1 : never;
+    }
+  >({
     query: query,
     params: {
       projectId,
@@ -199,20 +240,65 @@ export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
       kind: "list",
       projectId,
     },
+    clickhouseConfigs,
   });
 
-  return rows.map(convertToScore);
+  return rows.map((row) => {
+    const score = convertToScore({
+      ...row,
+      metadata: excludeMetadata ? {} : row.metadata,
+    });
+
+    if (includeHasMetadata) {
+      Object.assign(score, { hasMetadata: !!row.has_metadata });
+    }
+
+    return score;
+  });
 };
 
-export const getScoresForObservations = async (
-  projectId: string,
-  observationIds: string[],
-  limit?: number,
-  offset?: number,
+export type GetScoresForObservationsProps<
+  ExcludeMetadata extends boolean,
+  IncludeHasMetadata extends boolean,
+> = {
+  projectId: string;
+  observationIds: string[];
+  limit?: number;
+  offset?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions;
+  excludeMetadata?: ExcludeMetadata;
+  includeHasMetadata?: IncludeHasMetadata;
+};
+
+// Currently only used from the observations table, hence the exclusion of metadata without excludeMetadata flag
+export const getScoresForObservations = async <
+  ExcludeMetadata extends boolean,
+  IncludeHasMetadata extends boolean,
+>(
+  props: GetScoresForObservationsProps<ExcludeMetadata, IncludeHasMetadata>,
 ) => {
+  const {
+    projectId,
+    observationIds,
+    limit,
+    offset,
+    clickhouseConfigs,
+    excludeMetadata = false,
+    includeHasMetadata = false,
+  } = props;
+
+  const select = [
+    !excludeMetadata ? "*" : "* EXCEPT (metadata)",
+    includeHasMetadata
+      ? "length(mapKeys(s.metadata)) > 0 AS has_metadata"
+      : null,
+  ]
+    .filter((s) => s != null)
+    .join(", ");
+
   const query = `
       select 
-        *
+        ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
       AND s.observation_id IN ({observationIds: Array(String)})
@@ -221,7 +307,15 @@ export const getScoresForObservations = async (
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<ScoreRecordReadType>({
+  const rows = await queryClickhouse<
+    ScoreRecordReadType & {
+      metadata: ExcludeMetadata extends true
+        ? never
+        : ScoreRecordReadType["metadata"];
+      // has_metadata is 0 or 1 from ClickHouse, later converted to a boolean
+      has_metadata: IncludeHasMetadata extends true ? 0 | 1 : never;
+    }
+  >({
     query: query,
     params: {
       projectId: projectId,
@@ -235,9 +329,18 @@ export const getScoresForObservations = async (
       kind: "list",
       projectId,
     },
+    clickhouseConfigs,
   });
 
-  return rows.map(convertToScore);
+  return rows.map((row) => ({
+    ...convertToScore({
+      ...row,
+      metadata: excludeMetadata ? {} : row.metadata,
+    }),
+    hasMetadata: (includeHasMetadata
+      ? !!row.has_metadata
+      : undefined) as IncludeHasMetadata extends true ? boolean : never,
+  }));
 };
 
 export const getScoresGroupedByNameSourceType = async (
@@ -347,6 +450,7 @@ export const getScoresUiCount = async (props: {
 }) => {
   const rows = await getScoresUiGeneric<{ count: string }>({
     select: "count",
+    excludeMetadata: true,
     tags: { kind: "count" },
     ...props,
   });
@@ -354,19 +458,32 @@ export const getScoresUiCount = async (props: {
   return Number(rows[0].count);
 };
 
-export type ScoreUiTableRow = Score & {
+export type ScoreUiTableRow = ScoreDomain & {
   traceName: string | null;
   traceUserId: string | null;
   traceTags: Array<string> | null;
 };
 
-export const getScoresUiTable = async (props: {
+export async function getScoresUiTable<
+  ExcludeMetadata extends boolean,
+  IncludeHasMetadata extends boolean,
+>(props: {
   projectId: string;
   filter: FilterState;
   orderBy: OrderByState;
   limit?: number;
   offset?: number;
-}): Promise<ScoreUiTableRow[]> => {
+  clickhouseConfigs?: ClickHouseClientConfigOptions;
+  excludeMetadata?: ExcludeMetadata;
+  includeHasMetadataFlag?: IncludeHasMetadata;
+}) {
+  const {
+    excludeMetadata = false,
+    includeHasMetadataFlag = false,
+    clickhouseConfigs,
+    ...rest
+  } = props;
+
   const rows = await getScoresUiGeneric<{
     id: string;
     project_id: string;
@@ -378,6 +495,7 @@ export const getScoresUiTable = async (props: {
     source: string;
     data_type: string;
     comment: string | null;
+    metadata: ExcludeMetadata extends true ? never : Record<string, string>;
     trace_id: string;
     observation_id: string | null;
     author_user_id: string | null;
@@ -391,37 +509,53 @@ export const getScoresUiTable = async (props: {
     queue_id: string | null;
     created_at: string;
     updated_at: string;
+    // has_metadata is 0 or 1 from ClickHouse, later converted to a boolean
+    has_metadata: IncludeHasMetadata extends true ? 0 | 1 : never;
   }>({
     select: "rows",
     tags: { kind: "analytic" },
-    ...props,
+    excludeMetadata,
+    includeHasMetadataFlag,
+    clickhouseConfigs,
+    ...rest,
   });
 
-  return rows.map((row) => ({
-    projectId: row.project_id,
-    environment: row.environment,
-    authorUserId: row.author_user_id,
-    traceId: row.trace_id,
-    observationId: row.observation_id,
-    traceUserId: row.user_id,
-    traceName: row.trace_name,
-    traceTags: row.trace_tags,
-    configId: row.config_id,
-    queueId: row.queue_id,
-    createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
-    updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
-    stringValue: row.string_value,
-    comment: row.comment,
-    dataType: row.data_type as ScoreDataType,
-    source: row.source as ScoreSourceType,
-    name: row.name,
-    value: row.value,
-    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
-    id: row.id,
-  }));
-};
+  return rows.map((row) => {
+    return {
+      projectId: row.project_id,
+      environment: row.environment,
+      authorUserId: row.author_user_id,
+      traceId: row.trace_id,
+      observationId: row.observation_id,
+      traceUserId: row.user_id,
+      traceName: row.trace_name,
+      traceTags: row.trace_tags,
+      configId: row.config_id,
+      queueId: row.queue_id,
+      createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
+      updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
+      stringValue: row.string_value,
+      comment: row.comment,
+      dataType: row.data_type as ScoreDataType,
+      source: row.source as ScoreSourceType,
+      name: row.name,
+      value: row.value,
+      timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+      id: row.id,
+      metadata: (excludeMetadata
+        ? undefined
+        : (parseMetadataCHRecordToDomain(row.metadata ?? {}) ??
+          {})) as ExcludeMetadata extends true
+        ? never
+        : NonNullable<ReturnType<typeof parseMetadataCHRecordToDomain>>,
+      hasMetadata: (includeHasMetadataFlag
+        ? !!row.has_metadata
+        : undefined) as IncludeHasMetadata extends true ? boolean : never,
+    };
+  });
+}
 
-export const getScoresUiGeneric = async <T>(props: {
+const getScoresUiGeneric = async <T>(props: {
   select: "count" | "rows";
   projectId: string;
   filter: FilterState;
@@ -429,8 +563,20 @@ export const getScoresUiGeneric = async <T>(props: {
   limit?: number;
   offset?: number;
   tags?: Record<string, string>;
+  clickhouseConfigs?: ClickHouseClientConfigOptions;
+  excludeMetadata?: boolean;
+  includeHasMetadataFlag?: boolean;
 }): Promise<T[]> => {
-  const { projectId, filter, orderBy, limit, offset } = props;
+  const {
+    projectId,
+    filter,
+    orderBy,
+    limit,
+    offset,
+    clickhouseConfigs,
+    excludeMetadata = false,
+    includeHasMetadataFlag = false,
+  } = props;
 
   const select =
     props.select === "count"
@@ -446,6 +592,7 @@ export const getScoresUiGeneric = async <T>(props: {
         s.source,
         s.data_type,
         s.comment,
+        ${excludeMetadata ? "" : "s.metadata,"}
         s.trace_id,
         s.observation_id,
         s.author_user_id,
@@ -460,6 +607,7 @@ export const getScoresUiGeneric = async <T>(props: {
         t.user_id,
         t.name as trace_name,
         t.tags as trace_tags
+        ${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}
       `;
 
   const { scoresFilter } = getProjectIdDefaultFilter(projectId, {
@@ -500,6 +648,7 @@ export const getScoresUiGeneric = async <T>(props: {
       type: "score",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return rows;
@@ -626,18 +775,18 @@ export const deleteScoresByProjectId = async (projectId: string) => {
 
 export const deleteScoresOlderThanDays = async (
   projectId: string,
-  days: number,
+  beforeDate: Date,
 ) => {
   const query = `
     DELETE FROM scores
     WHERE project_id = {projectId: String}
-    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+    AND timestamp < {cutoffDate: DateTime64(3)};
   `;
   await commandClickhouse({
     query: query,
     params: {
       projectId,
-      numDays: days,
+      cutoffDate: convertDateToClickhouseDateTime(beforeDate),
     },
     clickhouseConfigs: {
       request_timeout: 120_000, // 2 minutes
@@ -705,7 +854,8 @@ export const getAggregatedScoresForPrompts = async (
       s.value,
       s.source,
       s.data_type,
-      s.comment
+      s.comment,
+      length(mapKeys(s.metadata)) > 0 AS has_metadata
     FROM scores s FINAL LEFT JOIN observations o FINAL 
       ON o.trace_id = s.trace_id 
       AND o.project_id = s.project_id 
@@ -718,7 +868,13 @@ export const getAggregatedScoresForPrompts = async (
     ${fetchScoreRelation === "trace" ? "AND s.observation_id IS NULL" : ""}
   `;
 
-  const rows = await queryClickhouse<ScoreAggregation & { prompt_id: string }>({
+  const rows = await queryClickhouse<
+    ScoreAggregation & {
+      prompt_id: string;
+      // has_metadata is 0 or 1 from ClickHouse, later converted to a boolean
+      has_metadata: 0 | 1;
+    }
+  >({
     query,
     params: {
       projectId,
@@ -735,6 +891,7 @@ export const getAggregatedScoresForPrompts = async (
   return rows.map((row) => ({
     ...convertScoreAggregation(row),
     promptId: row.prompt_id,
+    hasMetadata: !!row.has_metadata,
   }));
 };
 
@@ -805,16 +962,23 @@ export const getScoreCountOfProjectsSinceCreationDate = async ({
   return Number(rows[0]?.count ?? 0);
 };
 
-export const getDistinctScoreNames = async (
-  projectId: string,
-  cutoffCreatedAt: Date,
-  filter: FilterState,
-  isTimestampFilter: (filter: FilterCondition) => filter is TimeFilter,
-) => {
+export const getDistinctScoreNames = async (p: {
+  projectId: string;
+  cutoffCreatedAt: Date;
+  filter: FilterState;
+  isTimestampFilter: (filter: FilterCondition) => filter is TimeFilter;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
+}) => {
+  const {
+    projectId,
+    cutoffCreatedAt,
+    filter,
+    isTimestampFilter,
+    clickhouseConfigs,
+  } = p;
   const scoreTimestampFilter = filter?.find(isTimestampFilter);
 
-  const query = `
-    SELECT DISTINCT
+  const query = `    SELECT DISTINCT
       name
     FROM scores s 
     WHERE s.project_id = {projectId: String}
@@ -841,6 +1005,7 @@ export const getDistinctScoreNames = async (
       kind: "list",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return rows.map((row) => row.name);
@@ -894,8 +1059,7 @@ export const getScoresForPostHog = async function* (
   minTimestamp: Date,
   maxTimestamp: Date,
 ) {
-  const query = `
-    SELECT
+  const query = `    SELECT
       s.id as id,
       s.timestamp as timestamp,
       s.name as name,
@@ -930,6 +1094,13 @@ export const getScoresForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
@@ -958,8 +1129,7 @@ export const getScoresForPostHog = async function* (
 };
 
 export const hasAnyScore = async (projectId: string) => {
-  const query = `
-    SELECT 1
+  const query = `    SELECT 1
     FROM scores
     WHERE project_id = {projectId: String}
     LIMIT 1
@@ -980,3 +1150,41 @@ export const hasAnyScore = async (projectId: string) => {
 
   return rows.length > 0;
 };
+
+export const getScoreMetadataById = async (
+  projectId: string,
+  id: string,
+  source?: ScoreSourceType,
+) => {
+  const query = `    SELECT 
+      metadata
+    FROM scores s
+    WHERE s.project_id = {projectId: String}
+    AND s.id = {id: String}
+    ${source ? `AND s.source = {source: String}` : ""}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
+    LIMIT 1
+  `;
+
+  const rows = await queryClickhouse<Pick<ScoreRecordReadType, "metadata">>({
+    query,
+    params: {
+      projectId,
+      id,
+      ...(source !== undefined ? { source } : {}),
+    },
+    tags: {
+      feature: "tracing",
+      type: "score",
+      kind: "getScoreMetadataById",
+      projectId,
+    },
+  });
+
+  return rows
+    .map((row) =>
+      parseMetadataCHRecordToDomain(row.metadata as Record<string, string>),
+    )
+    .shift();
+};

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,6 +1,7 @@
 import { ScoreDataType } from "@prisma/client";
 import { ScoreRecordReadType } from "./definitions";
-import { Score, ScoreSourceType } from "./types";
+import { ScoreDomain, ScoreSourceType } from "../../domain/scores";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 
 export type ScoreAggregation = {
   id: string;
@@ -12,7 +13,7 @@ export type ScoreAggregation = {
   comment: string | null;
 };
 
-export const convertToScore = (row: ScoreRecordReadType): Score => {
+export const convertToScore = (row: ScoreRecordReadType): ScoreDomain => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),
@@ -24,6 +25,7 @@ export const convertToScore = (row: ScoreRecordReadType): Score => {
     value: row.value ?? null,
     source: row.source as ScoreSourceType,
     comment: row.comment ?? null,
+    metadata: parseMetadataCHRecordToDomain(row.metadata),
     authorUserId: row.author_user_id ?? null,
     configId: row.config_id ?? null,
     dataType: row.data_type as ScoreDataType,

packages/shared/src/server/repositories/traces.ts

@@ -26,7 +26,21 @@ import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
 import { env } from "../../env";
+import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 
+/**
+ * Checks if trace exists in clickhouse.
+ *
+ * @param {string} projectId - Project ID for the trace
+ * @param {string} traceId - ID of the trace to check
+ * @param {Date} timestamp - Timestamp for time-based filtering, uses event payload or job timestamp
+ * @param {FilterState} filter - Filter for the trace
+ * @returns {Promise<boolean>} - True if trace exists
+ *
+ * Notes:
+ * • Filters within ±2 day window
+ * • Used for validating trace references before eval job creation
+ */
 export const checkTraceExists = async (
   projectId: string,
   traceId: string,
@@ -82,11 +96,12 @@ export const checkTraceExists = async (
     SELECT 
       t.id as id, 
       t.project_id as project_id
-    FROM traces t FINAL 
+    FROM traces t FINAL
     ${observationFilterRes ? `INNER JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id` : ""}
     WHERE ${tracesFilterRes.query}
     AND t.project_id = {projectId: String}
     AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
+    AND timestamp <= {timestamp: DateTime64(3)} + INTERVAL 2 DAY
     GROUP BY t.id, t.project_id
   `;
 
@@ -136,6 +151,7 @@ export const getTracesByIds = async (
   traceIds: string[],
   projectId: string,
   timestamp?: Date,
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined,
 ) => {
   const query = `
       SELECT * 
@@ -158,6 +174,7 @@ export const getTracesByIds = async (
       kind: "byId",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return records.map(convertClickhouseToDomain);
@@ -546,18 +563,18 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
 
 export const deleteTracesOlderThanDays = async (
   projectId: string,
-  days: number,
+  beforeDate: Date,
 ) => {
   const query = `
     DELETE FROM traces
     WHERE project_id = {projectId: String}
-    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+    AND timestamp < {cutoffDate: DateTime64(3)};
   `;
   await commandClickhouse({
     query: query,
     params: {
       projectId,
-      numDays: days,
+      cutoffDate: convertDateToClickhouseDateTime(beforeDate),
     },
     clickhouseConfigs: {
       request_timeout: 120_000, // 2 minutes
@@ -907,6 +924,13 @@ export const getTracesForPostHog = async function* (
       kind: "analytic",
       projectId,
     },
+    clickhouseConfigs: {
+      request_timeout: 300_000, // 5 minutes
+      clickhouse_settings: {
+        join_algorithm: "grace_hash",
+        grace_hash_join_initial_buckets: "32",
+      },
+    },
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,12 +1,12 @@
-import { Prisma } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { parseJsonPrioritised } from "../../utils/json";
-import { Trace } from "./types";
+import { TraceDomain } from "../../domain";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 
 export const convertTraceDomainToClickhouse = (
-  trace: Trace,
+  trace: TraceDomain,
 ): TraceRecordReadType => {
   return {
     id: trace.id,
@@ -33,7 +33,7 @@ export const convertTraceDomainToClickhouse = (
 
 export const convertClickhouseToDomain = (
   record: TraceRecordReadType,
-): Trace => {
+): TraceDomain => {
   return {
     id: record.id,
     projectId: record.project_id,
@@ -47,22 +47,12 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: (record.input
-      ? parseJsonPrioritised(record.input)
-      : null) as Prisma.JsonValue | null,
-    output: (record.output
-      ? parseJsonPrioritised(record.output)
-      : null) as Prisma.JsonValue | null,
-    metadata:
-      record.metadata &&
-      Object.fromEntries(
-        Object.entries(record.metadata ?? {}).map(([key, val]) => [
-          key,
-          val && parseJsonPrioritised(val),
-        ]),
-      ),
+    input: record.input ? (parseJsonPrioritised(record.input) ?? null) : null,
+    output: record.output
+      ? (parseJsonPrioritised(record.output) ?? null)
+      : null,
+    metadata: parseMetadataCHRecordToDomain(record.metadata),
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
-    externalId: null,
   };
 };

packages/shared/src/server/repositories/types.ts

@@ -1,144 +1,3 @@
-import { Prisma, ScoreDataType } from "@prisma/client";
-import Decimal from "decimal.js";
-
 export type TableCount = {
   count: number;
 };
-
-export const ObservationType = {
-  SPAN: "SPAN",
-  EVENT: "EVENT",
-  GENERATION: "GENERATION",
-} as const;
-export type ObservationType =
-  (typeof ObservationType)[keyof typeof ObservationType];
-export const ObservationLevel = {
-  DEBUG: "DEBUG",
-  DEFAULT: "DEFAULT",
-  WARNING: "WARNING",
-  ERROR: "ERROR",
-} as const;
-export type ObservationLevelType =
-  (typeof ObservationLevel)[keyof typeof ObservationLevel];
-export const ScoreSource = {
-  ANNOTATION: "ANNOTATION",
-  API: "API",
-  EVAL: "EVAL",
-} as const;
-export type ScoreSourceType = (typeof ScoreSource)[keyof typeof ScoreSource];
-
-export type Observation = {
-  id: string;
-  traceId: string | null;
-  projectId: string;
-  environment: string;
-  type: ObservationType;
-  startTime: Date;
-  endTime: Date | null;
-  name: string | null;
-  metadata: Prisma.JsonValue | null;
-  parentObservationId: string | null;
-  level: ObservationLevelType;
-  statusMessage: string | null;
-  version: string | null;
-  createdAt: Date;
-  updatedAt: Date;
-  model: string | null;
-  internalModel: string | null;
-  internalModelId: string | null;
-  modelParameters: unknown | null;
-  input: unknown | null;
-  output: unknown | null;
-  promptTokens: number;
-  completionTokens: number;
-  totalTokens: number;
-  unit: string | null;
-  inputCost: Decimal | null;
-  outputCost: Decimal | null;
-  totalCost: Decimal | null;
-  calculatedInputCost: Decimal | null;
-  calculatedOutputCost: Decimal | null;
-  calculatedTotalCost: Decimal | null;
-  completionStartTime: Date | null;
-  promptId: string | null;
-};
-
-export type ObservationView = {
-  id: string;
-  traceId: string | null;
-  projectId: string;
-  type: ObservationType;
-  startTime: Date;
-  endTime: Date | null;
-  environment: string;
-  name: string | null;
-  metadata: Prisma.JsonValue | null;
-  parentObservationId: string | null;
-  level: ObservationLevelType;
-  statusMessage: string | null;
-  version: string | null;
-  createdAt: Date;
-  updatedAt: Date;
-  model: string | null;
-  modelParameters: unknown | null;
-  input: unknown | null;
-  output: unknown | null;
-  promptTokens: number;
-  completionTokens: number;
-  totalTokens: number;
-  unit: string | null;
-  completionStartTime: Date | null;
-  promptId: string | null;
-  promptName: string | null;
-  promptVersion: number | null;
-  modelId: string | null;
-  inputPrice: Decimal | null;
-  outputPrice: Decimal | null;
-  totalPrice: Decimal | null;
-  calculatedInputCost: Decimal | null;
-  calculatedOutputCost: Decimal | null;
-  calculatedTotalCost: Decimal | null;
-  latency: number | null;
-  timeToFirstToken: number | null;
-};
-
-export type Score = {
-  id: string;
-  timestamp: Date;
-  projectId: string;
-  environment: string;
-  name: string;
-  value: number | null;
-  source: ScoreSourceType;
-  authorUserId: string | null;
-  comment: string | null;
-  traceId: string;
-  observationId: string | null;
-  configId: string | null;
-  stringValue: string | null;
-  queueId: string | null;
-  createdAt: Date;
-  updatedAt: Date;
-  dataType: ScoreDataType;
-};
-
-export type Trace = {
-  id: string;
-  timestamp: Date;
-  name: string | null;
-  userId: string | null;
-  environment: string;
-  metadata: Prisma.JsonValue | null;
-  release: string | null;
-  version: string | null;
-  projectId: string;
-  public: boolean;
-  bookmarked: boolean;
-  tags: string[];
-  input: unknown | null;
-  output: unknown | null;
-  sessionId: string | null;
-  createdAt: Date;
-  updatedAt: Date;
-  externalId: string | null;
-};

packages/shared/src/server/services/DashboardService/DashboardService.ts

@@ -0,0 +1,329 @@
+import { prisma } from "../../../db";
+import { LangfuseConflictError, type OrderByState } from "../../../";
+import {
+  CreateWidgetInput,
+  WidgetDomain,
+  WidgetListResponse,
+  DashboardDomain,
+  DashboardListResponse,
+  DashboardDomainSchema,
+  WidgetDomainSchema,
+  DashboardDefinitionSchema,
+} from "./types";
+import { z } from "zod";
+
+export class DashboardService {
+  /**
+   * Retrieves a list of dashboards for a given project.
+   */
+  public static async listDashboards(props: {
+    projectId: string;
+    limit?: number;
+    page?: number;
+    orderBy?: OrderByState;
+  }): Promise<DashboardListResponse> {
+    const { projectId, limit, page, orderBy } = props;
+
+    const skip = page && limit ? (page - 1) * limit : undefined;
+    const take = limit;
+
+    const [dashboards, totalCount] = await Promise.all([
+      prisma.dashboard.findMany({
+        where: {
+          projectId,
+        },
+        orderBy: orderBy
+          ? [{ [orderBy.column]: orderBy.order.toLowerCase() }]
+          : [{ updatedAt: "desc" }],
+        skip,
+        take,
+      }),
+      prisma.dashboard.count({
+        where: {
+          projectId,
+        },
+      }),
+    ]);
+
+    const domainDashboards = dashboards.map((dashboard) =>
+      DashboardDomainSchema.parse(dashboard),
+    );
+
+    return {
+      dashboards: domainDashboards,
+      totalCount,
+    };
+  }
+
+  /**
+   * Creates a new dashboard.
+   */
+  public static async createDashboard(
+    projectId: string,
+    name: string,
+    description: string,
+    userId?: string,
+    initialDefinition: z.infer<typeof DashboardDefinitionSchema> = {
+      widgets: [],
+    },
+  ): Promise<DashboardDomain> {
+    const newDashboard = await prisma.dashboard.create({
+      data: {
+        name,
+        description,
+        projectId,
+        createdBy: userId,
+        updatedBy: userId,
+        definition: initialDefinition,
+      },
+    });
+
+    return DashboardDomainSchema.parse(newDashboard);
+  }
+
+  /**
+   * Updates a dashboard's definition.
+   */
+  public static async updateDashboardDefinition(
+    dashboardId: string,
+    projectId: string,
+    definition: z.infer<typeof DashboardDefinitionSchema>,
+    userId?: string,
+  ): Promise<DashboardDomain> {
+    const updatedDashboard = await prisma.dashboard.update({
+      where: {
+        id: dashboardId,
+        projectId,
+      },
+      data: {
+        updatedBy: userId,
+        definition: {
+          widgets: definition.widgets.map((widget) => ({
+            type: "widget",
+            id: widget.id,
+            widgetId: widget.widgetId,
+            x: widget.x,
+            y: widget.y,
+            x_size: widget.x_size,
+            y_size: widget.y_size,
+          })),
+        },
+      },
+    });
+
+    return DashboardDomainSchema.parse(updatedDashboard);
+  }
+
+  /**
+   * Gets a dashboard by ID.
+   */
+  public static async getDashboard(
+    dashboardId: string,
+    projectId: string,
+  ): Promise<DashboardDomain | null> {
+    const dashboard = await prisma.dashboard.findFirst({
+      where: {
+        id: dashboardId,
+        projectId,
+      },
+    });
+
+    if (!dashboard) {
+      return null;
+    }
+
+    return DashboardDomainSchema.parse(dashboard);
+  }
+
+  /**
+   * Deletes a dashboard.
+   */
+  public static async deleteDashboard(
+    dashboardId: string,
+    projectId: string,
+  ): Promise<void> {
+    await prisma.dashboard.delete({
+      where: {
+        id: dashboardId,
+        projectId,
+      },
+    });
+  }
+
+  /**
+   * Retrieves a list of dashboard widgets for a given project.
+   */
+  public static async listWidgets(props: {
+    projectId: string;
+    limit?: number;
+    page?: number;
+    orderBy?: OrderByState;
+  }): Promise<WidgetListResponse> {
+    const { projectId, limit, page, orderBy } = props;
+
+    const skip = page && limit ? (page - 1) * limit : undefined;
+    const take = limit;
+
+    const [widgets, totalCount] = await Promise.all([
+      prisma.dashboardWidget.findMany({
+        where: {
+          projectId,
+        },
+        orderBy: orderBy
+          ? [{ [orderBy.column]: orderBy.order.toLowerCase() }]
+          : [{ updatedAt: "desc" }],
+        skip,
+        take,
+      }),
+      prisma.dashboardWidget.count({
+        where: {
+          projectId,
+        },
+      }),
+    ]);
+
+    const domainWidgets = widgets.map((widget) =>
+      WidgetDomainSchema.parse(widget),
+    );
+
+    return {
+      widgets: domainWidgets,
+      totalCount,
+    };
+  }
+
+  /**
+   * Creates a new dashboard widget.
+   */
+  public static async createWidget(
+    projectId: string,
+    input: CreateWidgetInput,
+    userId?: string,
+  ): Promise<WidgetDomain> {
+    const newWidget = await prisma.dashboardWidget.create({
+      data: {
+        name: input.name,
+        description: input.description,
+        projectId,
+        view: input.view,
+        dimensions: input.dimensions,
+        metrics: input.metrics,
+        filters: input.filters,
+        chartType: input.chartType,
+        chartConfig: input.chartConfig,
+        createdBy: userId,
+        updatedBy: userId,
+      },
+    });
+
+    return WidgetDomainSchema.parse(newWidget);
+  }
+
+  /**
+   * Gets a dashboard widget by ID.
+   */
+  public static async getWidget(
+    widgetId: string,
+    projectId: string,
+  ): Promise<WidgetDomain | null> {
+    const widget = await prisma.dashboardWidget.findFirst({
+      where: {
+        id: widgetId,
+        projectId,
+      },
+    });
+
+    if (!widget) {
+      return null;
+    }
+
+    return WidgetDomainSchema.parse(widget);
+  }
+
+  /**
+   * Updates an existing dashboard widget.
+   */
+  public static async updateWidget(
+    projectId: string,
+    widgetId: string,
+    input: CreateWidgetInput,
+    userId?: string,
+  ): Promise<WidgetDomain> {
+    const updatedWidget = await prisma.dashboardWidget.update({
+      where: {
+        id: widgetId,
+        projectId,
+      },
+      data: {
+        name: input.name,
+        description: input.description,
+        view: input.view,
+        dimensions: input.dimensions,
+        metrics: input.metrics,
+        filters: input.filters,
+        chartType: input.chartType,
+        chartConfig: input.chartConfig,
+        updatedBy: userId,
+      },
+    });
+
+    return WidgetDomainSchema.parse({
+      id: widgetId,
+      projectId,
+      createdAt: updatedWidget.createdAt,
+      updatedAt: updatedWidget.updatedAt,
+      createdBy: updatedWidget.createdBy,
+      updatedBy: updatedWidget.updatedBy,
+      name: updatedWidget.name,
+      description: updatedWidget.description,
+      view: updatedWidget.view,
+      dimensions: updatedWidget.dimensions as WidgetDomain["dimensions"],
+      metrics: updatedWidget.metrics as WidgetDomain["metrics"],
+      filters: updatedWidget.filters as WidgetDomain["filters"],
+      chartType: updatedWidget.chartType,
+      chartConfig: updatedWidget.chartConfig as WidgetDomain["chartConfig"],
+    });
+  }
+
+  /**
+   * Deletes a dashboard widget.
+   * Throws an error if the widget is still referenced in any dashboard.
+   */
+  public static async deleteWidget(
+    widgetId: string,
+    projectId: string,
+  ): Promise<void> {
+    // First check if this widget is referenced in any dashboard definitions
+    const referencingDashboards = await prisma.dashboard.findMany({
+      where: {
+        projectId,
+        definition: {
+          path: ["widgets"],
+          array_contains: [{ widgetId }],
+        },
+      },
+      select: {
+        id: true,
+        name: true,
+      },
+    });
+
+    if (referencingDashboards.length > 0) {
+      const dashboardNames = referencingDashboards
+        .map((d) => `"${d.name}"`)
+        .join(", ");
+
+      throw new LangfuseConflictError(
+        `Cannot delete widget because it is still used in the following dashboards: ${dashboardNames}. Please remove the widget from these dashboards first.`,
+      );
+    }
+
+    // Delete the widget if it's not referenced
+    await prisma.dashboardWidget.delete({
+      where: {
+        id: widgetId,
+        projectId,
+      },
+    });
+  }
+}

packages/shared/src/server/services/DashboardService/index.ts

@@ -0,0 +1,2 @@
+export * from "./DashboardService";
+export * from "./types";

packages/shared/src/server/services/DashboardService/types.ts

@@ -0,0 +1,125 @@
+import { DashboardWidgetChartType, DashboardWidgetViews } from "@prisma/client";
+import { z } from "zod";
+import { singleFilter } from "../../../";
+
+export const BaseTimeSeriesChartConfig = z.object({});
+export const BaseTotalValueChartConfig = z.object({
+  row_limit: z.number().int().positive().lte(1000).optional(),
+});
+
+export const LineChartTimeSeriesConfig = BaseTimeSeriesChartConfig.extend({
+  type: z.literal("LINE_TIME_SERIES"),
+});
+export const BarChartTimeSeriesConfig = BaseTimeSeriesChartConfig.extend({
+  type: z.literal("BAR_TIME_SERIES"),
+});
+
+export const HorizontalBarChartConfig = BaseTotalValueChartConfig.extend({
+  type: z.literal("HORIZONTAL_BAR"),
+});
+export const VerticalBarChartConfig = BaseTotalValueChartConfig.extend({
+  type: z.literal("VERTICAL_BAR"),
+});
+export const PieChartConfig = BaseTotalValueChartConfig.extend({
+  type: z.literal("PIE"),
+});
+
+// Define dimension schema
+export const DimensionSchema = z.object({
+  field: z.string(),
+});
+
+// Define metric schema
+export const MetricSchema = z.object({
+  measure: z.string(),
+  agg: z.string(),
+});
+
+// Define chart config schema based on chart type
+export const ChartConfigSchema = z.discriminatedUnion("type", [
+  LineChartTimeSeriesConfig,
+  BarChartTimeSeriesConfig,
+  HorizontalBarChartConfig,
+  VerticalBarChartConfig,
+  PieChartConfig,
+]);
+
+export const DashboardDefinitionWidgetWidgetSchema = z.object({
+  type: z.literal("widget"),
+  id: z.string(),
+  widgetId: z.string(),
+  x: z.number().int().gte(0),
+  y: z.number().int().gte(0),
+  x_size: z.number().int().positive(),
+  y_size: z.number().int().positive(),
+});
+
+export const DashboardDefinitionWidgetSchema = z.discriminatedUnion("type", [
+  DashboardDefinitionWidgetWidgetSchema,
+]);
+
+export const DashboardDefinitionSchema = z.object({
+  widgets: z.array(DashboardDefinitionWidgetSchema),
+});
+
+// Define the dashboard domain object
+export const DashboardDomainSchema = z.object({
+  id: z.string(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  createdBy: z.string().nullable(),
+  updatedBy: z.string().nullable(),
+  projectId: z.string().nullable(),
+  name: z.string(),
+  description: z.string(),
+  definition: DashboardDefinitionSchema,
+});
+
+// Define the dashboard list response
+export const DashboardListResponseSchema = z.object({
+  dashboards: z.array(DashboardDomainSchema),
+  totalCount: z.number(),
+});
+
+// Define the widget domain object
+export const WidgetDomainSchema = z.object({
+  id: z.string(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  createdBy: z.string().nullable(),
+  updatedBy: z.string().nullable(),
+  projectId: z.string().nullable(),
+  name: z.string(),
+  description: z.string(),
+  view: z.nativeEnum(DashboardWidgetViews),
+  dimensions: z.array(DimensionSchema),
+  metrics: z.array(MetricSchema),
+  filters: z.array(singleFilter),
+  chartType: z.nativeEnum(DashboardWidgetChartType),
+  chartConfig: ChartConfigSchema,
+});
+
+// Define create widget input schema
+export const CreateWidgetInputSchema = z.object({
+  name: z.string().min(1, "Widget name is required"),
+  description: z.string(),
+  view: z.nativeEnum(DashboardWidgetViews),
+  dimensions: z.array(DimensionSchema),
+  metrics: z.array(MetricSchema),
+  filters: z.array(singleFilter),
+  chartType: z.nativeEnum(DashboardWidgetChartType),
+  chartConfig: ChartConfigSchema,
+});
+
+// Define the widget list response
+export const WidgetListResponseSchema = z.object({
+  widgets: z.array(WidgetDomainSchema),
+  totalCount: z.number(),
+});
+
+// Export types derived from schemas
+export type DashboardDomain = z.infer<typeof DashboardDomainSchema>;
+export type DashboardListResponse = z.infer<typeof DashboardListResponseSchema>;
+export type WidgetDomain = z.infer<typeof WidgetDomainSchema>;
+export type CreateWidgetInput = z.infer<typeof CreateWidgetInputSchema>;
+export type WidgetListResponse = z.infer<typeof WidgetListResponseSchema>;

packages/shared/src/server/services/PromptService/index.ts

@@ -372,7 +372,9 @@ export class PromptService {
             );
             const regex = new RegExp(combinedPattern, "g");
 
-            const replaceValue = JSON.stringify(resolvedDepPrompt).slice(1, -1); // this is necessary to avoid parsing errors as resolved value is unstringified
+            const replaceValue = JSON.stringify(resolvedDepPrompt)
+              .slice(1, -1) // this is necessary to avoid parsing errors as resolved value is unstringified
+              .replace(/\$/g, "$$$$"); // Escape dollar signs in replacement string. $ has special meaning in replace(), so we need to escape it with $$. Since we're in a string that will be used as a replacement, we need to double escape it (hence $$$$)
 
             resolvedPrompt = resolvedPrompt.replace(regex, replaceValue);
           }

packages/shared/src/server/services/StorageService.ts

@@ -14,8 +14,10 @@ import {
   ContainerClient,
   StorageSharedKeyCredential,
 } from "@azure/storage-blob";
+import { Storage, Bucket, GetSignedUrlConfig } from "@google-cloud/storage";
 import { logger } from "../logger";
 import { env } from "../../env";
+import { backOff } from "exponential-backoff";
 
 type UploadFile = {
   fileName: string;
@@ -60,6 +62,9 @@ export class StorageServiceFactory {
    * @param params.externalEndpoint - External endpoint to replace the internal endpoint in the signed URL.
    * @param params.region - Region in which the bucket resides
    * @param params.forcePathStyle - Add bucket name into the path instead of the domain name. Mainly used for MinIO.
+   * @param params.useAzureBlob - Use Azure Blob Storage instead of S3
+   * @param params.useGoogleCloudStorage - Use Google Cloud Storage instead of S3
+   * @param params.googleCloudCredentials - Google Cloud Storage credentials JSON string or path to credentials file
    */
   public static getInstance(params: {
     accessKeyId: string | undefined;
@@ -70,10 +75,25 @@ export class StorageServiceFactory {
     region: string | undefined;
     forcePathStyle: boolean;
     useAzureBlob?: boolean;
+    useGoogleCloudStorage?: boolean;
+    googleCloudCredentials?: string;
   }): StorageService {
     if (params.useAzureBlob || env.LANGFUSE_USE_AZURE_BLOB === "true") {
       return new AzureBlobStorageService(params);
     }
+    if (
+      params.useGoogleCloudStorage ||
+      env.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE === "true"
+    ) {
+      // Use provided credentials or fall back to environment variable
+      const googleParams = {
+        ...params,
+        googleCloudCredentials:
+          params.googleCloudCredentials ||
+          env.LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS,
+      };
+      return new GoogleCloudStorageService(googleParams);
+    }
     return new S3StorageService(params);
   }
 }
@@ -222,6 +242,12 @@ class AzureBlobStorageService implements StorageService {
   }
 
   public async deleteFiles(paths: string[]): Promise<void> {
+    await backOff(() => this.deleteFileNonRetrying(paths), {
+      numOfAttempts: 3,
+    });
+  }
+
+  async deleteFileNonRetrying(paths: string[]): Promise<void> {
     try {
       await this.createContainerIfNotExists();
 
@@ -490,6 +516,12 @@ class S3StorageService implements StorageService {
   }
 
   public async deleteFiles(paths: string[]): Promise<void> {
+    await backOff(() => this.deleteFilesNonRetrying(paths), {
+      numOfAttempts: 3,
+    });
+  }
+
+  async deleteFilesNonRetrying(paths: string[]): Promise<void> {
     const chunkSize = 900;
     const chunks = [];
 
@@ -508,14 +540,19 @@ class S3StorageService implements StorageService {
         });
         const result = await this.client.send(command);
         if (result?.Errors && result?.Errors?.length > 0) {
-          logger.error("Failed to delete files from S3", {
+          const errors = result.Errors.map((e) => e.Key).join(", ");
+          logger.error(`Failed to delete files from S3: ${errors} `, {
             errors: result.Errors,
+            files: chunk,
           });
-          throw new Error("Failed to delete files from S3");
+          throw new Error(`Failed to delete files from S3: ${errors}`);
         }
       }
     } catch (err) {
-      logger.error(`Failed to delete files from S3`, err);
+      logger.error(`Failed to delete files from S3`, {
+        error: err,
+        files: paths,
+      });
       throw new Error("Failed to delete files from S3");
     }
   }
@@ -553,3 +590,219 @@ class S3StorageService implements StorageService {
     return url;
   }
 }
+
+class GoogleCloudStorageService implements StorageService {
+  private storage: Storage;
+  private bucket: Bucket;
+
+  constructor(params: { bucketName: string; googleCloudCredentials?: string }) {
+    // Initialize Google Cloud Storage client
+    if (params.googleCloudCredentials) {
+      try {
+        // Check if the credentials are a JSON string or a path to a file
+        if (params.googleCloudCredentials.trim().startsWith("{")) {
+          // It's a JSON string
+          this.storage = new Storage({
+            credentials: JSON.parse(params.googleCloudCredentials),
+          });
+        } else {
+          // It's a path to a credentials file
+          this.storage = new Storage({
+            keyFilename: params.googleCloudCredentials,
+          });
+        }
+      } catch (err) {
+        logger.error("Failed to parse Google Cloud Storage credentials", err);
+        throw new Error("Failed to initialize Google Cloud Storage");
+      }
+    } else {
+      // Use default authentication (environment variables or instance metadata)
+      this.storage = new Storage();
+    }
+
+    this.bucket = this.storage.bucket(params.bucketName);
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      const file = this.bucket.file(fileName);
+      const options = {
+        contentType: fileType,
+        resumable: false,
+      };
+
+      if (typeof data === "string") {
+        await file.save(data, options);
+        const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+        return { signedUrl };
+      } else if (data instanceof Readable) {
+        return new Promise((resolve, reject) => {
+          const writeStream = file.createWriteStream(options);
+
+          data
+            .pipe(writeStream)
+            .on("error", (err) => {
+              reject(err);
+            })
+            .on("finish", async () => {
+              try {
+                const signedUrl = await this.getSignedUrl(
+                  fileName,
+                  expiresInSeconds,
+                );
+                resolve({ signedUrl });
+              } catch (err) {
+                reject(err);
+              }
+            });
+        });
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Google Cloud Storage ${fileName}`,
+        err,
+      );
+      throw new Error("Failed to upload to Google Cloud Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    try {
+      const file = this.bucket.file(path);
+      const content = JSON.stringify(body);
+
+      await file.save(content, {
+        contentType: "application/json",
+        resumable: false,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to upload JSON to Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to upload JSON to Google Cloud Storage");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      const file = this.bucket.file(path);
+      const [content] = await file.download();
+
+      return content.toString();
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Google Cloud Storage");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    try {
+      const [files] = await this.bucket.getFiles({ prefix });
+
+      return files.map((file) => ({
+        file: file.name,
+        createdAt: new Date(file.metadata.timeCreated ?? new Date()),
+      }));
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Google Cloud Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Google Cloud Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = false,
+  ): Promise<string> {
+    try {
+      const file = this.bucket.file(fileName);
+
+      const options: GetSignedUrlConfig = {
+        version: "v4",
+        action: "read",
+        expires: Date.now() + ttlSeconds * 1000,
+      };
+
+      if (asAttachment) {
+        options.responseDisposition = `attachment; filename="${fileName}"`;
+      }
+
+      const [url] = await file.getSignedUrl(options);
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate signed URL for Google Cloud Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate signed URL for Google Cloud Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+
+    try {
+      const file = this.bucket.file(path);
+
+      const options: GetSignedUrlConfig = {
+        version: "v4",
+        action: "write",
+        expires: Date.now() + ttlSeconds * 1000,
+        contentType,
+        extensionHeaders: {
+          "Content-Length": params.contentLength.toString(),
+        },
+      };
+
+      const [url] = await file.getSignedUrl(options);
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate signed upload URL for Google Cloud Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate signed upload URL for Google Cloud Storage",
+      );
+    }
+  }
+
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      await Promise.all(
+        paths.map(async (path) => {
+          const file = this.bucket.file(path);
+          await file.delete({ ignoreNotFound: true });
+        }),
+      );
+    } catch (err) {
+      logger.error(`Failed to delete files from Google Cloud Storage`, err);
+      throw Error("Failed to delete files from Google Cloud Storage");
+    }
+  }
+}

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -1,3 +1,4 @@
+import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { OrderByState } from "../../interfaces/orderBy";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { FilterState } from "../../types";
@@ -85,6 +86,7 @@ export const getSessionsWithMetrics = async (props: {
   orderBy?: OrderByState;
   limit?: number;
   page?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
 }) => {
   const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({
     select: "metrics",
@@ -93,6 +95,7 @@ export const getSessionsWithMetrics = async (props: {
     orderBy: props.orderBy,
     limit: props.limit,
     page: props.page,
+    clickhouseConfigs: props.clickhouseConfigs,
     tags: { kind: "analytic" },
   });
 
@@ -112,10 +115,12 @@ export type FetchSessionsTableProps = {
   limit?: number;
   page?: number;
   tags?: Record<string, string>;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
 };
 
 const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
-  const { select, projectId, filter, orderBy, limit, page } = props;
+  const { select, projectId, filter, orderBy, limit, page, clickhouseConfigs } =
+    props;
 
   let sqlSelect: string;
   switch (select) {
@@ -154,9 +159,10 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
         session_output_usage,
         session_total_usage`;
       break;
-    default:
+    default: {
       const exhaustiveCheckDefault: never = select;
-      throw new Error(`Unknown select type: ${select}`);
+      throw new Error(`Unknown select type: ${exhaustiveCheckDefault}`);
+    }
   }
 
   const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
@@ -334,6 +340,7 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
       type: "sessions-table",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return res;

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -18,11 +18,12 @@ import { ScoreAggregate } from "../../features/scores";
 import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
   SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
-  ObservationLevelType,
-  reduceUsageOrCostDetails,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  reduceUsageOrCostDetails,
 } from "../repositories";
+import { ObservationLevelType, TraceDomain } from "../../domain";
+import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 
 export type TracesTableReturnType = Pick<
   TraceRecordReadType,
@@ -40,20 +41,21 @@ export type TracesTableReturnType = Pick<
   | "public"
 >;
 
-export type TracesAllUiReturnType = {
-  id: string;
-  timestamp: Date;
-  name: string | null;
-  projectId: string;
-  userId: string | null;
-  release: string | null;
-  version: string | null;
-  public: boolean;
-  bookmarked: boolean;
-  environment: string | null;
-  sessionId: string | null;
-  tags: string[];
-};
+export type TracesTableUiReturnType = Pick<
+  TraceDomain,
+  | "id"
+  | "projectId"
+  | "timestamp"
+  | "tags"
+  | "bookmarked"
+  | "name"
+  | "release"
+  | "version"
+  | "userId"
+  | "environment"
+  | "sessionId"
+  | "public"
+>;
 
 export type TracesMetricsUiReturnType = {
   id: string;
@@ -78,7 +80,7 @@ export type TracesMetricsUiReturnType = {
 
 export const convertToUiTableRows = (
   row: TracesTableReturnType,
-): TracesAllUiReturnType => {
+): TracesTableUiReturnType => {
   return {
     id: row.id,
     projectId: row.project_id,
@@ -161,6 +163,7 @@ export type FetchTracesTableProps = {
   orderBy?: OrderByState;
   limit?: number;
   page?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
   tags?: Record<string, string>;
 };
 
@@ -192,6 +195,7 @@ export const getTracesTableMetrics = async (props: {
   orderBy?: OrderByState;
   limit?: number;
   page?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
 }): Promise<Array<Omit<TracesMetricsUiReturnType, "scores">>> => {
   const countRows =
     await getTracesTableGeneric<TracesTableMetricsClickhouseReturnType>({
@@ -203,14 +207,24 @@ export const getTracesTableMetrics = async (props: {
   return countRows.map(convertToUITableMetrics);
 };
 
-export const getTracesTable = async (
-  projectId: string,
-  filter: FilterState,
-  searchQuery?: string,
-  orderBy?: OrderByState,
-  limit?: number,
-  page?: number,
-) => {
+export const getTracesTable = async (p: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+  clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
+}) => {
+  const {
+    projectId,
+    filter,
+    searchQuery,
+    orderBy,
+    limit,
+    page,
+    clickhouseConfigs,
+  } = p;
   const rows = await getTracesTableGeneric<TracesTableReturnType>({
     select: "rows",
     tags: { kind: "list" },
@@ -220,14 +234,23 @@ export const getTracesTable = async (
     orderBy,
     limit,
     page,
+    clickhouseConfigs,
   });
 
   return rows.map(convertToUiTableRows);
 };
 
 const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, page, searchQuery } =
-    props;
+  const {
+    select,
+    projectId,
+    filter,
+    orderBy,
+    limit,
+    page,
+    searchQuery,
+    clickhouseConfigs,
+  } = props;
 
   let sqlSelect: string;
   switch (select) {
@@ -455,6 +478,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
       type: "traces-table",
       projectId,
     },
+    clickhouseConfigs,
   });
 
   return res;

packages/shared/src/server/test-utils/org-factory.ts

@@ -29,7 +29,7 @@ export const createOrgProjectAndApiKey = async (
       }),
     },
   });
-  await prisma.project.create({
+  const project = await prisma.project.create({
     data: {
       id: projectId,
       name: v4(),
@@ -47,8 +47,9 @@ export const createOrgProjectAndApiKey = async (
       publicKey: publicKey,
       hashedSecretKey: await hashSecretKey(secretKey),
       displaySecretKey: getDisplaySecretKey(secretKey),
+      scope: "PROJECT",
     },
   });
 
-  return { projectId, publicKey, secretKey, auth };
+  return { projectId, orgId: org.id, publicKey, secretKey, auth, org, project };
 };

packages/shared/src/server/test-utils/tracing-factory.ts

@@ -5,7 +5,9 @@ import {
   ScoreRecordInsertType,
 } from "../repositories/definitions";
 
-export const createTrace = (trace: Partial<TraceRecordInsertType>) => {
+export const createTrace = (
+  trace: Partial<TraceRecordInsertType>,
+): TraceRecordInsertType => {
   return {
     id: v4(),
     project_id: v4(),
@@ -33,7 +35,7 @@ export const createTrace = (trace: Partial<TraceRecordInsertType>) => {
 
 export const createObservation = (
   observation: Partial<ObservationRecordInsertType>,
-) => {
+): ObservationRecordInsertType => {
   return {
     id: v4(),
     trace_id: v4(),
@@ -72,7 +74,9 @@ export const createObservation = (
   };
 };
 
-export const createScore = (score: Partial<ScoreRecordInsertType>) => {
+export const createScore = (
+  score: Partial<ScoreRecordInsertType>,
+): ScoreRecordInsertType => {
   return {
     id: v4(),
     project_id: v4(),
@@ -84,6 +88,7 @@ export const createScore = (score: Partial<ScoreRecordInsertType>) => {
     value: 100.5,
     source: "API",
     comment: "comment",
+    metadata: { "test-key": "test-value" },
     data_type: "NUMERIC" as const,
     created_at: Date.now(),
     updated_at: Date.now(),

packages/shared/src/server/utils/metadata_conversion.ts

@@ -0,0 +1,15 @@
+import { parseJsonPrioritised } from "../../utils/json";
+import { MetadataDomain } from "../../domain";
+
+export function parseMetadataCHRecordToDomain(
+  metadata: Record<string, string>,
+): MetadataDomain {
+  return metadata
+    ? Object.fromEntries(
+        Object.entries(metadata ?? {}).map(([key, val]) => [
+          key,
+          val === null ? null : parseJsonPrioritised(val),
+        ]),
+      )
+    : {};
+}

packages/shared/src/server/utils/transforms/index.ts

@@ -3,6 +3,7 @@ import { Transform } from "stream";
 import { BatchExportFileFormat } from "../../../features/batchExport/types";
 import { transformStreamToCsv } from "./transformStreamToCsv";
 import { transformStreamToJson } from "./transformStreamToJson";
+import { transformStreamToJsonl } from "./transformStreamToJsonl";
 
 export const streamTransformations: Record<
   BatchExportFileFormat,
@@ -10,4 +11,5 @@ export const streamTransformations: Record<
 > = {
   CSV: transformStreamToCsv,
   JSON: transformStreamToJson,
+  JSONL: transformStreamToJsonl,
 };

packages/shared/src/server/utils/transforms/transformStreamToJsonl.ts

@@ -0,0 +1,17 @@
+import { Transform, type TransformCallback } from "stream";
+import { stringify } from "./stringify";
+
+export function transformStreamToJsonl(): Transform {
+  return new Transform({
+    objectMode: true,
+
+    transform(
+      row: Record<string, any>,
+      encoding: BufferEncoding,
+      callback: TransformCallback,
+    ): void {
+      this.push(stringify(row) + "\n");
+      callback();
+    },
+  });
+}

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -176,8 +176,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMappings = [
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
-    uiTableName: "Usage",
-    uiTableId: "usage",
+    uiTableName: "Tokens",
+    uiTableId: "tokens",
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -123,8 +123,8 @@ export const tracesTableUiColumnDefinitions: UiColumnMappings = [
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
-    uiTableName: "Usage",
-    uiTableId: "usage",
+    uiTableName: "Tokens",
+    uiTableId: "tokens",
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",

packages/shared/src/tableDefinitions/tracesTable.ts

@@ -1,5 +1,4 @@
-import { ObservationLevelType } from "../server";
-import { ColumnDefinition, OptionsDefinition } from "..";
+import { ColumnDefinition, ObservationLevelType, OptionsDefinition } from "..";
 
 export const tracesOnlyCols: ColumnDefinition[] = [
   {
@@ -17,6 +16,13 @@ export const tracesOnlyCols: ColumnDefinition[] = [
     options: [], // to be filled in at runtime
     nullable: true,
   },
+  {
+    name: "Environment",
+    id: "environment",
+    type: "stringOptions",
+    internal: 't."environment"',
+    options: [], // to be filled in at runtime
+  },
   {
     name: "Timestamp",
     id: "timestamp",
@@ -125,8 +131,8 @@ export const tracesTableCols: ColumnDefinition[] = [
     nullable: true,
   },
   {
-    name: "Usage",
-    id: "usage",
+    name: "Tokens",
+    id: "tokens",
     type: "number",
     internal: 'generation_metrics."totalTokens"',
     nullable: true,
@@ -180,9 +186,10 @@ export const datasetOnlyCols: ColumnDefinition[] = [datasetCol];
 export const evalTraceTableCols: ColumnDefinition[] = tracesOnlyCols;
 export const evalDatasetFormFilterCols: ColumnDefinition[] = datasetOnlyCols;
 export type TraceOptions = {
-  scores_avg: Array<string>;
-  name: Array<OptionsDefinition>;
-  tags: Array<OptionsDefinition>;
+  scores_avg?: Array<string>;
+  name?: Array<OptionsDefinition>;
+  tags?: Array<OptionsDefinition>;
+  environment?: Array<OptionsDefinition>;
 };
 export type DatasetOptions = {
   datasetId: Array<OptionsDefinition>;
@@ -215,6 +222,9 @@ export function tracesTableColsWithOptions(
     if (col.id === "tags") {
       return { ...col, options: options?.tags ?? [] };
     }
+    if (col.id === "environment") {
+      return { ...col, options: options?.environment ?? [] };
+    }
     return col;
   });
 }

packages/shared/src/types.ts

@@ -33,4 +33,5 @@ export type TableName =
   | "scores"
   | "prompts"
   | "dashboard"
+  | "widgets"
   | "users";

web/Dockerfile

@@ -33,8 +33,7 @@ ENV DOCKER_BUILD 1
 ENV NEXT_MANUAL_SIG_HANDLE true
 
 # pass public variables in build step
-ARG NEXT_PUBLIC_CRISP_WEBSITE_ID
-
+ARG NEXT_PUBLIC_PLAIN_APP_ID
 ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
 ENV NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
 ARG NEXT_PUBLIC_DEMO_ORG_ID

web/next.config.mjs

@@ -11,19 +11,19 @@ import { env } from "./src/env.mjs";
  * img-src https to allow loading images from SSO providers
  */
 const cspHeader = `
-  default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://settings.crisp.chat https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com;
-  style-src 'self' 'unsafe-inline' https://client.crisp.chat https://uptime.betterstack.com;
-  img-src 'self' https: blob: data: http://localhost:* https://client.crisp.chat https://image.crisp.chat https://storage.crisp.chat;
-  font-src 'self' https://client.crisp.chat;
-  frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com https://game.crisp.chat;
+  default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io;
+  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com https://chat.cdn-plain.com;
+  style-src 'self' 'unsafe-inline' https://uptime.betterstack.com https://fonts.googleapis.com;
+  img-src 'self' https: blob: data: http://localhost:*;
+  font-src 'self';
+  frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com;
   worker-src 'self' blob:;
   object-src 'none';
   base-uri 'self';
   form-action 'self';
   frame-ancestors 'none';
-  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://storage.crisp.chat wss://client.relay.crisp.chat wss://stream.relay.crisp.chat https://*.ingest.us.sentry.io https://uptime.betterstack.com;
-  media-src 'self' https: http://localhost:* https://client.crisp.chat;
+  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.ingest.us.sentry.io https://*.sentry.io https://uptime.betterstack.com https://chat.uk.plain.com;
+  media-src 'self' https: http://localhost:*;
   ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
 `;
@@ -139,22 +139,22 @@ const nextConfig = {
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
       // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
       {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.45.2",
+  "version": "3.52.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -18,7 +18,7 @@
     "test-sync": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects sync-server",
     "test-client": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
     "test:watch": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --watch --runInBand",
-    "test:e2e": "dotenv -e ../.env -- playwright test",
+    "test:e2e": "dotenv -e ../.env -- playwright test --reporter=line",
     "test:e2e:server": "cross-env NODE_OPTIONS='--no-experimental-require-module' dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server"
   },
   "dependencies": {
@@ -92,6 +92,7 @@
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.20.5",
+    "@team-plain/typescript-sdk": "^5.8.0",
     "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
@@ -122,7 +123,7 @@
     "langchain": "^0.3.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.462.0",
-    "next": "^14.2.25",
+    "next": "^14.2.26",
     "next-auth": "^4.24.11",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
@@ -142,6 +143,7 @@
     "react-responsive": "^10.0.0",
     "react-syntax-highlighter": "^15.5.0",
     "react18-json-view": "^0.2.8-canary.6",
+    "recharts": "^2.15.2",
     "remark-gfm": "^4.0.0",
     "remark-math": "^6.0.0",
     "remark-parse": "^11.0.0",
@@ -200,9 +202,6 @@
   "ct3aMetadata": {
     "initVersion": "7.13.0"
   },
-  "optionalDependencies": {
-    "crisp-sdk-web": "^1.0.25"
-  },
   "pnpm": {
     "overrides": {
       "nanoid": "^3.3.8",

web/public/generated/api-client/openapi.yml

@@ -72,6 +72,8 @@ components:
         comment:
           type: string
           nullable: true
+        metadata:
+          nullable: true
         dataType:
           $ref: '#/components/schemas/ScoreDataType'
           nullable: true
@@ -121,6 +123,8 @@ components:
         comment:
           type: string
           nullable: true
+        metadata:
+          nullable: true
         configId:
           type: string
           nullable: true

web/public/generated/organizations-api/openapi.yml

@@ -0,0 +1,541 @@
+openapi: 3.0.1
+info:
+  title: langfuse-organizations
+  version: ''
+  description: >-
+    ## General Notes
+
+
+    This admin API is only available on self-hosted instances, not on Langfuse
+    Cloud.
+
+    An administrator must set the `ADMIN_API_KEY` environment variable to use
+    this API.
+
+
+    ## Authentication
+
+
+    Authenticate with the API by setting the Authorization header to `Bearer
+    $ADMIN_API_KEY` where
+
+    `$ADMIN_API_KEY` is the API key set via the container environment.
+
+
+    ## Exports
+
+
+    - OpenAPI spec:
+    https://cloud.langfuse.com/generated/organizations-api/openapi.yml
+
+    - Postman collection:
+    https://cloud.langfuse.com/generated/organizations-postman/collection.json
+
+
+    ## Looking for the Langfuse API?
+
+
+    Are you looking for the full Langfuse API?
+
+    Check out our [API reference](https://api.reference.langfuse.com/#overview).
+paths:
+  /api/admin/organizations:
+    get:
+      description: Get all organizations
+      operationId: organizations_getAll
+      tags:
+        - Organizations
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Organizations'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: &ref_0
+        - BearerAuth: []
+    post:
+      description: Create a new organization
+      operationId: organizations_create
+      tags:
+        - Organizations
+      parameters: []
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Organization'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                name:
+                  type: string
+                metadata:
+                  type: object
+                  additionalProperties: true
+                  nullable: true
+                  description: Optional metadata for the organization
+              required:
+                - name
+  /api/admin/organizations/{organizationId}:
+    get:
+      description: Get organization by ID
+      operationId: organizations_getById
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Organization'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+    put:
+      description: Update an organization
+      operationId: organizations_update
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Organization'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                name:
+                  type: string
+                metadata:
+                  type: object
+                  additionalProperties: true
+                  nullable: true
+                  description: Optional metadata for the organization
+              required:
+                - name
+    delete:
+      description: Delete an organization
+      operationId: organizations_delete
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteOrganizationResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+  /api/admin/organizations/{organizationId}/apiKeys:
+    get:
+      description: Get all API keys for an organization
+      operationId: organizations_getApiKeys
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiKeyList'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+    post:
+      description: Create a new API key for an organization
+      operationId: organizations_createApiKey
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiKeyResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                note:
+                  type: string
+                  nullable: true
+                  description: Optional note for the API key
+  /api/admin/organizations/{organizationId}/apiKeys/{apiKeyId}:
+    delete:
+      description: Delete an API key for an organization
+      operationId: organizations_deleteApiKey
+      tags:
+        - Organizations
+      parameters:
+        - name: organizationId
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: apiKeyId
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DeleteOrganizationResponse'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: string
+      security: *ref_0
+components:
+  schemas:
+    Organizations:
+      title: Organizations
+      type: object
+      description: List of organizations
+      properties:
+        organizations:
+          type: array
+          items:
+            $ref: '#/components/schemas/Organization'
+      required:
+        - organizations
+    Organization:
+      title: Organization
+      type: object
+      description: Organization object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        createdAt:
+          type: string
+          format: date-time
+        metadata:
+          type: object
+          additionalProperties: true
+          description: Metadata for the organization
+      required:
+        - id
+        - name
+        - createdAt
+        - metadata
+    DeleteOrganizationResponse:
+      title: DeleteOrganizationResponse
+      type: object
+      description: Response for successful organization deletion
+      properties:
+        success:
+          type: boolean
+      required:
+        - success
+    ApiKeyList:
+      title: ApiKeyList
+      type: object
+      description: List of API keys for an organization
+      properties:
+        apiKeys:
+          type: array
+          items:
+            $ref: '#/components/schemas/ApiKeySummary'
+      required:
+        - apiKeys
+    ApiKeySummary:
+      title: ApiKeySummary
+      type: object
+      description: Summary of an API key
+      properties:
+        id:
+          type: string
+        createdAt:
+          type: string
+          format: date-time
+        expiresAt:
+          type: string
+          format: date-time
+          nullable: true
+        lastUsedAt:
+          type: string
+          format: date-time
+          nullable: true
+        note:
+          type: string
+          nullable: true
+        publicKey:
+          type: string
+        displaySecretKey:
+          type: string
+      required:
+        - id
+        - createdAt
+        - publicKey
+        - displaySecretKey
+    ApiKeyResponse:
+      title: ApiKeyResponse
+      type: object
+      description: Response for API key creation
+      properties:
+        id:
+          type: string
+        createdAt:
+          type: string
+          format: date-time
+        publicKey:
+          type: string
+        secretKey:
+          type: string
+        displaySecretKey:
+          type: string
+        note:
+          type: string
+          nullable: true
+      required:
+        - id
+        - createdAt
+        - publicKey
+        - secretKey
+        - displaySecretKey
+  securitySchemes:
+    BearerAuth:
+      type: http
+      scheme: bearer

web/public/generated/organizations-postman/collection.json

@@ -0,0 +1,313 @@
+{
+  "info": {
+    "name": "Langfuse Organizations",
+    "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
+    "description": "## General Notes\n\nThis admin API is only available on self-hosted instances, not on Langfuse Cloud.\nAn administrator must set the `ADMIN_API_KEY` environment variable to use this API.\n\n## Authentication\n\nAuthenticate with the API by setting the Authorization header to `Bearer $ADMIN_API_KEY` where\n`$ADMIN_API_KEY` is the API key set via the container environment.\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/generated/organizations-api/openapi.yml\n- Postman collection: https://cloud.langfuse.com/generated/organizations-postman/collection.json\n\n## Looking for the Langfuse API?\n\nAre you looking for the full Langfuse API?\nCheck out our [API reference](https://api.reference.langfuse.com/#overview)."
+  },
+  "variable": [
+    {
+      "key": "baseUrl",
+      "value": "",
+      "type": "string"
+    },
+    {
+      "key": "token",
+      "value": "",
+      "type": "string"
+    }
+  ],
+  "auth": {
+    "type": "bearer",
+    "bearer": [
+      {
+        "key": "token",
+        "value": "{{token}}",
+        "type": "string"
+      }
+    ]
+  },
+  "item": [
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Organizations",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Get All",
+          "request": {
+            "description": "Get all organizations",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a new organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"metadata\": {\n        \"example\": \"UNKNOWN\"\n    }\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get By Id",
+          "request": {
+            "description": "Get organization by ID",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Update",
+          "request": {
+            "description": "Update an organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"metadata\": {\n        \"example\": \"UNKNOWN\"\n    }\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete an organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Api Keys",
+          "request": {
+            "description": "Get all API keys for an organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId/apiKeys",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId",
+                "apiKeys"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create Api Key",
+          "request": {
+            "description": "Create a new API key for an organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId/apiKeys",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId",
+                "apiKeys"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"note\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete Api Key",
+          "request": {
+            "description": "Delete an API key for an organization",
+            "url": {
+              "raw": "{{baseUrl}}/api/admin/organizations/:organizationId/apiKeys/:apiKeyId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "admin",
+                "organizations",
+                ":organizationId",
+                "apiKeys",
+                ":apiKeyId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "organizationId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "apiKeyId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    }
+  ]
+}

web/public/generated/postman/collection.json

@@ -1195,6 +1195,147 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Memberships",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Get Organization Memberships",
+          "request": {
+            "description": "Get all memberships for the organization associated with the API key (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/organizations/memberships",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "organizations",
+                "memberships"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Update Organization Membership",
+          "request": {
+            "description": "Create or update a membership for the organization associated with the API key (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/organizations/memberships",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "organizations",
+                "memberships"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"userId\": \"example\",\n    \"role\": \"OWNER\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Project Memberships",
+          "request": {
+            "description": "Get all memberships for a specific project (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId/memberships",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId",
+                "memberships"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Update Project Membership",
+          "request": {
+            "description": "Create or update a membership for a specific project (requires organization-scoped API key). The user must already be a member of the organization.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId/memberships",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId",
+                "memberships"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"userId\": \"example\",\n    \"role\": \"OWNER\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -1558,6 +1699,224 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a new project (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"metadata\": {\n        \"example\": \"UNKNOWN\"\n    },\n    \"retention\": 0\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Update",
+          "request": {
+            "description": "Update a project by ID (requires organization-scoped API key).",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "PUT",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"metadata\": {\n        \"example\": \"UNKNOWN\"\n    },\n    \"retention\": 0\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a project by ID (requires organization-scoped API key). Project deletion is processed asynchronously.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Api Keys",
+          "request": {
+            "description": "Get all API keys for a project (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId/apiKeys",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId",
+                "apiKeys"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create Api Key",
+          "request": {
+            "description": "Create a new API key for a project (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId/apiKeys",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId",
+                "apiKeys"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"note\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete Api Key",
+          "request": {
+            "description": "Delete an API key for a project (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/projects/:projectId/apiKeys/:apiKeyId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "projects",
+                ":projectId",
+                "apiKeys",
+                ":apiKeyId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "projectId",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "apiKeyId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -1763,6 +2122,233 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Scim",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Get Service Provider Config",
+          "request": {
+            "description": "Get SCIM Service Provider Configuration (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/ServiceProviderConfig",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "ServiceProviderConfig"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Resource Types",
+          "request": {
+            "description": "Get SCIM Resource Types (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/ResourceTypes",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "ResourceTypes"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get Schemas",
+          "request": {
+            "description": "Get SCIM Schemas (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/Schemas",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "Schemas"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List Users",
+          "request": {
+            "description": "List users in the organization (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/Users?filter=&startIndex=&count=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "Users"
+              ],
+              "query": [
+                {
+                  "key": "filter",
+                  "value": "",
+                  "description": "Filter expression (e.g. userName eq \"value\")"
+                },
+                {
+                  "key": "startIndex",
+                  "value": "",
+                  "description": "1-based index of the first result to return (default 1)"
+                },
+                {
+                  "key": "count",
+                  "value": "",
+                  "description": "Maximum number of results to return (default 100)"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create User",
+          "request": {
+            "description": "Create a new user in the organization (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/Users",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "Users"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"userName\": \"example\",\n    \"name\": {\n        \"formatted\": \"example\"\n    },\n    \"emails\": [\n        {\n            \"primary\": true,\n            \"value\": \"example\",\n            \"type\": \"example\"\n        }\n    ],\n    \"active\": true,\n    \"password\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get User",
+          "request": {
+            "description": "Get a specific user by ID (requires organization-scoped API key)",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/Users/:userId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "Users",
+                ":userId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "userId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete User",
+          "request": {
+            "description": "Remove a user from the organization (requires organization-scoped API key). Note that this only removes the user from the organization but does not delete the user entity itself.",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scim/Users/:userId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scim",
+                "Users",
+                ":userId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "userId",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,

web/sentry.client.config.ts

@@ -7,10 +7,13 @@ Sentry.init({
   // Replay may only be enabled for the client-side
   integrations: [
     Sentry.replayIntegration(),
+    Sentry.browserTracingIntegration(),
+    Sentry.httpClientIntegration(),
     // Sentry.debugIntegration(),
-    // Sentry.captureConsoleIntegration({
-    //   levels: ["error"],
-    // }),
+    Sentry.captureConsoleIntegration({
+      levels: ["error"],
+    }),
+    Sentry.browserProfilingIntegration(),
   ],
 
   // Set tracesSampleRate to 1.0 to capture 100%
@@ -28,6 +31,12 @@ Sentry.init({
   replaysOnErrorSampleRate: 1.0,
   debug: false,
 
+  // Set profilesSampleRate to 1.0 to profile every transaction.
+  // Since profilesSampleRate is relative to tracesSampleRate,
+  // the final profiling rate can be computed as tracesSampleRate * profilesSampleRate
+  // For example, a tracesSampleRate of 0.5 and profilesSampleRate of 0.5 would
+  // result in 25% of transactions being profiled (0.5*0.5=0.25)
+  profilesSampleRate: 0.5,
   // ...
 
   // Note: if you want to override the automatic release value, do not set a