chore: release v3.53.0

* chore: make changes to `scores` data structure

* fixup: migrations

* chore: update CH score types and schemas

* feat: support session level scores

* chore: add mock session level scores

* feat(session-scores): create v2 APIs; extract score and trace api service

* chore: clean up todos

* eslint

* eslint

* chore: reorder migrations after rebase

* chore: order unclustered migrations after rebase

* chore: fixes after rebase

* chore

* revert changes to .json file

* chore

* chore: move session_id column

* typo

* chore: migration adjustment

* chore: styling adjustments

* chore: materialize index

* chore: fix types after rebase

* chore: fix types after rebase

* chore: feedback

* chore: refactor scores types file structure

* chore: fix lint

* chore: add api types

* chore: rename

* fix: wait for clickhouse mutations to succeed (#6629)

* fix: wait for clickhouse mutations to succeed

* push

* fix: wait for clickhouse mutations to succeed

---------

Co-authored-by: Max Deichmann <m.deichmann@tum.de>

.env.prod.example

@@ -219,17 +219,6 @@ OTEL_SERVICE_NAME="langfuse"
 ### END Enterprise Edition Configuration
 
 
-
-### START Envs to be deprecated in Langfuse v3.0
-
-# Disable the expensive analytics queries and related features
-# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
-
-### END Envs to be deprecated in Langfuse v3.0
-
-
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
@@ -261,8 +250,9 @@ OTEL_SERVICE_NAME="langfuse"
 # NEXT_PUBLIC_DEMO_ORG_ID=
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
-# Crisp chat
-# NEXT_PUBLIC_CRISP_WEBSITE_ID=
+# Plain Chat
+# NEXT_PUBLIC_PLAIN_APP_ID=
+# PLAIN_AUTHENTICATION_SECRET=
 
 # Admin API
 # ADMIN_API_KEY=

.github/workflows/_deploy_ecs_service.yml

@@ -52,16 +52,11 @@ jobs:
             --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
             --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
             --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
-            --build-arg NEXT_PUBLIC_CRISP_WEBSITE_ID=${{ vars.NEXT_PUBLIC_CRISP_WEBSITE_ID }} \
+            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }} \
             --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
             --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
-            --build-arg NEXT_PUBLIC_DATADOG_APPLICATION_ID=${{ vars.NEXT_PUBLIC_DATADOG_APPLICATION_ID }} \
-            --build-arg NEXT_PUBLIC_DATADOG_CLIENT_TOKEN=${{ vars.NEXT_PUBLIC_DATADOG_CLIENT_TOKEN }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SITE=${{ vars.NEXT_PUBLIC_DATADOG_SITE }} \
-            --build-arg NEXT_PUBLIC_DATADOG_ENVIRONMENT=${{ vars.NEXT_PUBLIC_DATADOG_ENVIRONMENT }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SESSION_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_DATADOG_SESSION_SAMPLE_RATE }} \
-            --build-arg NEXT_PUBLIC_DATADOG_SESSION_REPLAY_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_DATADOG_SESSION_REPLAY_SAMPLE_RATE }} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
       - name: Render AWS ECS Task Definition

.github/workflows/deploy.yml

@@ -21,6 +21,7 @@ on:
           - staging
           - prod-eu
           - prod-us
+          - prod-hipaa
         required: true
 
 concurrency:
@@ -78,7 +79,7 @@ jobs:
                 return `["staging"]`
               }
               if (context.ref === "refs/heads/production") {
-                return `["prod-eu", "prod-us"]`
+                return `["prod-eu", "prod-us", "prod-hipaa"]`
               }
             }
             return "[]"

.github/workflows/pipeline.yml

@@ -212,6 +212,9 @@ jobs:
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
+          echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100" >> .env
+          echo "ADMIN_API_KEY=admin-api-key" >> .env
+          echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
       - name: Run dev containers
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -338,14 +341,28 @@ jobs:
         run: |
           cp .env.dev.example .env
           cp .env.dev.example web/.env
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed
       - name: Build
         run: pnpm run build

.gitignore

@@ -62,4 +62,6 @@ node_modules
 **/.next/*
 **/.turbo/*
 .yarn
-.turbo
+.turbo
+
+web/test-results/*

CONTRIBUTING.md

@@ -27,7 +27,7 @@ The maintainers are available on [Discord](https://langfuse.com/discord) in case
 
 ## Making a change
 
-_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone. Large changes that were not discussed in an issue may be rejected.
+_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone. Changes that were not discussed in an issue may be rejected.
 
 Once we've discussed your changes and you've got your code ready, make sure that tests are passing and open your pull request.
 

docker-compose.dev-azure.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default

docker-compose.dev.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default

docker-compose.yml

@@ -40,9 +40,18 @@ services:
       LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
       LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
       LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090}
       LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
       LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}
+      LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse}
+      LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/}
+      LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto}
+      LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090}
+      LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true}
       LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
       LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
       REDIS_HOST: ${REDIS_HOST:-redis}

ee/package.json

@@ -28,7 +28,7 @@
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
-    "next": "^14.2.25",
+    "next": "^14.2.26",
     "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },

fern/apis/client/definition/score.yml

@@ -14,13 +14,15 @@ types:
   CreateScoreRequest:
     properties:
       id: optional<string>
-      traceId: string
+      traceId: optional<string>
+      sessionId: optional<string>
+      observationId: optional<string>
       name: string
       value:
         type: CreateScoreValue
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
-      observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       dataType:
         type: optional<ScoreDataType>
         docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
@@ -64,6 +66,10 @@ types:
           dataType: "BOOLEAN"
           configId: "1234-5678-90ab-cdef"
           traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "contextrelevant"
+          value: "not relevant"
+          sessionId: "abyt-1234-5678-80ab"
   BaseScore:
     properties:
       id: string
@@ -76,6 +82,7 @@ types:
       updatedAt: datetime
       authorUserId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       configId:
         type: optional<string>
         docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
@@ -103,6 +110,7 @@ types:
       stringValue:
         type: string
         docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  # Question: where is this used, if not needed remove BaseScore too
   Score:
     discriminant: "dataType"
     union:

fern/apis/organizations/definition/api.yml

@@ -0,0 +1,32 @@
+name: langfuse-organizations
+docs: |
+  ## General Notes
+  
+  This admin API is only available on self-hosted instances, not on Langfuse Cloud.
+  An administrator must set the `ADMIN_API_KEY` environment variable to use this API.
+  
+  ## Authentication
+
+  Authenticate with the API by setting the Authorization header to `Bearer $ADMIN_API_KEY` where
+  `$ADMIN_API_KEY` is the API key set via the container environment.
+  
+  ## Exports
+
+  - OpenAPI spec: https://cloud.langfuse.com/generated/organizations-api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/organizations-postman/collection.json
+  
+  ## Looking for the Langfuse API?
+  
+  Are you looking for the full Langfuse API?
+  Check out our [API reference](https://api.reference.langfuse.com/#overview).
+
+error-discrimination:
+  strategy: status-code
+auth: bearer
+imports:
+  commons: commons.yml
+errors:
+  - commons.Error
+  - commons.UnauthorizedError
+  - commons.AccessDeniedError
+  - commons.MethodNotAllowedError

fern/apis/organizations/definition/commons.yml

@@ -0,0 +1,13 @@
+errors:
+  Error:
+    status-code: 400
+    type: string
+  UnauthorizedError:
+    status-code: 401
+    type: string
+  AccessDeniedError:
+    status-code: 403
+    type: string
+  MethodNotAllowedError:
+    status-code: 405
+    type: string

fern/apis/organizations/definition/organizations.yml

@@ -0,0 +1,170 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/admin
+  endpoints:
+    getAll:
+      docs: Get all organizations
+      method: GET
+      path: /organizations
+      response: Organizations
+      errors:
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    create:
+      docs: Create a new organization
+      method: POST
+      path: /organizations
+      request:
+        name: CreateOrganizationRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the organization
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    getById:
+      docs: Get organization by ID
+      method: GET
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    update:
+      docs: Update an organization
+      method: PUT
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      request:
+        name: UpdateOrganizationRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the organization
+      response: Organization
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+    
+    delete:
+      docs: Delete an organization
+      method: DELETE
+      path: /organizations/{organizationId}
+      path-parameters:
+        organizationId: string
+      response: DeleteOrganizationResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    # API Key endpoints
+    getApiKeys:
+      docs: Get all API keys for an organization
+      method: GET
+      path: /organizations/{organizationId}/apiKeys
+      path-parameters:
+        organizationId: string
+      response: ApiKeyList
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    createApiKey:
+      docs: Create a new API key for an organization
+      method: POST
+      path: /organizations/{organizationId}/apiKeys
+      path-parameters:
+        organizationId: string
+      request:
+        name: CreateApiKeyRequest
+        body:
+          properties:
+            note:
+              type: optional<string>
+              docs: Optional note for the API key
+      response: ApiKeyResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+    deleteApiKey:
+      docs: Delete an API key for an organization
+      method: DELETE
+      path: /organizations/{organizationId}/apiKeys/{apiKeyId}
+      path-parameters:
+        organizationId: string
+        apiKeyId: string
+      response: DeleteOrganizationResponse
+      errors:
+        - commons.Error
+        - commons.UnauthorizedError
+        - commons.MethodNotAllowedError
+
+types:
+  Organizations:
+    docs: List of organizations
+    properties:
+      organizations: list<Organization>
+
+  Organization:
+    docs: Organization object
+    properties:
+      id: string
+      name: string
+      createdAt: datetime
+      metadata:
+        type: map<string, unknown>
+        docs: Metadata for the organization
+
+  DeleteOrganizationResponse:
+    docs: Response for successful organization deletion
+    properties:
+      success: boolean
+
+  ApiKeyList:
+    docs: List of API keys for an organization
+    properties:
+      apiKeys: list<ApiKeySummary>
+
+  ApiKeySummary:
+    docs: Summary of an API key
+    properties:
+      id: string
+      createdAt: datetime
+      expiresAt: optional<datetime>
+      lastUsedAt: optional<datetime>
+      note: optional<string>
+      publicKey: string
+      displaySecretKey: string
+
+  ApiKeyResponse:
+    docs: Response for API key creation
+    properties:
+      id: string
+      createdAt: datetime
+      publicKey: string
+      secretKey: string
+      displaySecretKey: string
+      note: optional<string>

fern/apis/organizations/generators.yml

@@ -0,0 +1,17 @@
+default-group: local
+groups:
+  local:
+    generators:
+      - name: fernapi/fern-openapi
+        version: 0.1.7
+        output:
+          location: local-file-system
+          path: ../../../web/public/generated/organizations-api
+        config:
+          namespaceExport: Langfuse
+          allowCustomFetcher: true
+      - name: fernapi/fern-postman
+        version: 0.0.45
+        output:
+          location: local-file-system
+          path: ../../../web/public/generated/organizations-postman

fern/apis/server/definition/commons.yml

@@ -75,7 +75,7 @@ types:
         type: list<ObservationsView>
         docs: List of observations
       scores:
-        type: list<Score>
+        type: list<ScoreV1>
         docs: List of scores
   Session:
     properties:
@@ -240,7 +240,7 @@ types:
     properties:
       value: double
       label: string
-  BaseScore:
+  BaseScoreV1:
     properties:
       id: string
       traceId: string
@@ -252,6 +252,67 @@ types:
       updatedAt: datetime
       authorUserId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
+      configId:
+        type: optional<string>
+        docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range
+      queueId:
+        type: optional<string>
+        docs: Reference an annotation queue on a score. Populated if the score was initially created in an annotation queue.
+      environment:
+        type: optional<string>
+        docs: The environment from which this score originated. Can be any lowercase alphanumeric string with hyphens and underscores that does not start with 'langfuse'.
+  NumericScoreV1:
+    extends: BaseScoreV1
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score
+  BooleanScoreV1:
+    extends: BaseScoreV1
+    properties:
+      value:
+        type: double
+        docs: The numeric value of the score. Equals 1 for "True" and 0 for "False"
+      stringValue:
+        type: string
+        docs: The string representation of the score value. Is inferred from the numeric value and equals "True" or "False"
+  CategoricalScoreV1:
+    extends: BaseScoreV1
+    properties:
+      value:
+        type: optional<double>
+        docs: Only defined if a config is linked. Represents the numeric category mapping of the stringValue
+      stringValue:
+        type: string
+        docs: The string representation of the score value. If no config is linked, can be any string. Otherwise, must map to a config category
+  ScoreV1:
+    discriminant: "dataType"
+    union:
+      NUMERIC:
+        type: NumericScoreV1
+        docs: "Score with NUMERIC data type"
+      CATEGORICAL:
+        type: CategoricalScoreV1
+        docs: "Score with CATEGORICAL data type"
+      BOOLEAN:
+        type: BooleanScoreV1
+        docs: "Score with BOOLEAN data type"
+
+  BaseScore:
+    properties:
+      id: string
+      traceId: optional<string>
+      sessionId: optional<string>
+      observationId: optional<string>
+      name: string
+      source: ScoreSource
+      timestamp: datetime
+      createdAt: datetime
+      updatedAt: datetime
+      authorUserId: optional<string>
+      comment: optional<string>
+      metadata: optional<unknown>
       configId:
         type: optional<string>
         docs: Reference a score config on a score. When set, config and score name must be equal and value must comply to optionally defined numerical range

fern/apis/server/definition/ingestion.yml

@@ -263,14 +263,16 @@ types:
   ScoreBody:
     properties:
       id: optional<string>
-      traceId: string
+      traceId: optional<string>
+      sessionId: optional<string>
+      observationId: optional<string>
       name: string
       environment: optional<string>
       value:
         type: commons.CreateScoreValue
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
-      observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       dataType:
         type: optional<commons.ScoreDataType>
         docs: When set, must match the score value's type. If not set, will be inferred from the score value or config
@@ -314,6 +316,10 @@ types:
           dataType: "BOOLEAN"
           configId: "1234-5678-90ab-cdef"
           traceId: "cdef-1234-5678-90ab"
+      - value:
+          name: "contextrelevant"
+          value: "not relevant"
+          sessionId: "abyt-1234-5678-80ab"
 
   BaseEvent:
     properties:

fern/apis/server/definition/memberships.yml

@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    getOrganizationMemberships:
+      docs: Get all memberships for the organization associated with the API key (requires organization-scoped API key)
+      method: GET
+      path: /organizations/memberships
+      response: MembershipsResponse
+
+    updateOrganizationMembership:
+      docs: Create or update a membership for the organization associated with the API key (requires organization-scoped API key)
+      method: PUT
+      path: /organizations/memberships
+      request: MembershipRequest
+      response: MembershipResponse
+
+    getProjectMemberships:
+      docs: Get all memberships for a specific project (requires organization-scoped API key)
+      method: GET
+      path: /projects/{projectId}/memberships
+      path-parameters:
+        projectId: string
+      response: MembershipsResponse
+
+    updateProjectMembership:
+      docs: Create or update a membership for a specific project (requires organization-scoped API key). The user must already be a member of the organization.
+      method: PUT
+      path: /projects/{projectId}/memberships
+      path-parameters:
+        projectId: string
+      request: MembershipRequest
+      response: MembershipResponse
+
+types:
+  MembershipRole:
+    enum:
+      - OWNER
+      - ADMIN
+      - MEMBER
+      - VIEWER
+
+  MembershipRequest:
+    properties:
+      userId: string
+      role: MembershipRole
+
+  MembershipResponse:
+    properties:
+      userId: string
+      role: MembershipRole
+      email: string
+      name: string
+
+  MembershipsResponse:
+    properties:
+      memberships: list<MembershipResponse>

fern/apis/server/definition/projects.yml

@@ -11,6 +11,85 @@ service:
       method: GET
       path: /projects
       response: Projects
+    
+    create:
+      docs: Create a new project (requires organization-scoped API key)
+      method: POST
+      path: /projects
+      request:
+        name: CreateProjectRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the project
+            retention:
+              type: integer
+              docs: Number of days to retain data. Must be 0 or at least 7 days. Requires data-retention entitlement for non-zero values. Optional.
+      response: Project
+    
+    update:
+      docs: Update a project by ID (requires organization-scoped API key).
+      method: PUT
+      path: /projects/{projectId}
+      path-parameters:
+        projectId: string
+      request:
+        name: UpdateProjectRequest
+        body:
+          properties:
+            name: string
+            metadata:
+              type: optional<map<string, unknown>>
+              docs: Optional metadata for the project
+            retention:
+              type: integer
+              docs: Number of days to retain data. Must be 0 or at least 7 days. Requires data-retention entitlement for non-zero values. Optional.
+      response: Project
+    
+    delete:
+      docs: Delete a project by ID (requires organization-scoped API key). Project deletion is processed asynchronously.
+      method: DELETE
+      path: /projects/{projectId}
+      path-parameters:
+        projectId: string
+      response:
+        status-code: 202
+        type: ProjectDeletionResponse
+        
+    # API Key endpoints
+    getApiKeys:
+      docs: Get all API keys for a project (requires organization-scoped API key)
+      method: GET
+      path: /projects/{projectId}/apiKeys
+      path-parameters:
+        projectId: string
+      response: ApiKeyList
+
+    createApiKey:
+      docs: Create a new API key for a project (requires organization-scoped API key)
+      method: POST
+      path: /projects/{projectId}/apiKeys
+      path-parameters:
+        projectId: string
+      request:
+        name: CreateApiKeyRequest
+        body:
+          properties:
+            note:
+              type: optional<string>
+              docs: Optional note for the API key
+      response: ApiKeyResponse
+
+    deleteApiKey:
+      docs: Delete an API key for a project (requires organization-scoped API key)
+      method: DELETE
+      path: /projects/{projectId}/apiKeys/{apiKeyId}
+      path-parameters:
+        projectId: string
+        apiKeyId: string
+      response: ApiKeyDeletionResponse
 
 types:
   Projects:
@@ -21,3 +100,45 @@ types:
     properties:
       id: string
       name: string
+      metadata:
+        type: map<string, unknown>
+        docs: Metadata for the project
+      retentionDays:
+        type: integer
+        docs: Number of days to retain data. Null or 0 means no retention. Omitted if no retention is configured.
+
+  ProjectDeletionResponse:
+    properties:
+      success: boolean
+      message: string
+      
+  ApiKeyList:
+    docs: List of API keys for a project
+    properties:
+      apiKeys: list<ApiKeySummary>
+
+  ApiKeySummary:
+    docs: Summary of an API key
+    properties:
+      id: string
+      createdAt: datetime
+      expiresAt: optional<datetime>
+      lastUsedAt: optional<datetime>
+      note: optional<string>
+      publicKey: string
+      displaySecretKey: string
+
+  ApiKeyResponse:
+    docs: Response for API key creation
+    properties:
+      id: string
+      createdAt: datetime
+      publicKey: string
+      secretKey: string
+      displaySecretKey: string
+      note: optional<string>
+      
+  ApiKeyDeletionResponse:
+    docs: Response for API key deletion
+    properties:
+      success: boolean

fern/apis/server/definition/scim.yml

@@ -0,0 +1,199 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/public/scim
+  endpoints:
+    getServiceProviderConfig:
+      docs: Get SCIM Service Provider Configuration (requires organization-scoped API key)
+      method: GET
+      path: /ServiceProviderConfig
+      response: ServiceProviderConfig
+
+    getResourceTypes:
+      docs: Get SCIM Resource Types (requires organization-scoped API key)
+      method: GET
+      path: /ResourceTypes
+      response: ResourceTypesResponse
+
+    getSchemas:
+      docs: Get SCIM Schemas (requires organization-scoped API key)
+      method: GET
+      path: /Schemas
+      response: SchemasResponse
+
+    listUsers:
+      docs: List users in the organization (requires organization-scoped API key)
+      method: GET
+      path: /Users
+      request:
+        name: ListUsersRequest
+        query-parameters:
+          filter:
+            docs: Filter expression (e.g. userName eq "value")
+            type: optional<string>
+          startIndex:
+            docs: 1-based index of the first result to return (default 1)
+            type: optional<integer>
+          count:
+            docs: Maximum number of results to return (default 100)
+            type: optional<integer>
+      response: ScimUsersListResponse
+
+    createUser:
+      docs: Create a new user in the organization (requires organization-scoped API key)
+      method: POST
+      path: /Users
+      request:
+        name: CreateUserRequest
+        body:
+          properties:
+            userName:
+              docs: User's email address (required)
+              type: string
+            name:
+              docs: User's name information
+              type: ScimName
+            emails:
+              docs: User's email addresses
+              type: optional<list<ScimEmail>>
+            active:
+              docs: Whether the user is active
+              type: optional<boolean>
+            password:
+              docs: Initial password for the user
+              type: optional<string>
+      response: ScimUser
+
+    getUser:
+      docs: Get a specific user by ID (requires organization-scoped API key)
+      method: GET
+      path: /Users/{userId}
+      path-parameters:
+        userId: string
+      response: ScimUser
+
+    deleteUser:
+      docs: Remove a user from the organization (requires organization-scoped API key). Note that this only removes the user from the organization but does not delete the user entity itself.
+      method: DELETE
+      path: /Users/{userId}
+      path-parameters:
+        userId: string
+      response: EmptyResponse
+
+types:
+  ServiceProviderConfig:
+    properties:
+      schemas: list<string>
+      documentationUri: string
+      patch: ScimFeatureSupport
+      bulk: BulkConfig
+      filter: FilterConfig
+      changePassword: ScimFeatureSupport
+      sort: ScimFeatureSupport
+      etag: ScimFeatureSupport
+      authenticationSchemes: list<AuthenticationScheme>
+      meta: ResourceMeta
+
+  ScimFeatureSupport:
+    properties:
+      supported: boolean
+
+  BulkConfig:
+    properties:
+      supported: boolean
+      maxOperations: integer
+      maxPayloadSize: integer
+
+  FilterConfig:
+    properties:
+      supported: boolean
+      maxResults: integer
+
+  ResourceMeta:
+    properties:
+      resourceType: string
+      location: string
+
+  AuthenticationScheme:
+    properties:
+      name: string
+      description: string
+      specUri: string
+      type: string
+      primary: boolean
+
+  ResourceTypesResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      Resources: list<ResourceType>
+
+  ResourceType:
+    properties:
+      schemas: optional<list<string>>
+      id: string
+      name: string
+      endpoint: string
+      description: string
+      schema: string
+      schemaExtensions: list<SchemaExtension>
+      meta: ResourceMeta
+
+  SchemaExtension:
+    properties:
+      schema: string
+      required: boolean
+
+  SchemasResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      Resources: list<SchemaResource>
+
+  SchemaResource:
+    properties:
+      id: string
+      name: string
+      description: string
+      attributes: list<unknown>
+      meta: ResourceMeta
+
+  ScimUsersListResponse:
+    properties:
+      schemas: list<string>
+      totalResults: integer
+      startIndex: integer
+      itemsPerPage: integer
+      Resources: list<ScimUser>
+
+  ScimUser:
+    properties:
+      schemas: list<string>
+      id: string
+      userName: string
+      name: ScimName
+      emails: list<ScimEmail>
+      meta: UserMeta
+
+  UserMeta:
+    properties:
+      resourceType: string
+      created: optional<string>
+      lastModified: optional<string>
+
+  ScimName:
+    properties:
+      formatted: optional<string>
+
+  ScimEmail:
+    properties:
+      primary: boolean
+      value: string
+      type: string
+
+  EmptyResponse:
+    docs: Empty response for 204 No Content responses
+    properties: {}

fern/apis/server/definition/score-v2.yml

@@ -0,0 +1,112 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public/v2
+  endpoints:
+    get:
+      docs: Get a list of scores (supports both trace and session scores)
+      method: GET
+      path: /scores
+      request:
+        name: GetScoresRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
+          userId:
+            type: optional<string>
+            docs: Retrieve only scores with this userId associated to the trace.
+          name:
+            type: optional<string>
+            docs: Retrieve only scores with this name.
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include scores created on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include scores created before a certain datetime (ISO 8601)
+          environment:
+            type: optional<string>
+            allow-multiple: true
+            docs: Optional filter for scores where the environment is one of the provided values.
+          source:
+            type: optional<commons.ScoreSource>
+            docs: Retrieve only scores from a specific source.
+          operator:
+            type: optional<string>
+            docs: Retrieve only scores with <operator> value.
+          value:
+            type: optional<double>
+            docs: Retrieve only scores with <operator> value.
+          scoreIds:
+            type: optional<string>
+            docs: Comma-separated list of score IDs to limit the results to.
+          configId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific configId.
+          queueId:
+            type: optional<string>
+            docs: Retrieve only scores with a specific annotation queueId.
+          dataType:
+            type: optional<commons.ScoreDataType>
+            docs: Retrieve only scores with a specific dataType.
+          traceTags:
+            type: optional<string>
+            allow-multiple: true
+            docs: Only scores linked to traces that include all of these tags will be returned.
+      response: GetScoresResponse
+    get-by-id:
+      docs: Get a score (supports both trace and session scores)
+      method: GET
+      path: /scores/{scoreId}
+      path-parameters:
+        scoreId:
+          type: string
+          docs: The unique langfuse identifier of a score
+      response: commons.Score
+
+types:
+  GetScoresResponseTraceData:
+    properties:
+      userId:
+        type: optional<string>
+        docs: The user ID associated with the trace referenced by score
+      tags:
+        type: optional<list<string>>
+        docs: A list of tags associated with the trace referenced by score
+      environment:
+        type: optional<string>
+        docs: The environment of the trace referenced by score
+
+  GetScoresResponseDataNumeric:
+    extends: commons.NumericScore
+    properties:
+      trace: optional<GetScoresResponseTraceData>
+
+  GetScoresResponseDataCategorical:
+    extends: commons.CategoricalScore
+    properties:
+      trace: optional<GetScoresResponseTraceData>
+
+  GetScoresResponseDataBoolean:
+    extends: commons.BooleanScore
+    properties:
+      trace: optional<GetScoresResponseTraceData>
+
+  GetScoresResponseData:
+    discriminant: dataType
+    union:
+      NUMERIC: GetScoresResponseDataNumeric
+      CATEGORICAL: GetScoresResponseDataCategorical
+      BOOLEAN: GetScoresResponseDataBoolean
+
+  GetScoresResponse:
+    properties:
+      data: list<GetScoresResponseData>
+      meta: pagination.MetaResponse

fern/apis/server/definition/score.yml

@@ -7,77 +7,13 @@ service:
   base-path: /api/public
   endpoints:
     create:
-      docs: Create a score
+      docs: Create a score (supports both trace and session scores)
       method: POST
       path: /scores
       request: CreateScoreRequest
       response: CreateScoreResponse
-    get:
-      docs: Get a list of scores
-      method: GET
-      path: /scores
-      request:
-        name: GetScoresRequest
-        query-parameters:
-          page:
-            type: optional<integer>
-            docs: Page number, starts at 1.
-          limit:
-            type: optional<integer>
-            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
-          userId:
-            type: optional<string>
-            docs: Retrieve only scores with this userId associated to the trace.
-          name:
-            type: optional<string>
-            docs: Retrieve only scores with this name.
-          fromTimestamp:
-            type: optional<datetime>
-            docs: Optional filter to only include scores created on or after a certain datetime (ISO 8601)
-          toTimestamp:
-            type: optional<datetime>
-            docs: Optional filter to only include scores created before a certain datetime (ISO 8601)
-          environment:
-            type: optional<string>
-            allow-multiple: true
-            docs: Optional filter for scores where the environment is one of the provided values.
-          source:
-            type: optional<commons.ScoreSource>
-            docs: Retrieve only scores from a specific source.
-          operator:
-            type: optional<string>
-            docs: Retrieve only scores with <operator> value.
-          value:
-            type: optional<double>
-            docs: Retrieve only scores with <operator> value.
-          scoreIds:
-            type: optional<string>
-            docs: Comma-separated list of score IDs to limit the results to.
-          configId:
-            type: optional<string>
-            docs: Retrieve only scores with a specific configId.
-          queueId:
-            type: optional<string>
-            docs: Retrieve only scores with a specific annotation queueId.
-          dataType:
-            type: optional<commons.ScoreDataType>
-            docs: Retrieve only scores with a specific dataType.
-          traceTags:
-            type: optional<string>
-            allow-multiple: true
-            docs: Only scores linked to traces that include all of these tags will be returned.
-      response: GetScoresResponse
-    get-by-id:
-      docs: Get a score
-      method: GET
-      path: /scores/{scoreId}
-      path-parameters:
-        scoreId:
-          type: string
-          docs: The unique langfuse identifier of a score
-      response: commons.Score
     delete:
-      docs: Delete a score
+      docs: Delete a score (supports both trace and session scores)
       method: DELETE
       path: /scores/{scoreId}
       path-parameters:
@@ -88,13 +24,15 @@ types:
   CreateScoreRequest:
     properties:
       id: optional<string>
-      traceId: string
+      traceId: optional<string>
+      sessionId: optional<string>
+      observationId: optional<string>
       name: string
       value:
         type: commons.CreateScoreValue
         docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. Boolean score values must equal either 1 or 0 (true or false)
-      observationId: optional<string>
       comment: optional<string>
+      metadata: optional<unknown>
       environment:
         type: optional<string>
         docs: The environment of the score. Can be any lowercase alphanumeric string with hyphens and underscores that does not start with 'langfuse'.
@@ -148,41 +86,3 @@ types:
       id:
         type: string
         docs: The id of the created object in Langfuse
-  GetScoresResponseTraceData:
-    properties:
-      userId:
-        type: optional<string>
-        docs: The user ID associated with the trace referenced by score
-      tags:
-        type: optional<list<string>>
-        docs: A list of tags associated with the trace referenced by score
-      environment:
-        type: optional<string>
-        docs: The environment of the trace referenced by score
-
-  GetScoresResponseDataNumeric:
-    extends: commons.NumericScore
-    properties:
-      trace: GetScoresResponseTraceData
-
-  GetScoresResponseDataCategorical:
-    extends: commons.CategoricalScore
-    properties:
-      trace: GetScoresResponseTraceData
-
-  GetScoresResponseDataBoolean:
-    extends: commons.BooleanScore
-    properties:
-      trace: GetScoresResponseTraceData
-
-  GetScoresResponseData:
-    discriminant: dataType
-    union:
-      NUMERIC: GetScoresResponseDataNumeric
-      CATEGORICAL: GetScoresResponseDataCategorical
-      BOOLEAN: GetScoresResponseDataBoolean
-
-  GetScoresResponse:
-    properties:
-      data: list<GetScoresResponseData>
-      meta: pagination.MetaResponse

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.45.2",
+  "version": "3.53.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -23,6 +23,8 @@
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
+    "dev:worker": "turbo run dev --filter=worker",
+    "dev:web": "turbo run dev --filter=web",
     "lint": "turbo run lint",
     "test": "turbo run test",
     "release": "dotenv -e ../.env -- release-it",

packages/shared/clickhouse/migrations/clustered/0010_add_metadata_column_on_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS metadata;

packages/shared/clickhouse/migrations/clustered/0010_add_metadata_column_on_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default ADD COLUMN metadata Map(LowCardinality(String), String) AFTER comment;

packages/shared/clickhouse/migrations/clustered/0011_add_blob_storage_file_log.down.sql

@@ -0,0 +1 @@
+DROP TABLE blob_storage_file_log ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0011_add_blob_storage_file_log.up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE blob_storage_file_log ON CLUSTER default (
+    `id`          String,
+    `project_id`  String,
+    `entity_type` String,
+    `entity_id`   String,
+    `event_id`    String,
+
+    `bucket_name` String,
+    `bucket_path` String,
+
+    `created_at`  DateTime64(3) DEFAULT now(),
+    `updated_at`  DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted)
+      ORDER BY (
+                project_id,
+                entity_type,
+                entity_id,
+                event_id
+          );

packages/shared/clickhouse/migrations/clustered/0012_add_session_id_column_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS session_id SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0012_add_session_id_column_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default ADD COLUMN session_id Nullable(String) AFTER trace_id SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0013_drop_scores_trace_id_index.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0013_drop_scores_trace_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;
+

packages/shared/clickhouse/migrations/clustered/0014_scores_modify_nullable_trace_id_column.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default MODIFY COLUMN trace_id Nullable(String) SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0014_scores_modify_nullable_trace_id_column.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default MODIFY COLUMN trace_id Nullable(String) SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0015_add_scores_trace_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0015_add_scores_trace_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0016_add_scores_session_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_project_session SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/clustered/0016_add_scores_session_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_project_session SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0010_add_metadata_column_on_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores DROP COLUMN IF EXISTS metadata;

packages/shared/clickhouse/migrations/unclustered/0010_add_metadata_column_on_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ADD COLUMN metadata Map(LowCardinality(String), String) AFTER comment;

packages/shared/clickhouse/migrations/unclustered/0011_add_blob_storage_file_log.down.sql

@@ -0,0 +1 @@
+DROP TABLE blob_storage_file_log;

packages/shared/clickhouse/migrations/unclustered/0011_add_blob_storage_file_log.up.sql

@@ -0,0 +1,22 @@
+CREATE TABLE blob_storage_file_log
+(
+    `id`          String,
+    `project_id`  String,
+    `entity_type` String,
+    `entity_id`   String,
+    `event_id`    String,
+
+    `bucket_name` String,
+    `bucket_path` String,
+
+    `created_at`  DateTime64(3) DEFAULT now(),
+    `updated_at`  DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+      ORDER BY (
+                project_id,
+                entity_type,
+                entity_id,
+                event_id
+          );

packages/shared/clickhouse/migrations/unclustered/0012_add_session_id_column_scores.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores DROP COLUMN IF EXISTS session_id SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0012_add_session_id_column_scores.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores ADD COLUMN session_id Nullable(String) AFTER trace_id SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0013_drop_scores_trace_id_index.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0013_drop_scores_trace_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores DROP INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;
+

packages/shared/clickhouse/migrations/unclustered/0014_scores_modify_nullable_trace_id_column.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores MODIFY COLUMN trace_id Nullable(String) SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0014_scores_modify_nullable_trace_id_column.up.sql

@@ -0,0 +1 @@
+ALTER TABLE scores MODIFY COLUMN trace_id Nullable(String) SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0015_add_scores_trace_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores DROP INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0015_add_scores_trace_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ADD INDEX IF NOT EXISTS idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2; 
+ALTER TABLE scores MATERIALIZE INDEX IF EXISTS idx_project_trace_observation SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0016_add_scores_session_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE scores DROP INDEX IF EXISTS idx_project_session SETTINGS mutations_sync = 2;

packages/shared/clickhouse/migrations/unclustered/0016_add_scores_session_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE scores ADD INDEX IF NOT EXISTS idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1 SETTINGS mutations_sync = 2;
+ALTER TABLE scores MATERIALIZE INDEX IF EXISTS idx_project_session SETTINGS mutations_sync = 2;

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 64",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 65",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -61,6 +61,7 @@
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
+    "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.12",
     "@langchain/aws": "^0.1.3",
     "@langchain/core": "^0.3.37",

packages/shared/prisma/generated/types.ts

@@ -4,6 +4,11 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
+export const ApiKeyScope = {
+    ORGANIZATION: "ORGANIZATION",
+    PROJECT: "PROJECT"
+} as const;
+export type ApiKeyScope = (typeof ApiKeyScope)[keyof typeof ApiKeyScope];
 export const Role = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
@@ -80,12 +85,33 @@ export const JobExecutionStatus = {
     CANCELLED: "CANCELLED"
 } as const;
 export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
+export const BlobStorageIntegrationFileType = {
+    JSON: "JSON",
+    CSV: "CSV",
+    JSONL: "JSONL"
+} as const;
+export type BlobStorageIntegrationFileType = (typeof BlobStorageIntegrationFileType)[keyof typeof BlobStorageIntegrationFileType];
 export const BlobStorageIntegrationType = {
     S3: "S3",
     S3_COMPATIBLE: "S3_COMPATIBLE",
     AZURE_BLOB_STORAGE: "AZURE_BLOB_STORAGE"
 } as const;
 export type BlobStorageIntegrationType = (typeof BlobStorageIntegrationType)[keyof typeof BlobStorageIntegrationType];
+export const DashboardWidgetViews = {
+    TRACES: "TRACES",
+    OBSERVATIONS: "OBSERVATIONS",
+    SCORES_NUMERIC: "SCORES_NUMERIC",
+    SCORES_CATEGORICAL: "SCORES_CATEGORICAL"
+} as const;
+export type DashboardWidgetViews = (typeof DashboardWidgetViews)[keyof typeof DashboardWidgetViews];
+export const DashboardWidgetChartType = {
+    LINE_TIME_SERIES: "LINE_TIME_SERIES",
+    BAR_TIME_SERIES: "BAR_TIME_SERIES",
+    HORIZONTAL_BAR: "HORIZONTAL_BAR",
+    VERTICAL_BAR: "VERTICAL_BAR",
+    PIE: "PIE"
+} as const;
+export type DashboardWidgetChartType = (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export type Account = {
     id: string;
     user_id: string;
@@ -137,7 +163,9 @@ export type ApiKey = {
     display_secret_key: string;
     last_used_at: Timestamp | null;
     expires_at: Timestamp | null;
-    project_id: string;
+    project_id: string | null;
+    organization_id: string | null;
+    scope: Generated<ApiKeyScope>;
 };
 export type AuditLog = {
     id: string;
@@ -208,6 +236,7 @@ export type BlobStorageIntegration = {
     last_sync_at: Timestamp | null;
     enabled: boolean;
     export_frequency: string;
+    file_type: Generated<BlobStorageIntegrationFileType>;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
@@ -227,6 +256,33 @@ export type CronJobs = {
     job_started_at: Timestamp | null;
     state: string | null;
 };
+export type Dashboard = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    created_by: string | null;
+    updated_by: string | null;
+    project_id: string | null;
+    name: string;
+    description: string;
+    definition: unknown;
+};
+export type DashboardWidget = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    created_by: string | null;
+    updated_by: string | null;
+    project_id: string | null;
+    name: string;
+    description: string;
+    view: DashboardWidgetViews;
+    dimensions: unknown;
+    metrics: unknown;
+    filters: unknown;
+    chart_type: DashboardWidgetChartType;
+    chart_config: unknown;
+};
 export type Dataset = {
     id: string;
     project_id: string;
@@ -401,6 +457,24 @@ export type LlmApiKeys = {
     config: unknown | null;
     project_id: string;
 };
+export type LlmSchema = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    schema: unknown;
+};
+export type LlmTool = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    description: string;
+    parameters: unknown;
+};
 export type Media = {
     id: string;
     sha_256_hash: string;
@@ -457,6 +531,7 @@ export type Organization = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
     cloud_config: unknown | null;
+    metadata: unknown | null;
 };
 export type OrganizationMembership = {
     id: string;
@@ -490,6 +565,7 @@ export type Project = {
     deleted_at: Timestamp | null;
     name: string;
     retention_days: number | null;
+    metadata: unknown | null;
 };
 export type ProjectMembership = {
     org_membership_id: string;
@@ -525,6 +601,13 @@ export type PromptDependency = {
     child_label: string | null;
     child_version: number | null;
 };
+export type PromptProtectedLabels = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    label: string;
+};
 export type ScoreConfig = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -598,6 +681,8 @@ export type DB = {
     blob_storage_integrations: BlobStorageIntegration;
     comments: Comment;
     cron_jobs: CronJobs;
+    dashboard_widgets: DashboardWidget;
+    dashboards: Dashboard;
     dataset_items: DatasetItem;
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
@@ -606,6 +691,8 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    llm_schemas: LlmSchema;
+    llm_tools: LlmTool;
     media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
@@ -618,6 +705,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompt_dependencies: PromptDependency;
+    prompt_protected_labels: PromptProtectedLabels;
     prompts: Prompt;
     score_configs: ScoreConfig;
     scores: LegacyPrismaScore;

packages/shared/prisma/migrations/20250326180640_add_llm_tools_and_schemas_tables/migration.sql

@@ -0,0 +1,37 @@
+-- CreateTable
+CREATE TABLE "llm_schemas" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "schema" JSON NOT NULL,
+
+    CONSTRAINT "llm_schemas_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "llm_tools" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "parameters" JSON NOT NULL,
+
+    CONSTRAINT "llm_tools_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_schemas_project_id_name_key" ON "llm_schemas"("project_id", "name");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_tools_project_id_name_key" ON "llm_tools"("project_id", "name");
+
+-- AddForeignKey
+ALTER TABLE "llm_schemas" ADD CONSTRAINT "llm_schemas_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "llm_tools" ADD CONSTRAINT "llm_tools_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250401122159_add_prompt_protected_labels_table/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "prompt_protected_labels" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "label" TEXT NOT NULL,
+
+    CONSTRAINT "prompt_protected_labels_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "prompt_protected_labels_project_id_label_key" ON "prompt_protected_labels"("project_id", "label");
+
+-- AddForeignKey
+ALTER TABLE "prompt_protected_labels" ADD CONSTRAINT "prompt_protected_labels_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250402142320_add_blobstorage_integration_file_type/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "BlobStorageIntegrationFileType" AS ENUM ('JSON', 'CSV', 'JSONL');
+
+-- AlterTable
+ALTER TABLE "blob_storage_integrations" ADD COLUMN     "file_type" "BlobStorageIntegrationFileType" NOT NULL DEFAULT 'CSV';

packages/shared/prisma/migrations/20250403153555_membership_invitations_no_duplicates/migration.sql

@@ -0,0 +1,17 @@
+-- Delete duplicate membership invitations, keeping only the newest one for each email and org_id combination
+WITH ranked_invitations AS (
+  SELECT 
+    id,
+    email,
+    org_id,
+    ROW_NUMBER() OVER (PARTITION BY email, org_id ORDER BY updated_at DESC) as rn
+  FROM membership_invitations
+)
+DELETE FROM membership_invitations
+WHERE id IN (
+  SELECT id FROM ranked_invitations WHERE rn > 1
+);
+
+
+-- CreateIndex
+CREATE UNIQUE INDEX "membership_invitations_email_org_id_key" ON "membership_invitations"("email", "org_id");

packages/shared/prisma/migrations/20250409154352_add_dashboard_data_model/migration.sql

@@ -0,0 +1,58 @@
+-- CreateEnum
+CREATE TYPE "DashboardWidgetViews" AS ENUM ('TRACES', 'OBSERVATIONS', 'SCORES_NUMERIC', 'SCORES_CATEGORICAL');
+
+-- CreateEnum
+CREATE TYPE "DashboardWidgetChartType" AS ENUM ('LINE_TIME_SERIES', 'BAR_TIME_SERIES', 'HORIZONTAL_BAR', 'VERTICAL_BAR', 'PIE');
+
+-- CreateTable
+CREATE TABLE "dashboards" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_by" TEXT,
+    "project_id" TEXT,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "definition" JSONB NOT NULL,
+
+    CONSTRAINT "dashboards_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "dashboard_widgets" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_by" TEXT,
+    "project_id" TEXT,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "view" "DashboardWidgetViews" NOT NULL,
+    "dimensions" JSONB NOT NULL,
+    "metrics" JSONB NOT NULL,
+    "filters" JSONB NOT NULL,
+    "chart_type" "DashboardWidgetChartType" NOT NULL,
+    "chart_config" JSONB NOT NULL,
+
+    CONSTRAINT "dashboard_widgets_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboards" ADD CONSTRAINT "dashboards_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "dashboard_widgets" ADD CONSTRAINT "dashboard_widgets_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250410145712_add_organization_scoped_api_keys/migration.sql

@@ -0,0 +1,13 @@
+-- CreateEnum
+CREATE TYPE "ApiKeyScope" AS ENUM ('ORGANIZATION', 'PROJECT');
+
+-- AlterTable
+ALTER TABLE "api_keys" ADD COLUMN     "organization_id" TEXT,
+ADD COLUMN     "scope" "ApiKeyScope" NOT NULL DEFAULT 'PROJECT',
+ALTER COLUMN "project_id" DROP NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "api_keys_organization_id_idx" ON "api_keys"("organization_id");
+
+-- AddForeignKey
+ALTER TABLE "api_keys" ADD CONSTRAINT "api_keys_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250420120553_add_organization_and_project_metadata/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "organizations" ADD COLUMN     "metadata" JSONB;
+
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN     "metadata" JSONB;

packages/shared/prisma/migrations/20250517173700_add_event_log_migration_background_migration.sql/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('c19b91d9-f9a2-468b-8209-95578f970c5b', '20250417_1737_migrate_event_log_to_blob_storage', 'migrateEventLogToBlobStorageRefTable', '{}');

packages/shared/prisma/schema.prisma

@@ -82,6 +82,10 @@ model User {
   featureFlags            String[]                 @default([]) @map("feature_flags")
   annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
   annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
+  dashboardWidgetsCreated DashboardWidget[]        @relation("CreatedByUser")
+  dashboardWidgetsUpdated DashboardWidget[]        @relation("UpdatedByUser")
+  dashboardCreated        Dashboard[]              @relation("CreatedByUser")
+  dashboardUpdated        Dashboard[]              @relation("UpdatedByUser")
 
   @@map("users")
 }
@@ -101,9 +105,11 @@ model Organization {
   createdAt               DateTime                 @default(now()) @map("created_at")
   updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
   cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+  metadata                Json?
   organizationMemberships OrganizationMembership[]
   projects                Project[]
   MembershipInvitation    MembershipInvitation[]
+  ApiKey                  ApiKey[]
 
   @@map("organizations")
 }
@@ -116,6 +122,7 @@ model Project {
   deletedAt              DateTime?                 @map("deleted_at")
   name                   String
   retentionDays          Int?                      @map("retention_days")
+  metadata               Json?
   projectMembers         ProjectMembership[]
   organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
   apiKeys                ApiKey[]
@@ -142,24 +149,38 @@ model Project {
   LegacyObservation      LegacyPrismaObservation[]
   LegacyScore            LegacyPrismaScore[]
   PromptDependency       PromptDependency[]
+  LlmSchema              LlmSchema[]
+  LlmTool                LlmTool[]
+  PromptProtectedLabels  PromptProtectedLabels[]
+  Dashboard              Dashboard[]
+  DashboardWidget        DashboardWidget[]
 
   @@index([orgId])
   @@map("projects")
 }
 
-model ApiKey {
-  id                  String    @id @unique @default(cuid())
-  createdAt           DateTime  @default(now()) @map("created_at")
-  note                String?
-  publicKey           String    @unique @map("public_key")
-  hashedSecretKey     String    @unique @map("hashed_secret_key")
-  fastHashedSecretKey String?   @unique @map("fast_hashed_secret_key")
-  displaySecretKey    String    @map("display_secret_key")
-  lastUsedAt          DateTime? @map("last_used_at")
-  expiresAt           DateTime? @map("expires_at")
-  projectId           String    @map("project_id")
-  project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+enum ApiKeyScope {
+  ORGANIZATION
+  PROJECT
+}
 
+model ApiKey {
+  id                  String        @id @unique @default(cuid())
+  createdAt           DateTime      @default(now()) @map("created_at")
+  note                String?
+  publicKey           String        @unique @map("public_key")
+  hashedSecretKey     String        @unique @map("hashed_secret_key")
+  fastHashedSecretKey String?       @unique @map("fast_hashed_secret_key")
+  displaySecretKey    String        @map("display_secret_key")
+  lastUsedAt          DateTime?     @map("last_used_at")
+  expiresAt           DateTime?     @map("expires_at")
+  projectId           String?       @map("project_id")
+  project             Project?      @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  orgId               String?       @map("organization_id")
+  organization        Organization? @relation(fields: [orgId], references: [id], onDelete: Cascade)
+  scope               ApiKeyScope   @default(PROJECT) @map("scope")
+
+  @@index(orgId)
   @@index(projectId)
   @@index(publicKey)
   @@index(hashedSecretKey)
@@ -254,6 +275,7 @@ model MembershipInvitation {
   createdAt       DateTime     @default(now()) @map("created_at")
   updatedAt       DateTime     @default(now()) @updatedAt @map("updated_at")
 
+  @@unique([email, orgId]) // do not include projectId as this leads to issues when processing the invites, needs new logic
   @@index([projectId])
   @@index([orgId])
   @@index([email])
@@ -681,6 +703,18 @@ model PromptDependency {
   @@map("prompt_dependencies")
 }
 
+model PromptProtectedLabels {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  projectId String   @map("project_id")
+  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  label     String
+
+  @@unique([projectId, label])
+  @@map("prompt_protected_labels")
+}
+
 // Update ObservationView below when making changes to this model!
 model Model {
   id        String   @id @default(cuid())
@@ -859,6 +893,7 @@ model SsoConfig {
   authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
 
   authConfig Json? @map("auth_config")
+
   // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
   // secrets like clientSecret are encrypted on the application level
   @@map("sso_configs")
@@ -891,10 +926,11 @@ model BlobStorageIntegration {
   forcePathStyle  Boolean                    @map("force_path_style")
 
   // Integration Config
-  nextSyncAt      DateTime? @map("next_sync_at")
-  lastSyncAt      DateTime? @map("last_sync_at")
+  nextSyncAt      DateTime?                      @map("next_sync_at")
+  lastSyncAt      DateTime?                      @map("last_sync_at")
   enabled         Boolean
-  exportFrequency String    @map("export_frequency")
+  exportFrequency String                         @map("export_frequency")
+  fileType        BlobStorageIntegrationFileType @default(CSV) @map("file_type")
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
@@ -902,6 +938,14 @@ model BlobStorageIntegration {
   @@map("blob_storage_integrations")
 }
 
+enum BlobStorageIntegrationFileType {
+  JSON
+  CSV
+  JSONL
+
+  @@map("BlobStorageIntegrationFileType")
+}
+
 enum BlobStorageIntegrationType {
   S3
   S3_COMPATIBLE
@@ -1009,3 +1053,100 @@ model BillingMeterBackup {
   @@index([stripeCustomerId, meterId, startTime, endTime])
   @@map("billing_meter_backups")
 }
+
+model LlmSchema {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  schema      Json   @map("schema") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_schemas")
+}
+
+model LlmTool {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+  parameters  Json   @map("parameters") @db.Json
+
+  @@unique([projectId, name])
+  @@map("llm_tools")
+}
+
+model Dashboard {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  createdBy     String? @map("created_by")
+  createdByUser User?   @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+  updatedBy     String? @map("updated_by")
+  updatedByUser User?   @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+  // Optional to account for Langfuse-managed dashboards
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+
+  definition Json @map("definition")
+
+  @@map("dashboards")
+}
+
+enum DashboardWidgetViews {
+  TRACES
+  OBSERVATIONS
+  SCORES_NUMERIC
+  SCORES_CATEGORICAL
+}
+
+enum DashboardWidgetChartType {
+  LINE_TIME_SERIES
+  BAR_TIME_SERIES
+  HORIZONTAL_BAR
+  VERTICAL_BAR
+  PIE
+}
+
+model DashboardWidget {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  createdBy     String? @map("created_by")
+  createdByUser User?   @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+  updatedBy     String? @map("updated_by")
+  updatedByUser User?   @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+  // Optional to account for Langfuse-managed dashboards
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  name        String @map("name")
+  description String @map("description")
+
+  view       DashboardWidgetViews @map("view")
+  dimensions Json                 @map("dimensions")
+  metrics    Json                 @map("metrics")
+  filters    Json                 @map("filters")
+
+  chartType   DashboardWidgetChartType @map("chart_type")
+  chartConfig Json                     @map("chart_config")
+
+  @@map("dashboard_widgets")
+}

packages/shared/prisma/seed.ts

@@ -14,6 +14,7 @@ import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey, logger } from "../src/server";
 import { encrypt } from "../src/encryption";
 import { redis } from "../src/server/redis/redis";
+import { randomUUID } from "crypto";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -180,6 +181,7 @@ async function main() {
         publicKey: seedApiKey.public,
         hashedSecretKey: await hashSecretKey(seedApiKey.secret),
         displaySecretKey: getDisplaySecretKey(seedApiKey.secret),
+        scope: "PROJECT",
         project: {
           connect: {
             id: project1.id,
@@ -241,6 +243,7 @@ async function main() {
           publicKey: secondKey.public,
           hashedSecretKey: await hashSecretKey(secondKey.secret),
           displaySecretKey: getDisplaySecretKey(secondKey.secret),
+          scope: "PROJECT",
           project: {
             connect: {
               id: project2.id,
@@ -374,6 +377,48 @@ async function main() {
     });
 
     await createDatasets(project1, project2, observations);
+
+    await createDashboardsAndWidgets([project1, project2]);
+
+    await prisma.llmSchema.createMany({
+      data: [
+        {
+          projectId: project1.id,
+          name: "get_weather",
+          description: "Fetches weather in Celsius for a given location",
+          schema: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+              unit: {
+                type: "string",
+                enum: ["celsius", "fahrenheit"],
+              },
+            },
+            required: ["location", "unit"],
+          },
+        },
+        {
+          projectId: project1.id,
+          name: "calculator",
+          description: "Performs basic arithmetic calculations",
+          schema: {
+            type: "object",
+            properties: {
+              expression: {
+                type: "string",
+                description:
+                  "The mathematical expression to evaluate, e.g. '2 + 2'",
+              },
+            },
+            required: ["expression"],
+          },
+        },
+      ],
+    });
   }
 }
 
@@ -391,6 +436,85 @@ main()
     process.exit(1);
   });
 
+async function createDashboardsAndWidgets(projects: Project[]) {
+  logger.info("Creating dashboards and widgets");
+
+  // Process each project
+  for (const project of projects) {
+    const widget = await prisma.dashboardWidget.upsert({
+      where: { id: "cabc" },
+      create: {
+        id: "cabc",
+        projectId: project.id,
+        name: "Trace Counts",
+        description: "Trace Counts by Name Over Time",
+        view: "TRACES",
+        dimensions: [{ field: "name" }],
+        metrics: [{ measure: "count", agg: "count" }],
+        filters: [],
+        chartType: "BAR_TIME_SERIES",
+        chartConfig: {
+          type: "BAR_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    const widget2 = await prisma.dashboardWidget.upsert({
+      where: { id: "cdef" },
+      create: {
+        id: "cdef",
+        projectId: project.id,
+        name: "Observation Latencies by Model",
+        description: "p95 Observation Latencies by Model Name",
+        view: "OBSERVATIONS",
+        dimensions: [{ field: "providedModelName" }],
+        metrics: [{ measure: "count", agg: "sum" }],
+        filters: [],
+        chartType: "LINE_TIME_SERIES",
+        chartConfig: {
+          type: "LINE_TIME_SERIES",
+        },
+      },
+      update: {},
+    });
+
+    // Create a dashboard with multiple widgets
+    await prisma.dashboard.upsert({
+      where: { id: "seed-dashboard" },
+      create: {
+        id: "seed-dashboard",
+        projectId: project.id,
+        name: "Performance Overview",
+        description: "Dashboard with various performance metrics",
+        definition: {
+          widgets: [
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget.id,
+              x: 0,
+              y: 0,
+              x_size: 6,
+              y_size: 2,
+            },
+            {
+              type: "widget",
+              id: randomUUID(),
+              widgetId: widget2.id,
+              x: 6,
+              y: 0,
+              x_size: 6,
+              y_size: 2,
+            },
+          ],
+        },
+      },
+      update: {},
+    });
+  }
+}
+
 export async function createDatasets(
   project1: {
     id: string;
@@ -730,6 +854,7 @@ function createObjects(
               source: "API",
               projectId,
               dataType: ScoreDataType.NUMERIC,
+              metadata: {},
             },
           ]
         : []),
@@ -745,6 +870,7 @@ function createObjects(
               dataType: ScoreDataType.CATEGORICAL,
               stringValue:
                 Math.floor(Math.random() * 2) === 1 ? "Fully" : "Partially",
+              metadata: {},
             },
           ]
         : []),

packages/shared/scripts/load-seed.ts

@@ -58,6 +58,7 @@ const prepareProjectsAndApiKeys = async (
           publicKey: `pk-${Math.random().toString(36).substr(2, 9)}`,
           hashedSecretKey: sk,
           displaySecretKey: getDisplaySecretKey(sk),
+          scope: "PROJECT",
           project: {
             connect: {
               id: projectId,

packages/shared/scripts/prepareClickhouse.ts

@@ -141,8 +141,6 @@ export const prepareClickhouse = async (
     FROM numbers(${observationsPerProject});
   `;
 
-    console.log(observationsQuery);
-
     const scoresQuery = `
     INSERT INTO scores
     SELECT toString(number) AS id,
@@ -150,6 +148,7 @@ export const prepareClickhouse = async (
       '${projectId}' AS project_id,
       'default' AS environment,
       toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
+      NULL AS session_id,
       if(
         rand() > 0.9,
         toString(floor(randUniform(0, ${observationsPerProject}))),
@@ -159,6 +158,7 @@ export const prepareClickhouse = async (
       randUniform(0, 100) as value,
       'API' as source,
       'comment' as comment,
+      map('key', 'value') AS metadata,
       toString(rand() % 100) as author_user_id,
       toString(rand() % 100) as config_id,
       if (rand() < 0.33, 'NUMERIC', if (rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) as data_type,
@@ -199,6 +199,47 @@ export const prepareClickhouse = async (
       project_id: string;
     }>();
 
+    const sessionsToScore = sessionData
+      .filter(() => Math.random() < 0.5)
+      .slice(0, Math.min(500, sessionData.length));
+
+    if (sessionsToScore.length > 0) {
+      // Generate session scores query with specific session IDs
+      const sessionScoresQuery = `
+        INSERT INTO scores
+        SELECT
+          concat('session-', toString(number)) AS id,
+          toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
+          '${projectId}' AS project_id,
+          'default' AS environment,
+          NULL AS trace_id,
+          arrayElement(['${sessionsToScore.map((s) => s.session_id).join("','")}'], 1 + (number % ${sessionsToScore.length})) AS session_id,
+          NULL AS observation_id,
+          concat('session_quality_', toString(rand() % 10)) AS name,
+          randUniform(0, 100) AS value,
+          'API' AS source,
+          'Session-level assessment score' AS comment,
+          map('key', 'value') AS metadata,
+          toString(rand() % 100) AS author_user_id,
+          toString(rand() % 100) AS config_id,
+          if(rand() < 0.33, 'NUMERIC', if(rand() < 0.5, 'CATEGORICAL', 'BOOLEAN')) AS data_type,
+          toString(rand() % 100) AS string_value,
+          NULL AS queue_id,
+          timestamp AS created_at,
+          timestamp AS updated_at,
+          timestamp AS event_ts,
+          0 AS is_deleted
+        FROM numbers(${sessionsToScore.length})
+      `;
+
+      await clickhouseClient().command({
+        query: sessionScoresQuery,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        },
+      });
+    }
+
     const idProjectIdCombinations = sessionData.map((session) => ({
       id: session.session_id,
       projectId: session.project_id,

packages/shared/src/domain/index.ts

@@ -0,0 +1,3 @@
+export * from "./observations";
+export * from "./traces";
+export * from "./scores";

packages/shared/src/domain/observations.ts

@@ -0,0 +1,76 @@
+import z from "zod";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const ObservationType = {
+  SPAN: "SPAN",
+  EVENT: "EVENT",
+  GENERATION: "GENERATION",
+} as const;
+export const ObservationTypeDomain = z.enum(["SPAN", "EVENT", "GENERATION"]);
+export type ObservationType = z.infer<typeof ObservationTypeDomain>;
+
+export const ObservationLevel = {
+  DEBUG: "DEBUG",
+  DEFAULT: "DEFAULT",
+  WARNING: "WARNING",
+  ERROR: "ERROR",
+} as const;
+export const ObservationLevelDomain = z.enum([
+  "DEBUG",
+  "DEFAULT",
+  "WARNING",
+  "ERROR",
+]);
+export type ObservationLevelType = z.infer<typeof ObservationLevelDomain>;
+
+export const ObservationDomain = z.object({
+  id: z.string(),
+  name: z.string(),
+  timestamp: z.date(),
+  environment: z.string(),
+  tags: z.array(z.string()),
+});
+export type ObservationDomain = z.infer<typeof ObservationDomain>;
+
+export const ObservationSchema = z.object({
+  id: z.string(),
+  traceId: z.string().nullable(),
+  projectId: z.string(),
+  environment: z.string(),
+  type: ObservationTypeDomain,
+  startTime: z.date(),
+  endTime: z.date().nullable(),
+  name: z.string().nullable(),
+  metadata: MetadataDomain,
+  parentObservationId: z.string().nullable(),
+  level: ObservationLevelDomain,
+  statusMessage: z.string().nullable(),
+  version: z.string().nullable(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  model: z.string().nullable(),
+  internalModelId: z.string().nullable(),
+  modelParameters: jsonSchema.nullable(),
+  input: jsonSchema.nullable(),
+  output: jsonSchema.nullable(),
+  completionStartTime: z.date().nullable(),
+  promptId: z.string().nullable(),
+  promptName: z.string().nullable(),
+  promptVersion: z.number().nullable(),
+  latency: z.number().nullable(),
+  timeToFirstToken: z.number().nullable(),
+  usageDetails: z.record(z.string(), z.number()),
+  costDetails: z.record(z.string(), z.number()),
+  providedCostDetails: z.record(z.string(), z.number()),
+  // aggregated data from cost_details
+  inputCost: z.number().nullable(),
+  outputCost: z.number().nullable(),
+  totalCost: z.number().nullable(),
+  // aggregated data from usage_details
+  inputUsage: z.number(),
+  outputUsage: z.number(),
+  totalUsage: z.number(),
+});
+
+export type Observation = z.infer<typeof ObservationSchema>;

packages/shared/src/domain/scores.ts

@@ -0,0 +1,36 @@
+import { ScoreDataType } from "@prisma/client";
+import z from "zod";
+import { MetadataDomain } from "./traces";
+
+export const ScoreSource = {
+  ANNOTATION: "ANNOTATION",
+  API: "API",
+  EVAL: "EVAL",
+} as const;
+
+export const ScoreSourceDomain = z.enum(["ANNOTATION", "API", "EVAL"]);
+export type ScoreSourceType = z.infer<typeof ScoreSourceDomain>;
+
+export const ScoreSchema = z.object({
+  id: z.string(),
+  timestamp: z.date(),
+  projectId: z.string(),
+  environment: z.string(),
+  name: z.string(),
+  value: z.number().nullable(),
+  source: ScoreSourceDomain,
+  authorUserId: z.string().nullable(),
+  comment: z.string().nullable(),
+  traceId: z.string().nullable(),
+  sessionId: z.string().nullable(),
+  metadata: MetadataDomain,
+  observationId: z.string().nullable(),
+  configId: z.string().nullable(),
+  stringValue: z.string().nullable(),
+  queueId: z.string().nullable(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  dataType: z.nativeEnum(ScoreDataType),
+});
+
+export type ScoreDomain = z.infer<typeof ScoreSchema>;

packages/shared/src/domain/traces.ts

@@ -0,0 +1,32 @@
+import z from "zod";
+import { jsonSchema, jsonSchemaNullable } from "../utils/zod";
+
+export const MetadataDomain = z.record(
+  z.string(),
+  jsonSchemaNullable.or(z.undefined()),
+);
+
+export type MetadataDomain = z.infer<typeof MetadataDomain>;
+
+// to be used across the application in frontend and backend.
+export const TraceDomain = z.object({
+  id: z.string(),
+  name: z.string().nullable(),
+  timestamp: z.date(),
+  environment: z.string(),
+  tags: z.array(z.string()),
+  bookmarked: z.boolean(),
+  public: z.boolean(),
+  release: z.string().nullable(),
+  version: z.string().nullable(),
+  input: jsonSchema.nullable(),
+  output: jsonSchema.nullable(),
+  metadata: MetadataDomain,
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  sessionId: z.string().nullable(),
+  userId: z.string().nullable(),
+  projectId: z.string(),
+});
+
+export type TraceDomain = z.infer<typeof TraceDomain>;

packages/shared/src/env.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { removeEmptyEnvVariables } from "./utils/environment";
 
 const EnvSchema = z.object({
+  NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
@@ -62,7 +63,18 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: z.string().default(""),
+  LANGFUSE_S3_MEDIA_UPLOAD_REGION: z.string().optional(),
+  LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: z.string().optional(),
+  LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: z.string().optional(),
+  LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: z.string().optional(),
+  LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: z
+    .enum(["true", "false"])
+    .default("false"),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
   LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
@@ -75,6 +87,9 @@ const EnvSchema = z.object({
   LANGFUSE_CUSTOM_SSO_EMAIL_CLAIM: z.string().default("email"),
   LANGFUSE_CUSTOM_SSO_NAME_CLAIM: z.string().default("name"),
   LANGFUSE_CUSTOM_SSO_SUB_CLAIM: z.string().default("sub"),
+  LANGFUSE_API_TRACE_OBSERVATIONS_SIZE_LIMIT_BYTES: z.coerce
+    .number()
+    .default(10e6), // 10MB
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/errors/ConflictError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class LangfuseConflictError extends BaseError {
+  constructor(description = "Conflict") {
+    super("LangfuseConflictError", 409, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -6,3 +6,4 @@ export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
 export { ApiError } from "./ApiError";
 export { InternalServerError } from "./InternalServerError";
+export { LangfuseConflictError } from "./ConflictError";

packages/shared/src/features/annotation/types.ts

@@ -25,13 +25,33 @@ const BooleanData = z.object({
   dataType: z.literal("BOOLEAN"),
 });
 
+const ScoreTargetTrace = z.object({
+  type: z.literal("trace"),
+  traceId: z.string(),
+  observationId: z.string().optional(),
+});
+
+const ScoreTargetSession = z.object({
+  type: z.literal("session"),
+  sessionId: z.string(),
+});
+
+// Your existing ScoreTarget remains the same, but can now use these components
+const ScoreTarget = z.discriminatedUnion("type", [
+  ScoreTargetTrace,
+  ScoreTargetSession,
+]);
+
+export type ScoreTargetTrace = z.infer<typeof ScoreTargetTrace>;
+export type ScoreTargetSession = z.infer<typeof ScoreTargetSession>;
+export type ScoreTarget = z.infer<typeof ScoreTarget>;
+
 const CreateAnnotationScoreBase = z.object({
   name: z.string(),
   projectId: z.string(),
   environment: z.string().default("default"),
-  traceId: z.string(),
+  scoreTarget: ScoreTarget,
   configId: z.string().optional(),
-  observationId: z.string().optional(),
   comment: z.string().nullish(),
   queueId: z.string().nullish(),
 });

packages/shared/src/features/batchExport/types.ts

@@ -15,6 +15,7 @@ export enum BatchExportStatus {
 export enum BatchExportFileFormat {
   JSON = "JSON",
   CSV = "CSV",
+  JSONL = "JSONL",
 }
 
 export enum BatchExportTableName {
@@ -35,6 +36,11 @@ export const exportOptions: Record<
 > = {
   CSV: { label: "CSV", extension: "csv", fileType: "text/csv" },
   JSON: { label: "JSON", extension: "json", fileType: "application/json" },
+  JSONL: {
+    label: "JSONL",
+    extension: "jsonl",
+    fileType: "application/x-ndjson",
+  },
 } as const;
 
 export const BatchExportQuerySchema = z.object({

packages/shared/src/features/scores/index.ts

@@ -1,3 +1,2 @@
-export * from "./scoreTypes";
-export * from "./types";
+export * from "./interfaces";
 export * from "./scoreConfigTypes";

packages/shared/src/features/scores/interfaces/README.md

@@ -0,0 +1,68 @@
+# Score Interfaces
+
+This directory contains all type definitions, schemas, and validation logic for Langfuse scores.
+
+## Structure
+
+```
+interfaces/
+├── api/              # API-specific schemas and validations
+│   ├── v1/           # Legacy API types (trace-focused)
+│   │   ├── endpoints.ts # Endpoint types and schemas
+│   │   ├── schemas.ts   # Type definitions
+│   │   └── validation.ts  # Validation logic
+│   ├── v2/           # Current API types (supports traces, sessions)
+│   │   ├── endpoints.ts # Endpoint types and schemas
+│   │   ├── schemas.ts   # Type definitions
+│   │   └── validation.ts  # Validation logic
+│   └── shared.ts     # Common schemas used across API versions
+├── application/      # Internal application logic
+│   └── validation.ts # Validation functions for application layers
+├── ingestion/        # Types for data ingestion
+│   └── validation.ts # Validation for ingestion endpoints
+├── ui/               # Simplified types for UI components
+│   └── types.ts      # UI-specific type definitions
+└── index.ts          # Exports all interfaces
+```
+
+## API Versioning
+
+We have added a new v2 api and will continue to support the v1 api for the foreseeable future.
+POST and DELETE APIs will continue to support both trace and session level scores across v1 and v2.
+
+For GET APIs:
+
+- **V1 API**: Requires `traceId`, ONLY supports trace-level scores
+- **V2 API**: Makes `traceId` optional, adds `sessionId` support, supports BOTH trace and session level scores
+- Both versions maintain compatibility for existing clients
+
+## Usage Guide
+
+### When building API endpoints:
+
+- **New endpoints**: Use types from `api/v2/schemas.ts`
+- **Legacy compatibility**: Use types from `api/v1/schemas.ts`
+- **Validation**: Use validators from the corresponding `validation.ts` files
+
+### When building UI components:
+
+Use the simplified types from `ui/types.ts` which are optimized for frontend use:
+
+```typescript
+import { ScoreSimplified, LastUserScore } from "../interfaces/ui/types";
+```
+
+### When working with internal application logic:
+
+Use the validation functions from `application/validation.ts`:
+
+```typescript
+import {
+  validateDbScore,
+  filterAndValidateDbScoreList,
+} from "../interfaces/application/validation";
+```
+
+## Type Flow
+
+Client → `PostScoresBody` → Validation → Database → `ScoreDomain` → API response (`APIScoreV2`) → UI (`ScoreSimplified` or `LastUserScore`)

packages/shared/src/features/scores/interfaces/api/index.ts

@@ -0,0 +1,6 @@
+export * from "./v1/schemas";
+export * from "./v1/validation";
+export * from "./v1/endpoints";
+export * from "./v2/schemas";
+export * from "./v2/validation";
+export * from "./v2/endpoints";

packages/shared/src/features/scores/interfaces/api/shared.ts

@@ -0,0 +1,139 @@
+import { z } from "zod";
+import { jsonSchema, publicApiPaginationZod } from "../../../../utils/zod";
+import { stringDateTime } from "../../../../utils/typeChecks";
+import { applyScoreValidation } from "../../../../utils/scores";
+import { PostScoreBodyFoundationSchema } from "../shared";
+
+const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
+const ScoreDataType = ["NUMERIC", "CATEGORICAL", "BOOLEAN"] as const;
+const ScoreSource = ["API", "EVAL", "ANNOTATION"] as const;
+
+/**
+ * Objects
+ */
+export const NumericData = z.object({
+  value: z.number(),
+  stringValue: z.undefined().nullish(),
+  dataType: z.literal("NUMERIC"),
+});
+
+export const CategoricalData = z.object({
+  value: z.number().nullish(),
+  stringValue: z.string(),
+  dataType: z.literal("CATEGORICAL"),
+});
+
+export const BooleanData = z.object({
+  value: z.number(),
+  stringValue: z.string(),
+  dataType: z.literal("BOOLEAN"),
+});
+
+/**
+ * Foundation schema for all score types, needs to be extended with entity score may be associated with. Note there are two API versions, where v1 allows only trace and observation scores, while v2 additionally allows session scores
+ * @see {@link ScoreFoundationSchemaV1}, {@link ScoreFoundationSchemaV2}
+ *
+ * Must also be extended with score data specific schema (numeric, categorical, boolean)
+ * @see {@link NumericData}, {@link CategoricalData}, {@link BooleanData}
+ */
+export const ScoreFoundationSchema = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  projectId: z.string(),
+  environment: z.string().default("default"),
+  name: z.string(),
+  source: z.enum(ScoreSource),
+  authorUserId: z.string().nullish(),
+  comment: z.string().nullish(),
+  metadata: jsonSchema.nullish(),
+  configId: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  queueId: z.string().nullish(),
+});
+
+/**
+ * Endpoints
+ */
+
+// GET /scores/{scoreId}
+export const GetScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+// GET /scores
+export const GetScoresQuery = z.object({
+  ...publicApiPaginationZod,
+  userId: z.string().nullish(),
+  dataType: z.enum(ScoreDataType).nullish(),
+  configId: z.string().nullish(),
+  queueId: z.string().nullish(),
+  traceTags: z.union([z.array(z.string()), z.string()]).nullish(),
+  environment: z.union([z.array(z.string()), z.string()]).nullish(),
+  name: z.string().nullish(),
+  fromTimestamp: stringDateTime,
+  toTimestamp: stringDateTime,
+  source: z.enum(ScoreSource).nullish(),
+  value: z.coerce.number().nullish(),
+  operator: z.enum(operators).nullish(),
+  scoreIds: z
+    .string()
+    .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
+    .refine((arr) => arr.every((id) => typeof id === "string"), {
+      message: "Each score ID must be a string",
+    })
+    .nullish(),
+});
+
+// POST /scores
+// Please note that the POST /scores endpoint supports both trace and session level scores across v1 and v2.
+/**
+ * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
+ */
+export const PostScoresBody = applyScoreValidation(
+  z.discriminatedUnion("dataType", [
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.number().refine((value) => value === 0 || value === 1, {
+          message:
+            "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
+        }),
+        dataType: z.literal("BOOLEAN"),
+        configId: z.string().nullish(),
+      }),
+    ),
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.union([z.string(), z.number()]),
+        dataType: z.undefined(),
+        configId: z.string().nullish(),
+      }),
+    ),
+  ]),
+);
+
+export const PostScoresResponse = z.object({ id: z.string() });
+
+// DELETE /scores/{scoreId}
+// Please note that the DELETE /scores/{scoreId} endpoint supports both trace and session level scores across v1 and v2.
+export const DeleteScoreQuery = z.object({
+  scoreId: z.string(),
+});
+
+export const DeleteScoreResponse = z.object({
+  message: z.string(),
+});

packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts

@@ -0,0 +1,44 @@
+import { z } from "zod";
+import { paginationMetaResponseZod } from "../../../../../utils/zod";
+import {
+  DeleteScoreQuery,
+  DeleteScoreResponse,
+  GetScoreQuery,
+  GetScoresQuery,
+  PostScoresBody,
+  PostScoresResponse,
+} from "../shared";
+import { APIScoreSchemaV1 } from "./schemas";
+
+// GET /scores/{scoreId}
+export const GetScoreQueryV1 = GetScoreQuery;
+export const GetScoreResponseV1 = APIScoreSchemaV1;
+
+// DELETE /scores/{scoreId}
+export const DeleteScoreQueryV1 = DeleteScoreQuery;
+export const DeleteScoreResponseV1 = DeleteScoreResponse;
+
+// GET /scores
+export const GetScoresQueryV1 = GetScoresQuery;
+
+// GetScoreResponseDataV1 is only used for response of GET /scores list endpoint
+export const GetScoreResponseDataV1 = z.intersection(
+  APIScoreSchemaV1,
+  z.object({
+    trace: z.object({
+      userId: z.string().nullish(),
+      tags: z.array(z.string()).nullish(),
+      environment: z.string().nullish(),
+    }),
+  }),
+);
+
+export const GetScoresResponseV1 = z.object({
+  data: z.array(GetScoreResponseDataV1),
+  meta: paginationMetaResponseZod,
+});
+
+// POST /scores
+export const PostScoresBodyV1 = PostScoresBody;
+
+export const PostScoresResponseV1 = PostScoresResponse;

packages/shared/src/features/scores/interfaces/api/v1/schemas.ts

@@ -0,0 +1,26 @@
+import { z } from "zod";
+import {
+  CategoricalData,
+  NumericData,
+  BooleanData,
+  ScoreFoundationSchema,
+} from "../shared";
+
+/**
+ * Foundation schema for scores API v1 i.e. trace and observation scores ONLY
+ *
+ * Must be extended with score data specific schema (numeric, categorical, boolean)
+ * @see {@link NumericData}, {@link CategoricalData}, {@link BooleanData}
+ */
+const ScoreFoundationSchemaV1 = ScoreFoundationSchema.extend({
+  traceId: z.string(),
+  observationId: z.string().nullish(),
+});
+
+export const APIScoreSchemaV1 = z.discriminatedUnion("dataType", [
+  ScoreFoundationSchemaV1.merge(NumericData),
+  ScoreFoundationSchemaV1.merge(CategoricalData),
+  ScoreFoundationSchemaV1.merge(BooleanData),
+]);
+
+export type APIScoreV1 = z.infer<typeof APIScoreSchemaV1>;

packages/shared/src/features/scores/interfaces/api/v1/validation.ts

@@ -0,0 +1,29 @@
+import z from "zod";
+import { GetScoreResponseDataV1 } from "./endpoints";
+
+/**
+ * @deprecated
+ * Use `filterAndValidateV2GetScoreList` instead. This function is only used for the legacy v1 API where scores were only associated with traces.
+ * Use this function when pulling a list of scores from the database before returning to the public API to ensure type safety.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scores
+ * @returns list of validated scores with trace information
+ */
+export const filterAndValidateV1GetScoreList = (
+  scores: unknown[],
+  // eslint-disable-next-line no-unused-vars
+  onParseError?: (error: z.ZodError) => void,
+): z.infer<typeof GetScoreResponseDataV1>[] =>
+  scores.reduce(
+    (acc: z.infer<typeof GetScoreResponseDataV1>[], ts) => {
+      const result = GetScoreResponseDataV1.safeParse(ts);
+      if (result.success) {
+        acc.push(result.data);
+      } else {
+        console.error(`Score parsing error ${JSON.stringify(result.error)}`);
+        onParseError?.(result.error);
+      }
+      return acc;
+    },
+    [] as z.infer<typeof GetScoreResponseDataV1>[],
+  );

packages/shared/src/features/scores/interfaces/api/v2/endpoints.ts

@@ -0,0 +1,28 @@
+import z from "zod";
+import { paginationMetaResponseZod } from "../../../../../utils/zod";
+import { GetScoreQuery, GetScoresQuery } from "../shared";
+import { APIScoreSchemaV2 } from "./schemas";
+
+// GET /scores/{scoreId} v2
+export const GetScoreQueryV2 = GetScoreQuery;
+export const GetScoreResponseV2 = APIScoreSchemaV2;
+
+// GET /scores v2
+export const GetScoresQueryV2 = GetScoresQuery;
+export const GetScoreResponseDataV2 = z.intersection(
+  APIScoreSchemaV2,
+  z.object({
+    trace: z
+      .object({
+        userId: z.string().nullish(),
+        tags: z.array(z.string()).nullish(),
+        environment: z.string().nullish(),
+      })
+      .nullish(),
+  }),
+);
+
+export const GetScoresResponseV2 = z.object({
+  data: z.array(GetScoreResponseDataV2),
+  meta: paginationMetaResponseZod,
+});

packages/shared/src/features/scores/interfaces/api/v2/schemas.ts

@@ -0,0 +1,27 @@
+import { z } from "zod";
+import {
+  CategoricalData,
+  NumericData,
+  BooleanData,
+  ScoreFoundationSchema,
+} from "../shared";
+
+/**
+ * Foundation schema for scores API v2 i.e. trace, observation AND session scores
+ *
+ * Must be extended with score data specific schema (numeric, categorical, boolean)
+ * @see {@link NumericData}, {@link CategoricalData}, {@link BooleanData}
+ */
+const ScoreFoundationSchemaV2 = ScoreFoundationSchema.extend({
+  traceId: z.string().nullish(),
+  observationId: z.string().nullish(),
+  sessionId: z.string().nullish(),
+});
+
+export const APIScoreSchemaV2 = z.discriminatedUnion("dataType", [
+  ScoreFoundationSchemaV2.merge(NumericData),
+  ScoreFoundationSchemaV2.merge(CategoricalData),
+  ScoreFoundationSchemaV2.merge(BooleanData),
+]);
+
+export type APIScoreV2 = z.infer<typeof APIScoreSchemaV2>;

packages/shared/src/features/scores/interfaces/api/v2/validation.ts

@@ -0,0 +1,27 @@
+import z from "zod";
+import { GetScoreResponseDataV2 } from "./endpoints";
+
+/**
+ * Use this function when pulling a list of scores from the database before returning to the public API to ensure type safety.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scores
+ * @returns list of validated scores with optional trace information in case of trace scores
+ */
+export const filterAndValidateV2GetScoreList = (
+  scores: unknown[],
+  // eslint-disable-next-line no-unused-vars
+  onParseError?: (error: z.ZodError) => void,
+): z.infer<typeof GetScoreResponseDataV2>[] =>
+  scores.reduce(
+    (acc: z.infer<typeof GetScoreResponseDataV2>[], ts) => {
+      const result = GetScoreResponseDataV2.safeParse(ts);
+      if (result.success) {
+        acc.push(result.data);
+      } else {
+        console.error("Score parsing error: ", result.error);
+        onParseError?.(result.error);
+      }
+      return acc;
+    },
+    [] as z.infer<typeof GetScoreResponseDataV2>[],
+  );

packages/shared/src/features/scores/interfaces/application/validation.ts

@@ -0,0 +1,94 @@
+import z from "zod";
+import { APIScoreSchemaV2, APIScoreV2 } from "../api/v2/schemas";
+import { APIScoreSchemaV1, APIScoreV1 } from "../api/v1/schemas";
+import { ScoreDomain } from "../../../../domain";
+
+type ValidatedAPIScore<IncludeHasMetadata extends boolean> = APIScoreV2 & {
+  hasMetadata: IncludeHasMetadata extends true ? boolean : never;
+};
+
+type InputScore = ScoreDomain & { hasMetadata?: boolean };
+
+/**
+ * Use this function when pulling a single score from the database before using in the application to ensure type safety.
+ * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
+ * @param score
+ * @returns validated score
+ * @throws error if score fails validation
+ */
+export const validateDbScore = (score: ScoreDomain): APIScoreV2 =>
+  APIScoreSchemaV2.parse(score);
+
+/**
+ * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scores
+ * @returns list of validated scores
+ */
+export const filterAndValidateDbScoreList = <
+  IncludeHasMetadata extends boolean,
+>({
+  scores,
+  includeHasMetadata = false as IncludeHasMetadata,
+  onParseError,
+}: {
+  scores: InputScore[];
+  includeHasMetadata?: IncludeHasMetadata;
+  // eslint-disable-next-line no-unused-vars
+  onParseError?: (error: z.ZodError) => void;
+}): ValidatedAPIScore<IncludeHasMetadata>[] => {
+  return scores.reduce((acc, ts) => {
+    const result = APIScoreSchemaV2.safeParse(ts);
+    if (result.success) {
+      const score = { ...result.data };
+      if (includeHasMetadata) {
+        Object.assign(score, { hasMetadata: ts.hasMetadata ?? false });
+      }
+      acc.push(score as ValidatedAPIScore<IncludeHasMetadata>);
+    } else {
+      console.error("Score parsing error: ", result.error);
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as ValidatedAPIScore<IncludeHasMetadata>[]);
+};
+
+type ValidatedAPITraceScore<IncludeHasMetadata extends boolean> = APIScoreV1 & {
+  hasMetadata: IncludeHasMetadata extends true ? boolean : never;
+};
+
+/**
+ * @deprecated
+ * Use `filterAndValidateDbScoreList` instead. This function is only used for the legacy v1 API where scores were only associated with traces.
+ * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
+ * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scores
+ * @returns list of validated scores
+ */
+export const filterAndValidateDbTraceScoreList = <
+  IncludeHasMetadata extends boolean,
+>({
+  scores,
+  includeHasMetadata = false as IncludeHasMetadata,
+  onParseError,
+}: {
+  scores: InputScore[];
+  includeHasMetadata?: IncludeHasMetadata;
+  // eslint-disable-next-line no-unused-vars
+  onParseError?: (error: z.ZodError) => void;
+}): ValidatedAPITraceScore<IncludeHasMetadata>[] => {
+  return scores.reduce((acc, ts) => {
+    const result = APIScoreSchemaV1.safeParse(ts);
+    if (result.success) {
+      const score = { ...result.data };
+      if (includeHasMetadata) {
+        Object.assign(score, { hasMetadata: ts.hasMetadata ?? false });
+      }
+      acc.push(score as ValidatedAPITraceScore<IncludeHasMetadata>);
+    } else {
+      console.error("Score parsing error: ", result.error);
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as ValidatedAPITraceScore<IncludeHasMetadata>[]);
+};

packages/shared/src/features/scores/interfaces/index.ts

@@ -0,0 +1,4 @@
+export * from "./api";
+export * from "./application/validation";
+export * from "./ingestion/validation";
+export * from "./ui/types";

packages/shared/src/features/scores/interfaces/ingestion/validation.ts

@@ -0,0 +1,78 @@
+import z from "zod";
+import { applyScoreValidation } from "../../../../utils/scores";
+import { PostScoreBodyFoundationSchema } from "../shared";
+import { isPresent } from "../../../../utils/typeChecks";
+import { Category as ConfigCategory } from "../../scoreConfigTypes";
+
+export const ScoreBodyWithoutConfig = applyScoreValidation(
+  z.discriminatedUnion("dataType", [
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.number(),
+        dataType: z.literal("NUMERIC"),
+      }),
+    ),
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.string(),
+        dataType: z.literal("CATEGORICAL"),
+      }),
+    ),
+    PostScoreBodyFoundationSchema.merge(
+      z.object({
+        value: z.number().refine((val) => val === 0 || val === 1, {
+          message: "Value must be either 0 or 1",
+        }),
+        dataType: z.literal("BOOLEAN"),
+      }),
+    ),
+  ]),
+);
+
+const ScorePropsAgainstConfigNumeric = z
+  .object({
+    value: z.number(),
+    maxValue: z.number().optional(),
+    minValue: z.number().optional(),
+    dataType: z.literal("NUMERIC"),
+  })
+  .superRefine((data, ctx) => {
+    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
+      });
+    }
+    if (isPresent(data.minValue) && data.value <= data.minValue) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value is below minimum value of ${data.minValue} defined in config`,
+      });
+    }
+  });
+
+const ScorePropsAgainstConfigCategorical = z
+  .object({
+    value: z.string(),
+    categories: z.array(ConfigCategory),
+    dataType: z.literal("CATEGORICAL"),
+  })
+  .superRefine((data, ctx) => {
+    if (!data.categories.some(({ label }) => label === data.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Value ${data.value} does not map to a valid category. Pass a valid category value.`,
+      });
+    }
+  });
+
+export const ScorePropsAgainstConfig = z.union([
+  ScorePropsAgainstConfigNumeric,
+  ScorePropsAgainstConfigCategorical,
+  z.object({
+    value: z.number().refine((val) => val === 0 || val === 1, {
+      message: "Value must be either 0 or 1",
+    }),
+    dataType: z.literal("BOOLEAN"),
+  }),
+]);

packages/shared/src/features/scores/interfaces/shared.ts

@@ -0,0 +1,19 @@
+import { jsonSchema } from "../../../utils/zod";
+import z from "zod";
+import { NonEmptyString } from "../../../utils/zod";
+
+/**
+ * Foundation schema for all score types. Used for ingestion and public API. Supports trace, observation and session scores.
+ * Needs to be extended with with score data specific schema (numeric, categorical, boolean)
+ * @see {@link NumericData}, {@link CategoricalData}, {@link BooleanData}
+ */
+export const PostScoreBodyFoundationSchema = z.object({
+  id: z.string().nullish(),
+  name: NonEmptyString,
+  traceId: z.string().nullish(),
+  sessionId: z.string().nullish(),
+  observationId: z.string().nullish(),
+  comment: z.string().nullish(),
+  metadata: jsonSchema.nullish(),
+  environment: z.string().default("default"),
+});

packages/shared/src/features/scores/interfaces/ui/types.ts

@@ -1,18 +1,22 @@
 import { ScoreDataType } from "@prisma/client";
-import { ScoreSourceType } from "../../server";
+import { MetadataDomain, ScoreSourceType } from "../../../../domain";
 
-export type CategoricalAggregate = {
+export type BaseAggregate = {
+  comment?: string | null;
+  id?: string | null;
+  hasMetadata?: boolean | null;
+};
+
+export type CategoricalAggregate = BaseAggregate & {
   type: "CATEGORICAL";
   values: string[];
   valueCounts: { value: string; count: number }[];
-  comment?: string | null;
 };
 
-export type NumericAggregate = {
+export type NumericAggregate = BaseAggregate & {
   type: "NUMERIC";
   values: number[];
   average: number;
-  comment?: string | null;
 };
 
 export type ScoreAggregate = Record<
@@ -21,11 +25,13 @@ export type ScoreAggregate = Record<
 >;
 
 export type ScoreSimplified = {
+  id: string;
   name: string;
   dataType: ScoreDataType;
   source: ScoreSourceType;
   value?: number | null;
   comment?: string | null;
+  metadata?: MetadataDomain | null;
   stringValue?: string | null;
 };
 

packages/shared/src/features/scores/scoreTypes.ts

@@ -1,305 +0,0 @@
-import { z } from "zod";
-
-import { isPresent, stringDateTime } from "../../utils/typeChecks";
-import { Score } from "../../server";
-import {
-  NonEmptyString,
-  paginationMetaResponseZod,
-  publicApiPaginationZod,
-} from "../../utils/zod";
-import { Category as ConfigCategory } from "./scoreConfigTypes";
-
-/**
- * Types to use across codebase
- */
-export type APIScore = z.infer<typeof APIScoreSchema>;
-
-/**
- * Helpers
- */
-
-const ScoreSource = ["API", "EVAL", "ANNOTATION"] as const;
-const ScoreDataType = ["NUMERIC", "CATEGORICAL", "BOOLEAN"] as const;
-
-const operators = ["<", ">", "<=", ">=", "!=", "="] as const;
-
-const NumericData = z.object({
-  value: z.number(),
-  stringValue: z.undefined().nullish(),
-  dataType: z.literal("NUMERIC"),
-});
-
-const CategoricalData = z.object({
-  value: z.number().nullish(),
-  stringValue: z.string(),
-  dataType: z.literal("CATEGORICAL"),
-});
-
-const BooleanData = z.object({
-  value: z.number(),
-  stringValue: z.string(),
-  dataType: z.literal("BOOLEAN"),
-});
-
-const ScoreBase = z.object({
-  id: z.string(),
-  timestamp: z.coerce.date(),
-  projectId: z.string(),
-  environment: z.string().default("default"),
-  name: z.string(),
-  source: z.enum(ScoreSource),
-  authorUserId: z.string().nullish(),
-  comment: z.string().nullish(),
-  traceId: z.string(),
-  observationId: z.string().nullish(),
-  configId: z.string().nullish(),
-  createdAt: z.coerce.date(),
-  updatedAt: z.coerce.date(),
-  queueId: z.string().nullish(),
-});
-
-const BaseScoreBody = z.object({
-  id: z.string().nullish(),
-  name: NonEmptyString,
-  traceId: z.string(),
-  observationId: z.string().nullish(),
-  comment: z.string().nullish(),
-  environment: z.string().default("default"),
-});
-
-/**
- * Objects
- */
-
-export const APIScoreSchema = z.discriminatedUnion("dataType", [
-  ScoreBase.merge(NumericData),
-  ScoreBase.merge(CategoricalData),
-  ScoreBase.merge(BooleanData),
-]);
-
-/**
- * Validation objects
- */
-export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
-  BaseScoreBody.merge(
-    z.object({
-      value: z.number(),
-      dataType: z.literal("NUMERIC"),
-    }),
-  ),
-  BaseScoreBody.merge(
-    z.object({
-      value: z.string(),
-      dataType: z.literal("CATEGORICAL"),
-    }),
-  ),
-  BaseScoreBody.merge(
-    z.object({
-      value: z.number().refine((val) => val === 0 || val === 1, {
-        message: "Value must be either 0 or 1",
-      }),
-      dataType: z.literal("BOOLEAN"),
-    }),
-  ),
-]);
-
-const ScorePropsAgainstConfigNumeric = z
-  .object({
-    value: z.number(),
-    maxValue: z.number().optional(),
-    minValue: z.number().optional(),
-    dataType: z.literal("NUMERIC"),
-  })
-  .superRefine((data, ctx) => {
-    if (isPresent(data.maxValue) && data.value >= data.maxValue) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Value exceeds maximum value of ${data.maxValue} defined in config`,
-      });
-    }
-    if (isPresent(data.minValue) && data.value <= data.minValue) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Value is below minimum value of ${data.minValue} defined in config`,
-      });
-    }
-  });
-
-const ScorePropsAgainstConfigCategorical = z
-  .object({
-    value: z.string(),
-    categories: z.array(ConfigCategory),
-    dataType: z.literal("CATEGORICAL"),
-  })
-  .superRefine((data, ctx) => {
-    if (!data.categories.some(({ label }) => label === data.value)) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Value ${data.value} does not map to a valid category. Pass a valid category value.`,
-      });
-    }
-  });
-
-export const ScorePropsAgainstConfig = z.union([
-  ScorePropsAgainstConfigNumeric,
-  ScorePropsAgainstConfigCategorical,
-  z.object({
-    value: z.number().refine((val) => val === 0 || val === 1, {
-      message: "Value must be either 0 or 1",
-    }),
-    dataType: z.literal("BOOLEAN"),
-  }),
-]);
-
-/**
- * Transformations
- */
-
-/**
- * Use this function when pulling a list of scores from the database before using in the application to ensure type safety.
- * All scores are expected to pass the validation. If a score fails validation, it will be logged to Otel.
- * @param scores
- * @returns list of validated scores
- */
-export const filterAndValidateDbScoreList = (
-  scores: Score[],
-  onParseError?: (error: z.ZodError) => void,
-): APIScore[] =>
-  scores.reduce((acc, ts) => {
-    const result = APIScoreSchema.safeParse(ts);
-    if (result.success) {
-      acc.push(result.data);
-    } else {
-      console.error("Score parsing error: ", result.error);
-      onParseError?.(result.error);
-    }
-    return acc;
-  }, [] as APIScore[]);
-
-/**
- * Use this function when pulling a single score from the database before using in the application to ensure type safety.
- * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
- * @param score
- * @returns validated score
- * @throws error if score fails validation
- */
-export const validateDbScore = (score: Score): APIScore =>
-  APIScoreSchema.parse(score);
-
-/**
- * Endpoints
- */
-
-// POST /scores
-/**
- * PostScoresBody is copied for the ingestion API as `ScoreBody`. Please copy any changes here in `packages/shared/src/features/ingestion/types.ts`
- */
-export const PostScoresBody = z.discriminatedUnion("dataType", [
-  BaseScoreBody.merge(
-    z.object({
-      value: z.number(),
-      dataType: z.literal("NUMERIC"),
-      configId: z.string().nullish(),
-    }),
-  ),
-  BaseScoreBody.merge(
-    z.object({
-      value: z.string(),
-      dataType: z.literal("CATEGORICAL"),
-      configId: z.string().nullish(),
-    }),
-  ),
-  BaseScoreBody.merge(
-    z.object({
-      value: z.number().refine((value) => value === 0 || value === 1, {
-        message:
-          "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
-      }),
-      dataType: z.literal("BOOLEAN"),
-      configId: z.string().nullish(),
-    }),
-  ),
-  BaseScoreBody.merge(
-    z.object({
-      value: z.union([z.string(), z.number()]),
-      dataType: z.undefined(),
-      configId: z.string().nullish(),
-    }),
-  ),
-]);
-
-export const PostScoresResponse = z.object({ id: z.string() });
-
-// GET /scores
-export const GetScoresQuery = z.object({
-  ...publicApiPaginationZod,
-  userId: z.string().nullish(),
-  dataType: z.enum(ScoreDataType).nullish(),
-  configId: z.string().nullish(),
-  queueId: z.string().nullish(),
-  traceTags: z.union([z.array(z.string()), z.string()]).nullish(),
-  environment: z.union([z.array(z.string()), z.string()]).nullish(),
-  name: z.string().nullish(),
-  fromTimestamp: stringDateTime,
-  toTimestamp: stringDateTime,
-  source: z.enum(ScoreSource).nullish(),
-  value: z.coerce.number().nullish(),
-  operator: z.enum(operators).nullish(),
-  scoreIds: z
-    .string()
-    .transform((str) => str.split(",").map((id) => id.trim())) // Split the comma-separated string
-    .refine((arr) => arr.every((id) => typeof id === "string"), {
-      message: "Each score ID must be a string",
-    })
-    .nullish(),
-});
-
-// LegacyGetScoreResponseDataV1 is only used for response of GET /scores list endpoint
-const LegacyGetScoreResponseDataV1 = z.intersection(
-  APIScoreSchema,
-  z.object({
-    trace: z.object({
-      userId: z.string().nullish(),
-      tags: z.array(z.string()).nullish(),
-      environment: z.string().nullish(),
-    }),
-  }),
-);
-export const GetScoresResponse = z.object({
-  data: z.array(LegacyGetScoreResponseDataV1),
-  meta: paginationMetaResponseZod,
-});
-
-export const legacyFilterAndValidateV1GetScoreList = (
-  scores: unknown[],
-  onParseError?: (error: z.ZodError) => void,
-): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
-  scores.reduce(
-    (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
-      const result = LegacyGetScoreResponseDataV1.safeParse(ts);
-      if (result.success) {
-        acc.push(result.data);
-      } else {
-        console.error("Score parsing error: ", result.error);
-        onParseError?.(result.error);
-      }
-      return acc;
-    },
-    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[],
-  );
-
-// GET /scores/{scoreId}
-export const GetScoreQuery = z.object({
-  scoreId: z.string(),
-});
-
-export const GetScoreResponse = APIScoreSchema;
-
-// DELETE /scores/{scoreId}
-export const DeleteScoreQuery = z.object({
-  scoreId: z.string(),
-});
-
-export const DeleteScoreResponse = z.object({
-  message: z.string(),
-});

packages/shared/src/index.ts

@@ -15,6 +15,7 @@ export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
+export * from "./tableDefinitions/typeHelpers";
 
 // llm api
 export * from "./server/llm/types";
@@ -46,7 +47,13 @@ export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
 export * from "./server/repositories/types";
 
+// metadata conversion
+export * from "./server/utils/metadata_conversion";
+
 // errors
 export * from "./errors/index";
 
 export * from "./utils/environment";
+
+// domain
+export * from "./domain";

packages/shared/src/interfaces/filters.ts

@@ -4,6 +4,7 @@ export const filterOperators = {
   datetime: [">", "<", ">=", "<="],
   string: ["=", "contains", "does not contain", "starts with", "ends with"],
   stringOptions: ["any of", "none of"],
+  categoryOptions: ["any of", "none of"],
   arrayOptions: ["any of", "none of", "all of"],
   number: ["=", ">", "<", ">=", "<="],
   stringObject: [
@@ -75,11 +76,19 @@ export const nullFilter = z.object({
   operator: z.enum(filterOperators.null),
   value: z.literal(""),
 });
+export const categoryOptionsFilter = z.object({
+  type: z.literal("categoryOptions"),
+  column: z.string(),
+  key: z.string(),
+  operator: z.enum(filterOperators.categoryOptions),
+  value: z.array(z.string()),
+});
 export const singleFilter = z.discriminatedUnion("type", [
   timeFilter,
   stringFilter,
   numberFilter,
   stringOptionsFilter,
+  categoryOptionsFilter,
   arrayOptionsFilter,
   stringObjectFilter,
   numberObjectFilter,

packages/shared/src/observationsTable.ts

@@ -1,8 +1,10 @@
+import { ObservationLevelType } from "./domain";
 import {
-  type OptionsDefinition,
+  type SingleValueOption,
   type ColumnDefinition,
+  MultiValueOption,
 } from "./tableDefinitions";
-import { ObservationLevelType } from "./server";
+import { formatColumnOptions } from "./tableDefinitions/typeHelpers";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
@@ -154,8 +156,8 @@ export const observationsTableCols: ColumnDefinition[] = [
     nullable: true,
   },
   {
-    name: "Usage",
-    id: "usage",
+    name: "Tokens",
+    id: "tokens",
     type: "number",
     internal: 'o."total_tokens"',
   },
@@ -166,11 +168,19 @@ export const observationsTableCols: ColumnDefinition[] = [
     internal: 'o."metadata"',
   },
   {
-    name: "Scores",
+    name: "Scores (numeric)",
     id: "scores_avg",
     type: "numberObject",
     internal: "scores_avg",
   },
+  {
+    name: "Scores (categorical)",
+    id: "score_categories",
+    type: "categoryOptions",
+    internal: "score_categories",
+    options: [], // to be added at runtime
+    nullable: true,
+  },
   {
     name: "Version",
     id: "version",
@@ -205,14 +215,15 @@ export const observationsTableCols: ColumnDefinition[] = [
 // to be used client side, insert options for use in filter-builder
 // allows for undefined options, to offer filters while options are still loading
 export type ObservationOptions = {
-  model: Array<OptionsDefinition>;
-  modelId: Array<OptionsDefinition>;
-  name: Array<OptionsDefinition>;
-  traceName: Array<OptionsDefinition>;
+  model: Array<SingleValueOption>;
+  modelId: Array<SingleValueOption>;
+  name: Array<SingleValueOption>;
+  traceName: Array<SingleValueOption>;
   scores_avg: Array<string>;
-  promptName: Array<OptionsDefinition>;
-  tags: Array<OptionsDefinition>;
-  type: Array<OptionsDefinition>;
+  score_categories: Array<MultiValueOption>;
+  promptName: Array<SingleValueOption>;
+  tags: Array<SingleValueOption>;
+  type: Array<SingleValueOption>;
 };
 
 export function observationsTableColsWithOptions(
@@ -220,28 +231,31 @@ export function observationsTableColsWithOptions(
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {
-      return { ...col, options: options?.model ?? [] };
+      return formatColumnOptions(col, options?.model ?? []);
     }
     if (col.id === "modelId") {
-      return { ...col, options: options?.modelId ?? [] };
+      return formatColumnOptions(col, options?.modelId ?? []);
     }
     if (col.id === "name") {
-      return { ...col, options: options?.name ?? [] };
+      return formatColumnOptions(col, options?.name ?? []);
     }
     if (col.id === "traceName") {
-      return { ...col, options: options?.traceName ?? [] };
+      return formatColumnOptions(col, options?.traceName ?? []);
     }
     if (col.id === "scores_avg") {
-      return { ...col, keyOptions: options?.scores_avg ?? [] };
+      return formatColumnOptions(col, options?.scores_avg ?? []);
+    }
+    if (col.id === "score_categories") {
+      return formatColumnOptions(col, options?.score_categories ?? []);
     }
     if (col.id === "promptName") {
-      return { ...col, options: options?.promptName ?? [] };
+      return formatColumnOptions(col, options?.promptName ?? []);
     }
     if (col.id === "tags") {
-      return { ...col, options: options?.tags ?? [] };
+      return formatColumnOptions(col, options?.tags ?? []);
     }
     if (col.id === "type") {
-      return { ...col, options: options?.type ?? [] };
+      return formatColumnOptions(col, options?.type ?? []);
     }
     return col;
   });

packages/shared/src/server/auth/apiKeys.ts

@@ -1,4 +1,4 @@
-import { PrismaClient } from "@prisma/client";
+import { PrismaClient, ApiKeyScope } from "@prisma/client";
 import { compare, hash } from "bcryptjs";
 import { randomUUID } from "crypto";
 import * as crypto from "crypto";
@@ -38,7 +38,8 @@ export function createShaHash(privateKey: string, salt: string): string {
 
 export async function createAndAddApiKeysToDb(p: {
   prisma: PrismaClient;
-  projectId: string;
+  entityId: string;
+  scope: ApiKeyScope;
   note?: string;
   predefinedKeys?: {
     secretKey: string;
@@ -59,14 +60,18 @@ export async function createAndAddApiKeysToDb(p: {
 
   const hashFromProvidedKey = createShaHash(sk, salt);
 
+  const entity =
+    p.scope === "PROJECT" ? { projectId: p.entityId } : { orgId: p.entityId };
+
   const apiKey = await p.prisma.apiKey.create({
     data: {
-      projectId: p.projectId,
+      ...entity,
       publicKey: pk,
       hashedSecretKey: hashedSk,
       displaySecretKey: displaySk,
       fastHashedSecretKey: hashFromProvidedKey,
       note: p.note,
+      scope: p.scope,
     },
   });
 

packages/shared/src/server/auth/types.ts

@@ -1,8 +1,9 @@
 import z from "zod";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
+import { ApiKeyScope } from "../../";
 
-export const OrgEnrichedApiKey = z.object({
+const ApiKeyBaseSchema = z.object({
   id: z.string(),
   note: z.string().nullable(),
   publicKey: z.string(),
@@ -10,7 +11,6 @@ export const OrgEnrichedApiKey = z.object({
   createdAt: z.string().datetime().nullable(),
   lastUsedAt: z.string().datetime().nullable(),
   expiresAt: z.string().datetime().nullable(),
-  projectId: z.string(),
   fastHashedSecretKey: z.string(),
   hashedSecretKey: z.string(),
   orgId: z.string(),
@@ -18,6 +18,17 @@ export const OrgEnrichedApiKey = z.object({
   rateLimitOverrides: CloudConfigRateLimit.nullish(),
 });
 
+export const OrgEnrichedApiKey = z.discriminatedUnion("scope", [
+  ApiKeyBaseSchema.extend({
+    scope: z.literal(ApiKeyScope.ORGANIZATION),
+    projectId: z.null(),
+  }),
+  ApiKeyBaseSchema.extend({
+    scope: z.literal(ApiKeyScope.PROJECT),
+    projectId: z.string(),
+  }),
+]);
+
 export const API_KEY_NON_EXISTENT = "api-key-non-existent";
 
 export const CachedApiKey = z.union([
@@ -38,8 +49,8 @@ export type AuthHeaderValidVerificationResult = {
 };
 
 export type ApiAccessScope = {
-  projectId: string;
-  accessLevel: "all" | "scores";
+  projectId: string | null;
+  accessLevel: "organization" | "project" | "scores";
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;

packages/shared/src/server/clickhouse/client.ts

@@ -6,32 +6,112 @@ import { propagation, context } from "@opentelemetry/api";
 
 export type ClickhouseClientType = ReturnType<typeof createClient>;
 
-export const clickhouseClient = (
-  params: {
-    tags?: Record<string, string>;
-    opts?: NodeClickHouseClientConfigOptions;
-  } = {},
-) => {
-  const headers = params.opts?.http_headers ?? {};
-  const activeSpan = getCurrentSpan();
-  if (activeSpan) {
-    propagation.inject(context.active(), headers);
+/**
+ * ClickHouseClientManager provides a singleton pattern for managing ClickHouse clients.
+ * It creates and reuses clients based on their configuration to avoid creating
+ * a new connection for each query.
+ */
+export class ClickHouseClientManager {
+  private static instance: ClickHouseClientManager;
+  private clientMap: Map<string, ClickhouseClientType> = new Map();
+
+  /**
+   * Private constructor to enforce singleton pattern
+   */
+  private constructor() {}
+
+  /**
+   * Get the singleton instance of the ClickHouseClientManager
+   */
+  public static getInstance(): ClickHouseClientManager {
+    if (!ClickHouseClientManager.instance) {
+      ClickHouseClientManager.instance = new ClickHouseClientManager();
+    }
+    return ClickHouseClientManager.instance;
   }
 
-  return createClient({
-    ...params.opts,
-    url: env.CLICKHOUSE_URL,
-    username: env.CLICKHOUSE_USER,
-    password: env.CLICKHOUSE_PASSWORD,
-    database: env.CLICKHOUSE_DB,
-    http_headers: headers,
-    clickhouse_settings: {
-      log_comment: JSON.stringify(params.tags ?? {}),
-      async_insert: 1,
-      wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
-      input_format_json_throw_on_bad_escape_sequence: 0,
-    },
-  });
+  /**
+   * Generate a consistent hash key for client configurations
+   * @param opts Client parameters
+   * @returns String hash key
+   */
+  private generateClientSettingsKey(
+    opts: NodeClickHouseClientConfigOptions,
+  ): string {
+    const keyParams = {
+      url: env.CLICKHOUSE_URL,
+      username: env.CLICKHOUSE_USER,
+      password: env.CLICKHOUSE_PASSWORD,
+      database: env.CLICKHOUSE_DB,
+      http_headers: opts?.http_headers,
+      settings: opts?.clickhouse_settings,
+      // Include any other relevant config options
+    };
+
+    return JSON.stringify(keyParams);
+  }
+
+  /**
+   * Get or create a client based on the provided parameters
+   * @param opts Client configuration parameters
+   * @returns ClickHouse client instance
+   */
+  public getClient(
+    opts: NodeClickHouseClientConfigOptions,
+  ): ClickhouseClientType {
+    const key = this.generateClientSettingsKey(opts);
+
+    if (!this.clientMap.has(key)) {
+      const headers = opts?.http_headers ?? {};
+      const activeSpan = getCurrentSpan();
+      if (activeSpan) {
+        propagation.inject(context.active(), headers);
+      }
+
+      const cloudOptions: Record<string, unknown> = {};
+      if (
+        ["STAGING", "EU", "US", "HIPAA"].includes(
+          env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ?? "",
+        )
+      ) {
+        cloudOptions.input_format_json_throw_on_bad_escape_sequence = 0;
+      }
+
+      const client = createClient({
+        ...opts,
+        url: env.CLICKHOUSE_URL,
+        username: env.CLICKHOUSE_USER,
+        password: env.CLICKHOUSE_PASSWORD,
+        database: env.CLICKHOUSE_DB,
+        http_headers: headers,
+        clickhouse_settings: {
+          ...cloudOptions,
+          ...opts.clickhouse_settings,
+          async_insert: 1,
+          wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
+        },
+      });
+
+      this.clientMap.set(key, client);
+    }
+
+    return this.clientMap.get(key)!;
+  }
+
+  /**
+   * Close all client connections - useful for application shutdown
+   */
+  public closeAllConnections(): Promise<void[]> {
+    const closePromises = Array.from(this.clientMap.values()).map((client) =>
+      client.close(),
+    );
+    this.clientMap.clear();
+    return Promise.all(closePromises);
+  }
+}
+
+export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) => {
+  return ClickHouseClientManager.getInstance().getClient(opts ?? {});
 };
 
 /**

packages/shared/src/server/data-deletion/ingestionFileDeletion.ts

@@ -0,0 +1,108 @@
+import {
+  BlobStorageFileRefRecordReadType,
+  getBlobStorageByProjectId,
+  getBlobStorageByProjectIdAndEntityIds,
+  getBlobStorageByProjectIdAndTraceIds,
+  getBlobStorageByProjectIdBeforeDate,
+  logger,
+} from "..";
+import { env } from "../../env";
+import { clickhouseClient } from "../clickhouse/client";
+import { getS3EventStorageClient } from "../s3";
+
+export const deleteIngestionEventsFromS3AndClickhouseForScores = async (p: {
+  projectId: string;
+  scoreIds: string[];
+}) => {
+  const stream = getBlobStorageByProjectIdAndEntityIds(
+    p.projectId,
+    "score",
+    p.scoreIds,
+  );
+
+  return removeIngestionEventsFromS3AndDeleteClickhouseRefs({
+    projectId: p.projectId,
+    stream,
+  });
+};
+
+export const removeIngestionEventsFromS3AndDeleteClickhouseRefsForTraces =
+  async (p: { projectId: string; traceIds: string[] }) => {
+    const stream = getBlobStorageByProjectIdAndTraceIds(
+      p.projectId,
+      p.traceIds,
+    );
+
+    return removeIngestionEventsFromS3AndDeleteClickhouseRefs({
+      projectId: p.projectId,
+      stream: stream,
+    });
+  };
+
+export const removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject = (
+  projectId: string,
+  cutOffDate: Date | undefined,
+) => {
+  const stream = cutOffDate
+    ? getBlobStorageByProjectIdBeforeDate(projectId, cutOffDate)
+    : getBlobStorageByProjectId(projectId);
+
+  return removeIngestionEventsFromS3AndDeleteClickhouseRefs({
+    projectId: projectId,
+    stream: stream,
+  });
+};
+
+async function removeIngestionEventsFromS3AndDeleteClickhouseRefs(p: {
+  projectId: string;
+  stream: AsyncGenerator<BlobStorageFileRefRecordReadType>;
+}) {
+  const { projectId, stream } = p;
+
+  let batch = 0;
+
+  let blobStorageRefs: BlobStorageFileRefRecordReadType[] = [];
+  const eventStorageClient = getS3EventStorageClient(
+    env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+  );
+  for await (const eventLog of stream) {
+    blobStorageRefs.push(eventLog);
+    if (blobStorageRefs.length > 500) {
+      // Delete the current batch and reset the list
+      await eventStorageClient.deleteFiles(
+        blobStorageRefs.map((r) => r.bucket_path),
+      );
+
+      // soft delete the blob storage references in clickhouse
+      await softDeleteInClickhouse(blobStorageRefs);
+      batch++;
+      logger.info(
+        `Deleted batch ${batch} of size ${blobStorageRefs.length} for ${projectId} of deleting s3 refs`,
+      );
+      blobStorageRefs = [];
+    }
+  }
+  // Delete any remaining files
+  await eventStorageClient.deleteFiles(
+    blobStorageRefs.map((r) => r.bucket_path),
+  );
+  await softDeleteInClickhouse(blobStorageRefs);
+  logger.info(
+    `Deleted batch ${batch} of size ${blobStorageRefs.length} for ${projectId} of deleting s3 refs`,
+  );
+}
+
+async function softDeleteInClickhouse(
+  blobStorageRefs: BlobStorageFileRefRecordReadType[],
+) {
+  await clickhouseClient().insert({
+    table: "blob_storage_file_log",
+    values: blobStorageRefs.map((e) => ({
+      ...e,
+      is_deleted: "1",
+      event_ts: new Date().getTime(),
+      updated_at: new Date().getTime(),
+    })),
+    format: "JSONEachRow",
+  });
+}

packages/shared/src/server/filterToPrisma.ts

@@ -109,6 +109,10 @@ export function tableColumnsToSqlFilter(
       case "boolean":
         valuePrisma = Prisma.sql`${filter.value}`;
         break;
+      case "categoryOptions":
+        // LFE-4815: Support category options in postgres
+        logger.warn("Category options not supported in postgres yet");
+        throw new Error("Category options not supported in postgres yet");
       case "null":
         valuePrisma = Prisma.sql``;
         break;

packages/shared/src/server/index.ts

@@ -41,6 +41,7 @@ export * from "./redis/dataRetentionProcessingQueue";
 export * from "./redis/coreDataS3ExportQueue";
 export * from "./redis/meteringDataPostgresExportQueue";
 export * from "./redis/experimentCreateQueue";
+export * from "./redis/dlxRetryQueue";
 export * from "./auth/types";
 export * from "./queues";
 export * from "./orderByToPrisma";
@@ -52,6 +53,10 @@ export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
 export * from "./services/datasets-ui-table-service";
+export * from "./services/DashboardService";
+
+export * from "./data-deletion/ingestionFileDeletion";
+export * from "./s3";
 
 // test utils
 export * from "./test-utils";

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -98,13 +98,20 @@ export const processEventBatch = async (
   recordDistribution("langfuse.ingestion.event_distribution", input.length);
 
   currentSpan?.setAttribute("langfuse.ingestion.batch_size", input.length);
-  currentSpan?.setAttribute("langfuse.project.id", authCheck.scope.projectId);
+  currentSpan?.setAttribute(
+    "langfuse.project.id",
+    authCheck.scope.projectId ?? "",
+  );
   currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
   currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
    **************/
+  if (!authCheck.scope.projectId) {
+    throw new UnauthorizedError("Missing project ID");
+  }
+
   const validationErrors: { id: string; error: unknown }[] = [];
   const authenticationErrors: { id: string; error: unknown }[] = [];
 
@@ -245,7 +252,13 @@ export const processEventBatch = async (
                   eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
                   fileKey: sortedBatchByEventBodyId[id].key,
                 },
-                authCheck,
+                authCheck: authCheck as {
+                  validKey: true;
+                  scope: {
+                    projectId: string;
+                    accessLevel: "project" | "scores";
+                  };
+                },
               },
             },
             { delay: getDelay(delay) },
@@ -272,11 +285,11 @@ const isAuthorized = (
   if (event.type === eventTypes.SCORE_CREATE) {
     return (
       authScope.scope.accessLevel === "scores" ||
-      authScope.scope.accessLevel === "all"
+      authScope.scope.accessLevel === "project"
     );
   }
 
-  return authScope.scope.accessLevel === "all";
+  return authScope.scope.accessLevel === "project";
 };
 
 /**