chore: release v3.168.0

* fix(evals): return full JSONPath slice result and deduplicate eval JSONPath logic

JSONPath slice expressions (e.g. $[1:]) returned only the first matched
element due to an unconditional result[0] in parseJsonDefault. Now
multi-match results return the full array while single-match results
remain unwrapped for backward compatibility.

Also consolidates three separate JSONPath evaluation paths (UI preview,
trace eval, observation eval) into the shared extractValueFromObject,
removing duplicated logic from the worker. The snakeToCamel column ID
fallback and parseUnknownToString remain in the worker since they are
database-specific concerns.

Closes LFE-8416

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(evals): address PR review — scope parseMultiEncodedJson, fix error logging and test expectations

- Only call parseMultiEncodedJson when a jsonSelector is present to avoid
  mutating formatting on the no-selector passthrough path.
- Preserve the raw original value (not the parsed one) in the error
  fallback.
- Add error logging in extractObservationVariables (was already done in
  parseDatabaseRowToString but missed here).
- Update three pre-existing test expectations to match the new unwrap
  semantics: single-match results are unwrapped, non-matching paths
  return empty string instead of "[]".

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(evals): update evalService test expectations for single-match unwrap

Four more test assertions in evalService.test.ts still expected
array-wrapped JSONPath results (e.g. '["Hello world"]'). Updated to
match the new unwrap semantics for single-match queries.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* test(evals): remove duplicate slice/single-element tests from extractObservationVariables

These cases are already covered by extractValueFromObject.test.ts.
The pre-existing tests ($.prompt, $.response, non-matching path) remain
as integration tests for the observation eval path.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* style(evals): use @langfuse/shared alias instead of relative path in test import

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.env.dev-oci.example

@@ -0,0 +1,196 @@
+#####################################################################
+# .env (template) — OCI Object Storage / S3-compatible configuration
+#
+# IMPORTANT SECURITY NOTES (Oracle best practice)
+# - Prefer OCI-native auth (Instance Principal / Workload Identity / Resource Principal)
+#   over static keys.
+# - If you must use static keys, store them in a secure secret manager
+#   (e.g., Kubernetes Secret / OCI Vault) and inject at runtime.
+# - Rotate/revoke any credentials that were previously shared or committed.
+#####################################################################
+
+
+#####################################################################
+# 1) Storage/Auth category (CHOOSE ONE)
+#
+# The app can read/write/download to/from an OCI object store for:
+# - Batch exports (exports/)
+# - Media uploads (media/)
+# - Event uploads (events/)
+#
+# Pick exactly ONE auth mechanism for OCI-native object storage by setting:
+#   LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+#   LANGFUSE_OCI_AUTH_TYPE=<one of the values below>
+#
+# Supported values:
+#   workload_identity | instance_principal | resource_principal | oci_profile | session_token
+#####################################################################
+
+
+#####################################################################
+# Category A — OCI Object Storage with INSTANCE PRINCIPAL (recommended on OCI Compute)
+# Use when:
+# - Running on OCI Compute with IAM set up (dynamic group + policies)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=instance_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category B — OCI Object Storage with WORKLOAD IDENTITY (common on OKE)
+# Use when:
+# - Running on OKE with OCI Workload Identity configured
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=workload_identity
+#
+# Optional (only if your environment requires additional CA trust):
+# NODE_EXTRA_CA_CERTS=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category C — OCI Object Storage with RESOURCE PRINCIPAL (common for OCI services)
+# Use when:
+# - Running inside an OCI service/runtime that injects Resource Principal env vars
+#   (e.g., certain managed services / automation contexts)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=resource_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category D — OCI Object Storage with OCI CONFIG PROFILE (developer local)
+# Use when:
+# - You have an OCI config file locally or mounted in the runtime
+# - You want to use a named profile
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=oci_profile
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#
+# NOTE: Avoid adding config files into images; mount/inject securely.
+#####################################################################
+
+
+#####################################################################
+# Category E — OCI Object Storage with SESSION TOKEN (short-lived user auth)
+# Use when:
+# - You use OCI CLI session authentication (short-lived token flow)
+# - USE oci session authenticate
+# - Appropriate for interactive/dev use; less common for long-running services
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=session_token
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#####################################################################
+
+
+#####################################################################
+# Other possible setup — Non-OCI provider (AWS S3 / GCP / Azure / MinIO / etc.)
+# Use when:
+# - Your object storage is NOT OCI Object Storage
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=false
+#
+# Then configure endpoints/regions/credentials for your provider.
+#####################################################################
+
+
+
+#####################################################################
+# 2) Feature: S3 Batch Export
+#
+# Required (when enabled):
+# - *_BUCKET, *_REGION, *_ENDPOINT, *_PREFIX
+# Optional:
+# - *_EXTERNAL_ENDPOINT
+# - *_FORCE_PATH_STYLE=true (needed for many S3-compatible providers like MinIO)
+#
+# Credentials:
+# - Set *_ACCESS_KEY_ID/_SECRET_ACCESS_KEY ONLY for static-key auth
+#   (non-OCI S3-compatible providers)
+#####################################################################
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse-bucket
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# OCI example region/endpoint:
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-chicago-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+
+# MinIO / S3-compat setting (safe to keep true for many S3-compatible endpoints)
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 3) Feature: S3 Media Upload
+#####################################################################
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 4) Feature: S3 Event Upload (optional)
+#####################################################################
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 5) OCI native auth configuration (used by oci_profile / session_token)
+#####################################################################
+# Only required when LANGFUSE_OCI_AUTH_TYPE is: oci_profile OR session_token
+OCI_CONFIG_FILE=__REPLACE_ME__/config
+OCI_CONFIG_PROFILE=DEFAULT
+
+
+
+#####################################################################
+# 6) Troubleshooting notes (comments only)
+#
+# - If you see TLS errors to the endpoint in Kubernetes/OKE, set NODE_EXTRA_CA_CERTS to the
+#   correct CA bundle path for your environment.
+# - If using MinIO or certain S3-compatible providers and you get bucket addressing errors,
+#   set *_FORCE_PATH_STYLE=true.
+# - If downloads work inside the cluster but not externally, configure
+#   LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT to a publicly reachable endpoint/DNS.
+#####################################################################

.env.dev.example

@@ -153,6 +153,7 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
 # Langfuse AI Bedrock credentials
 AWS_ACCESS_KEY_ID="A123456789"
 AWS_SECRET_ACCESS_KEY="SAK123456789"
+LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY="1234567890abcdef"
 LANGFUSE_AWS_BEDROCK_REGION="eu-west-1"
 LANGFUSE_AWS_BEDROCK_MODEL="eu.anthropic.claude-3-haiku-20240307-v1:0"
 

.github/dependabot.yml

@@ -12,7 +12,7 @@ updates:
     schedule:
       interval: "daily"
     cooldown:
-      default-days: 5
+      default-days: 7
     versioning-strategy: "increase"
     commit-message:
       prefix: chore
@@ -54,6 +54,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 8
     commit-message:
       prefix: ci
       include: scope

.github/workflows/_deploy_ecs_service.yml

@@ -10,10 +10,21 @@ on:
         type: string
         description: Name of the service to be deployed, e.g. web-ingestion, web, or worker.
         required: true
+    # Environment secrets don't auto-resolve in reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      SENTRY_AUTH_TOKEN:
+        required: false
 jobs:
   ecs-deploy:
     runs-on: blacksmith-4vcpu-ubuntu-2404
     environment: ${{ inputs.environment }}
+    permissions:
+      contents: read
     steps:
       - name: Get app name
         uses: winterjung/split@a211a1c46e35fcdc4097d59dd6282d4a9859651b # v2
@@ -22,41 +33,57 @@ jobs:
           msg: ${{ inputs.service }}
           separator: "-"
       - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Authenticate with AWS
         # GitHub/AWS recommend to use OIDC here: https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc
         # Probably more painful to configure, but would remove all long-lived credentials.
-        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_REGION }}
       - name: Login to AWS ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2
+        uses: aws-actions/amazon-ecr-login@f2e9fc6c2b355c1890b65e6f6f0e2ac3e6e22f78 # v2
       - name: Build, tag, and push Docker image
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           REPOSITORY: ${{ steps.split.outputs._0 }}
           IMAGE_TAG: ${{ github.sha }}
+          STEPS_SPLIT_OUTPUTS__0: ${{ steps.split.outputs._0 }}
+          VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: ${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }}
+          VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }}
+          VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT: ${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }}
+          VARS_NEXT_PUBLIC_DEMO_ORG_ID: ${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }}
+          VARS_NEXT_PUBLIC_DEMO_PROJECT_ID: ${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }}
+          VARS_NEXT_PUBLIC_SENTRY_DSN: ${{ vars.NEXT_PUBLIC_SENTRY_DSN }}
+          VARS_NEXT_PUBLIC_POSTHOG_KEY: ${{ vars.NEXT_PUBLIC_POSTHOG_KEY }}
+          VARS_NEXT_PUBLIC_POSTHOG_HOST: ${{ vars.NEXT_PUBLIC_POSTHOG_HOST }}
+          VARS_NEXT_PUBLIC_PLAIN_APP_ID: ${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }}
+          VARS_SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          VARS_SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
+          VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }}
+          SECRETS_SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
         run: |
           docker build \
             -t $REGISTRY/$REPOSITORY:$IMAGE_TAG \
-            -f ./${{ steps.split.outputs._0 }}/Dockerfile \
-            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }} \
-            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }} \
-            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }} \
-            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }} \
-            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }} \
-            --build-arg NEXT_PUBLIC_SENTRY_DSN=${{ vars.NEXT_PUBLIC_SENTRY_DSN }} \
-            --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
-            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }} \
-            --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
-            --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
-            --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
-            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
+            -f ./${STEPS_SPLIT_OUTPUTS__0}/Dockerfile \
+            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION} \
+            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE} \
+            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT} \
+            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${VARS_NEXT_PUBLIC_DEMO_ORG_ID} \
+            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${VARS_NEXT_PUBLIC_DEMO_PROJECT_ID} \
+            --build-arg NEXT_PUBLIC_SENTRY_DSN=${VARS_NEXT_PUBLIC_SENTRY_DSN} \
+            --build-arg NEXT_PUBLIC_BUILD_ID=${IMAGE_TAG} \
+            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${VARS_NEXT_PUBLIC_POSTHOG_KEY} \
+            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${VARS_NEXT_PUBLIC_POSTHOG_HOST} \
+            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${VARS_NEXT_PUBLIC_PLAIN_APP_ID} \
+            --build-arg SENTRY_AUTH_TOKEN=${SECRETS_SENTRY_AUTH_TOKEN} \
+            --build-arg SENTRY_ORG=${VARS_SENTRY_ORG} \
+            --build-arg SENTRY_PROJECT=${VARS_SENTRY_PROJECT} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
       - name: Render AWS ECS Task Definition

.github/workflows/cla-assistant.yml

@@ -9,6 +9,8 @@ on:
   issue_comment:
     types: [created]
 
+permissions: {}
+
 jobs:
   retrigger_cla:
     # Only run on PR comments (not issue comments) with the /check-cla command

.github/workflows/claude-review-maintainer-prs.yml

@@ -1,14 +1,15 @@
 name: Claude Review on Maintainer PRs
 
 on:
-  pull_request_target:
+  pull_request:
     types:
       - opened
       - ready_for_review
 
 jobs:
   comment:
-    if: github.event.pull_request.draft == false
+    # Only run on PRs that are not drafts and are from the same repository (i.e., not from forks)
+    if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     permissions:
       issues: write
@@ -16,7 +17,7 @@ jobs:
     steps:
       - name: Check author permission and existing review request
         id: check
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const owner = context.repo.owner;
@@ -57,7 +58,7 @@ jobs:
 
       - name: Add Claude review comment
         if: steps.check.outputs.should_comment == 'true'
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             await github.rest.issues.createComment({

.github/workflows/codeql.yml

@@ -55,11 +55,13 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
       - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
@@ -87,6 +89,6 @@ jobs:
           exit 1
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/codespell.yml

@@ -22,6 +22,8 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Codespell
-        uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # v2
+        uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2

.github/workflows/dependabot-rebase-stale.yml

@@ -6,6 +6,8 @@ on:
       - main
   workflow_dispatch:
 
+permissions: {}
+
 jobs:
   rebase-dependabot:
     runs-on: ubuntu-latest

.github/workflows/deploy.yml

@@ -27,6 +27,8 @@ on:
           - prod-jp
         required: true
 
+permissions: {}
+
 concurrency:
   # Support concurrent `push` and `workflow_dispatch`` actions
   group: deploy-${{ github.event_name }}-${{ github.ref }}
@@ -41,7 +43,7 @@ jobs:
     steps:
       - name: Get affected services
         id: affected-services
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             if (context.eventName === "workflow_dispatch") {
@@ -56,7 +58,7 @@ jobs:
             return "[]"
           result-encoding: string
       - name: Print services to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           services: ${{ steps.affected-services.outputs.result }}
         with:
@@ -71,7 +73,7 @@ jobs:
     steps:
       - name: Get affected environments
         id: affected-environments
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             if (context.eventName === "workflow_dispatch") {
@@ -88,7 +90,7 @@ jobs:
             return "[]"
           result-encoding: string
       - name: Print environments to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           environments: ${{ steps.affected-environments.outputs.result }}
         with:
@@ -99,7 +101,14 @@ jobs:
   ecs-deploy:
     uses: ./.github/workflows/_deploy_ecs_service.yml
     needs: [affected-services, affected-environments]
-    secrets: inherit
+    permissions:
+      contents: read
+    # Environment secrets must be passed explicitly to reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
     strategy:
       matrix:
         service: ${{ fromJson(needs.affected-services.outputs.services) }}

.github/workflows/licencecheck.yml

@@ -9,15 +9,21 @@ on:
     branches:
       - "main"
 
+permissions:
+  contents: read
+  checks: write # Needed to create a check run for the license compliance check results
+
 jobs:
   license_check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Setup node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version: 18
 
@@ -32,7 +38,7 @@ jobs:
 
       - name: Check license-checker CSV file without headers
         id: license_check_report
-        uses: pilosus/action-pip-license-checker@cc7a461bfa27b44ad187b8578c881ef5138c13fd # v2
+        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
         with:
           external: "npm-license-checker.csv"
           external-format: "csv"
@@ -44,6 +50,8 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Echo error
         if: failure()
-        run: echo "::error::${{ steps.license_check_report.outputs.report }}"
+        run: echo "::error::${STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT}"
+        env:
+          STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT: ${{ steps.license_check_report.outputs.report }}
       - name: Delete license-checker CSV file
         run: rm npm-license-checker.csv

.github/workflows/pipeline.yml

@@ -12,6 +12,9 @@ on:
     branches:
       - "**"
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -24,19 +27,39 @@ jobs:
       llm_connections_changed: ${{ steps.filter.outputs.llm_connections }}
     timeout-minutes: 15
     permissions:
+      contents: read
       pull-requests: read
     steps:
+      # Replaces fkirc/skip-duplicate-actions — skips runs whose git tree
+      # was already tested in a prior successful run of this workflow.
       - id: skip_check
-        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5
-        with:
-          do_not_skip: '["workflow_dispatch"]'
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          CURRENT_TREE=$(gh api "repos/${{ github.repository }}/git/commits/${{ github.sha }}" --jq '.tree.sha')
+
+          MATCH=$(gh api "repos/${{ github.repository }}/actions/workflows/pipeline.yml/runs?status=success&per_page=20" \
+            --jq "[.workflow_runs[] | select(.id != ${{ github.run_id }} and .head_commit.tree_id == \"$CURRENT_TREE\")] | first | .head_sha // empty")
+
+          if [[ -n "$MATCH" ]]; then
+            echo "::notice::Tree $CURRENT_TREE already tested in a prior successful run (commit $MATCH) — skipping"
+            echo "should_skip=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           # Recommended for paths-filter action
           # may save additional git fetch roundtrip if
           # merge-base is found within latest N commits
           fetch-depth: 20
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
+          persist-credentials: false
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
         id: filter
         with:
           filters: |
@@ -50,17 +73,19 @@ jobs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] lint job dependency cache only; no released artifacts are built or published from this cached state
         with:
           node-version: 24
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] lint cache only; publish jobs rebuild artifacts and do not restore this cache
         with:
           path: .turbo
           key: ${{ runner.os }}-turbo-lint-${{ github.sha }}
@@ -82,13 +107,14 @@ jobs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] prettier check dependency cache only; no released artifacts are built or published from this cached state
         with:
           node-version: 24
           cache: "pnpm"
@@ -129,10 +155,12 @@ jobs:
       DOCKER_COMPOSE_FILE: docker-compose.build.yml
     steps:
       - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
@@ -178,7 +206,7 @@ jobs:
           done
       - name: Upload docker diagnostics
         if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: test-docker-build-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
           path: /tmp/docker-diagnostics
@@ -197,23 +225,25 @@ jobs:
         deploy-mode: ["", "-azure", "-redis-cluster"]
         shard: [1, 2, 3]
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Install golang-migrate for Clickhouse migrations
         run: |
           curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] test job dependency cache only; no released artifacts are built or published from this cached state
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -232,14 +262,14 @@ jobs:
           echo "ADMIN_API_KEY=admin-api-key" >> .env
           echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
       - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test job cache only; publish jobs rebuild artifacts and do not restore this cache
         with:
           path: .turbo
           key: ${{ runner.os }}-turbo-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-turbo-
       - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test-only Next.js cache; not consumed by artifact publishing or release jobs
         with:
           path: |
             ~/.npm
@@ -310,18 +340,20 @@ jobs:
         postgres-version: [12, 15]
         deploy-mode: ["", "-azure", "-redis-cluster"]
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] worker test dependency cache only; no release artifacts are produced from this cache
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
@@ -381,18 +413,20 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/') || (needs.pre-job.outputs.should_skip != 'true' && (needs.pre-job.outputs.llm_connections_changed == 'true' || github.event_name == 'workflow_dispatch'))
     name: test-worker-llm-connections (node24, pg15)
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Use Node.js 24
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] llm-connection test dependency cache only; privileged secrets are used here, but this cache is not consumed by artifact publishing
         with:
           node-version: 24
           cache: "pnpm"
@@ -442,6 +476,7 @@ jobs:
           LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID }}
           LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY }}
           LANGFUSE_LLM_CONNECTION_BEDROCK_REGION: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY }}
           LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY }}
           LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY }}
 
@@ -451,23 +486,25 @@ jobs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] e2e dependency cache only; release images are rebuilt later without restoring this cache
         with:
           node-version: 24
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
       - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e cache only; no published artifact path restores this cache
         with:
           path: .turbo
           key: ${{ runner.os }}-turbo-e2e-${{ github.sha }}
@@ -475,7 +512,7 @@ jobs:
             ${{ runner.os }}-turbo-e2e-
             ${{ runner.os }}-turbo-
       - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e-only Next.js cache; not part of any artifact build or publishing flow
         with:
           path: |
             ~/.npm
@@ -528,17 +565,19 @@ jobs:
       - pre-job
     if: needs.pre-job.outputs.should_skip != 'true'
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Login to Docker Hub
         if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
           password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] server e2e dependency cache only; release/publish steps rebuild separately
         with:
           node-version: 24
           cache: "pnpm"
@@ -589,7 +628,8 @@ jobs:
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [
+    needs:
+      [
         lint,
         prettier-check,
         tests-web,
@@ -620,16 +660,58 @@ jobs:
         run: exit 1
         working-directory: .
       - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        if: failure() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
         with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ CI failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ CI Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch/Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }
       - name: Output job results
         run: |
-          echo "Job results: ${{ steps.set-success-output.outputs.success }}"
+          echo "Job results: ${STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS}"
+        env:
+          STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}
 
   build-docker-image-release:
     needs: all-ci-passed
@@ -671,17 +753,19 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
       - name: Set NEXT_PUBLIC_BUILD_ID
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Log in to the GitHub Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Log in to Docker Hub
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -689,7 +773,7 @@ jobs:
         uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1
       - name: Extract metadata (labels) for Docker
         id: meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
         with:
           images: |
             ghcr.io/langfuse/${{ matrix.image_name }}
@@ -727,17 +811,21 @@ jobs:
           provenance: false
           sbom: false
       - name: Record pushed digests
+        env:
+          DIGEST_GHCR: ${{ steps.build-ghcr.outputs.digest }}
+          DIGEST_DOCKERHUB: ${{ steps.build-dockerhub.outputs.digest }}
+          PLATFORM_TAG: ${{ matrix.platform_tag }}
         run: |
-          if [ -z '${{ steps.build-ghcr.outputs.digest }}' ] || [ -z '${{ steps.build-dockerhub.outputs.digest }}' ]; then
+          if [ -z "$DIGEST_GHCR" ] || [ -z "$DIGEST_DOCKERHUB" ]; then
             echo "Missing registry digest output"
             exit 1
           fi
 
           mkdir -p "$RUNNER_TEMP/digests/ghcr" "$RUNNER_TEMP/digests/dockerhub"
-          printf '%s\n' '${{ steps.build-ghcr.outputs.digest }}' > "$RUNNER_TEMP/digests/ghcr/${{ matrix.platform_tag }}.txt"
-          printf '%s\n' '${{ steps.build-dockerhub.outputs.digest }}' > "$RUNNER_TEMP/digests/dockerhub/${{ matrix.platform_tag }}.txt"
+          printf '%s\n' "$DIGEST_GHCR" > "$RUNNER_TEMP/digests/ghcr/${PLATFORM_TAG}.txt"
+          printf '%s\n' "$DIGEST_DOCKERHUB" > "$RUNNER_TEMP/digests/dockerhub/${PLATFORM_TAG}.txt"
       - name: Upload release digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: release-digests-${{ matrix.component }}-${{ matrix.platform_tag }}
           path: |
@@ -765,27 +853,27 @@ jobs:
 
     steps:
       - name: Log in to the GitHub Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Log in to Docker Hub
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Setup Blacksmith Builder
         uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1
       - name: Download release digests
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           pattern: release-digests-${{ matrix.component }}-*
           merge-multiple: true
           path: ${{ runner.temp }}/digests
       - name: Extract metadata (tags) for GitHub Container Registry
         id: meta-ghcr
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
         with:
           images: ghcr.io/langfuse/${{ matrix.image_name }}
           flavor: |
@@ -800,7 +888,7 @@ jobs:
             type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Extract metadata (tags) for Docker Hub
         id: meta-dockerhub
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
         with:
           images: langfuse/${{ matrix.image_name }}
           flavor: |
@@ -814,6 +902,10 @@ jobs:
             type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
             type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Publish multi-platform manifest to GitHub Container Registry
+        env:
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
         run: |
           ghcr_tags=()
           ghcr_sources=()
@@ -821,23 +913,27 @@ jobs:
           while IFS= read -r tag; do
             [ -n "$tag" ] || continue
             ghcr_tags+=("-t" "$tag")
-          done <<'EOF'
-          ${{ steps.meta-ghcr.outputs.tags }}
+          done <<EOF
+          ${STEPS_META_GHCR_OUTPUTS_TAGS}
           EOF
 
           shopt -s nullglob
           for digest_file in "$RUNNER_TEMP"/digests/ghcr/*.txt; do
             digest="$(cat "$digest_file")"
-            ghcr_sources+=("ghcr.io/langfuse/${{ matrix.image_name }}@$digest")
+            ghcr_sources+=("ghcr.io/langfuse/${IMAGE_NAME}@$digest")
           done
 
           if [ "${#ghcr_sources[@]}" -lt 2 ]; then
-            echo "Expected amd64 and arm64 GHCR digests for ${{ matrix.component }}"
+            echo "Expected amd64 and arm64 GHCR digests for $COMPONENT"
             exit 1
           fi
 
           docker buildx imagetools create "${ghcr_tags[@]}" "${ghcr_sources[@]}"
       - name: Publish multi-platform manifest to Docker Hub
+        env:
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
         run: |
           dockerhub_tags=()
           dockerhub_sources=()
@@ -845,29 +941,32 @@ jobs:
           while IFS= read -r tag; do
             [ -n "$tag" ] || continue
             dockerhub_tags+=("-t" "$tag")
-          done <<'EOF'
-          ${{ steps.meta-dockerhub.outputs.tags }}
+          done <<EOF
+          ${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}
           EOF
 
           shopt -s nullglob
           for digest_file in "$RUNNER_TEMP"/digests/dockerhub/*.txt; do
             digest="$(cat "$digest_file")"
-            dockerhub_sources+=("langfuse/${{ matrix.image_name }}@$digest")
+            dockerhub_sources+=("langfuse/${IMAGE_NAME}@$digest")
           done
 
           if [ "${#dockerhub_sources[@]}" -lt 2 ]; then
-            echo "Expected amd64 and arm64 Docker Hub digests for ${{ matrix.component }}"
+            echo "Expected amd64 and arm64 Docker Hub digests for $COMPONENT"
             exit 1
           fi
 
           docker buildx imagetools create "${dockerhub_tags[@]}" "${dockerhub_sources[@]}"
       - name: Inspect published manifests
         run: |
-          ghcr_first_tag="$(printf '%s\n' "${{ steps.meta-ghcr.outputs.tags }}" | sed -n '1p')"
-          dockerhub_first_tag="$(printf '%s\n' "${{ steps.meta-dockerhub.outputs.tags }}" | sed -n '1p')"
+          ghcr_first_tag="$(printf '%s\n' "${STEPS_META_GHCR_OUTPUTS_TAGS}" | sed -n '1p')"
+          dockerhub_first_tag="$(printf '%s\n' "${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}" | sed -n '1p')"
 
           docker buildx imagetools inspect "$ghcr_first_tag"
           docker buildx imagetools inspect "$dockerhub_first_tag"
+        env:
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}
 
   notify-docker-image-release:
     needs:
@@ -880,10 +979,50 @@ jobs:
         if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: exit 1
       - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always()
+        if: failure()
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
         with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ Docker release failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ Docker Release Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }

.github/workflows/promote-main-to-production.yml

@@ -12,6 +12,8 @@ concurrency:
   group: promote-main-to-production
   cancel-in-progress: false
 
+permissions: {}
+
 jobs:
   promote:
     runs-on: ubuntu-latest
@@ -19,16 +21,19 @@ jobs:
     steps:
       - name: Validate confirmation
         run: |
-          if [ "${{ github.event.inputs.confirm }}" != "promote" ]; then
+          if [ "${INPUT_CONFIRM}" != "promote" ]; then
             echo "Input 'confirm' must be 'promote'."
             exit 1
           fi
+        env:
+          INPUT_CONFIRM: ${{ github.event.inputs.confirm }}
 
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           ref: main
           fetch-depth: 0
           token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false
 
       - name: Print commit refs
         run: |
@@ -41,4 +46,6 @@ jobs:
           fi
 
       - name: Force push main to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release.yml

@@ -5,15 +5,20 @@ on:
     tags:
       - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
+permissions: {}
+
 jobs:
   release:
     runs-on: ubuntu-latest
     environment: "protected branches"
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           ref: main # Always checkout main even for tagged releases
           fetch-depth: 0
           token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false
       - name: Push to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/sdk-api-spec.yml

@@ -12,20 +12,26 @@ concurrency:
   group: sdk-api-spec-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   generate-sdk-api-specs:
     runs-on: ubuntu-latest
+    environment: "protected branches"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
       - name: Install pnpm
-        uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b # v2
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
         with:
           version: 10.33.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version: "24"
           cache: "pnpm"
@@ -39,7 +45,7 @@ jobs:
         run: npx fern-api generate --api server --force
 
       - name: Install uv
-        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
         with:
           version: "0.11.2"
 

.github/workflows/snyk-web.yml

@@ -3,46 +3,55 @@ on:
   push:
     branches: ["production", "main"]
 
+permissions:
+  contents: read
+  security-events: write
+
 jobs:
   snyk:
     runs-on: ubuntu-latest
+    environment: snyk
     steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
       - name: Run Snyk to check Docker image for vulnerabilities
         # Snyk can be used to break the build when it detects vulnerabilities.
         # In this case we want to upload the issues to GitHub Code Scanning
         continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
         env:
           # In order to use the Snyk Action you will need to have a Snyk API token.
           # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
           # or you can sign up for free at https://snyk.io/login
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse \
+            --file=web/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
         if: always()
         run: |
           if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
             echo "SARIF file fixed"
           else
             echo "No SARIF file found"
           fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
       - name: Upload result to GitHub Code Scanning
         uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
         if: always()
         with:
           sarif_file: snyk.sarif
+          category: snyk-container-web
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif

.github/workflows/snyk-worker.yml

@@ -3,46 +3,55 @@ on:
   push:
     branches: ["production", "main"]
 
+permissions:
+  contents: read
+  security-events: write
+
 jobs:
   snyk:
     runs-on: ubuntu-latest
+    environment: snyk
     steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
       - name: Run Snyk to check Docker image for vulnerabilities
         # Snyk can be used to break the build when it detects vulnerabilities.
         # In this case we want to upload the issues to GitHub Code Scanning
         continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
         env:
           # In order to use the Snyk Action you will need to have a Snyk API token.
           # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
           # or you can sign up for free at https://snyk.io/login
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse-worker
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse-worker \
+            --file=worker/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
         if: always()
         run: |
           if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
             echo "SARIF file fixed"
           else
             echo "No SARIF file found"
           fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
       - name: Upload result to GitHub Code Scanning
         uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
         if: always()
         with:
           sarif_file: snyk.sarif
+          category: snyk-container-worker
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif

.github/workflows/stale_issues.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
         with:
           days-before-issue-stale: 30
           days-before-issue-close: 14

.github/workflows/zizmor.yml

@@ -0,0 +1,35 @@
+---
+name: Check GitHub Actions
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+  merge_group:
+  pull_request:
+    branches:
+      - "main"
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: Check GitHub Actions security
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
+        with:
+          # Means that the action will only report issues, but not fail the workflow.
+          # Blocking merges are handled by rulesets:
+          # https://docs.github.com/en/code-security/concepts/code-scanning/about-code-scanning-alerts#pull-request-check-failures-for-code-scanning-alerts
+          advanced-security: true

.github/zizmor.yml

@@ -0,0 +1,21 @@
+rules:
+  secrets-outside-env:
+    config:
+      allow:
+        # Shared read-only CI credentials used to avoid Docker Hub rate limits in test jobs.
+        - DOCKERHUB_USERNAME_READ
+        - DOCKERHUB_TOKEN_READ
+        # Shared integration-test credentials intentionally kept together for the multi-provider LLM connection test job.
+        - LANGFUSE_LLM_CONNECTION_OPENAI_KEY
+        - LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL
+        - LANGFUSE_LLM_CONNECTION_AZURE_MODEL
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_REGION
+        - LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY
+        - LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY
+        # Fern token is generation-only; GitHub write actions are handled by GH_ACCESS_TOKEN in a protected environment.
+        - FERN_TOKEN

.gitignore

@@ -47,6 +47,7 @@ yarn-error.log*
 !.env.dev-redis-cluster.example
 !.env.prod.example
 !.env.test.example
+!.env.dev-oci.example
 
 # vercel
 .vercel

docker-compose.yml

@@ -31,6 +31,8 @@ services:
       CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
       CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
       LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
+      LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
+      LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
       LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.167.4",
+  "version": "3.168.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/AGENTS.md

@@ -25,6 +25,7 @@ Use root [AGENTS.md](../../AGENTS.md) for monorepo-level rules.
 - Main exports: `src/index.ts`
 - DB clients and types: `src/db.ts`
 - Server exports: `src/server/index.ts`
+- Server cache utilities: `src/server/cache/*`
 - Domain model types: `src/domain/*`
 - Repository layer: `src/server/repositories/*`
 - Queue payload schemas: `src/server/queues.ts`

packages/shared/clickhouse/scripts/dev-tables.sh

@@ -372,7 +372,8 @@ CREATE TABLE IF NOT EXISTS events_core
     INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,
     INDEX idx_updated_at updated_at TYPE minmax GRANULARITY 1,
-    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2
+    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2,
+    INDEX idx_experiment_id experiment_id TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree(event_ts, is_deleted)
 PARTITION BY toYYYYMM(start_time)

packages/shared/package.json

@@ -112,8 +112,11 @@
     "langfuse-langchain": "3.38.20",
     "lodash": "^4.18.1",
     "lossless-json": "^4.1.1",
+    "lru-cache": "^11.2.7",
     "next-auth": "^4.24.13",
     "nodemailer": "^7.0.11",
+    "oci-objectstorage": "^2.125.0",
+    "oci-common": "^2.125.0",
     "safe-regex2": "^5.0.0",
     "undici": "^7.24.6",
     "uuid": "^9.0.1",

packages/shared/scripts/seeder/seed-postgres.ts

@@ -55,7 +55,6 @@ async function main() {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
-      featureFlags: ["experimentsV4Enabled"],
     },
     create: {
       id: seedUserId1,
@@ -63,7 +62,6 @@ async function main() {
       email: "demo@langfuse.com",
       password: await hash("password", 12),
       image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
-      featureFlags: ["experimentsV4Enabled"],
     },
   });
   const user2 = await prisma.user.upsert({

packages/shared/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.167.4";
+export const VERSION = "v3.168.0";

packages/shared/src/domain/observations.ts

@@ -116,6 +116,7 @@ export const EventsObservationSchema = ObservationSchema.extend({
   userId: z.string().nullable(),
   sessionId: z.string().nullable(),
   traceName: z.string().nullable(),
+  tags: z.array(z.string()).optional(),
   bookmarked: z.boolean().optional(),
   public: z.boolean().optional(),
 });

packages/shared/src/env.ts

@@ -57,6 +57,18 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
   LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_TTL_MS: z.coerce
+    .number()
+    .positive()
+    .default(10_000),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_MAX: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(20_000),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(3600), // 1h
   CLICKHOUSE_URL: z.string().url(),
@@ -179,6 +191,21 @@ const EnvSchema = z.object({
     .default("true"),
   LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
+  LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_OCI_AUTH_TYPE: z
+    .enum([
+      "workload_identity",
+      "instance_principal",
+      "resource_principal",
+      "oci_profile",
+      "session_token",
+    ])
+    .optional(),
+  LANGFUSE_OCI_CONFIG_FILE: z.string().optional(),
+  LANGFUSE_OCI_CONFIG_PROFILE: z.string().optional(),
+  NODE_EXTRA_CA_CERTS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
 
   LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG: z

packages/shared/src/features/evals/utilities.ts

@@ -53,14 +53,23 @@ function parseMultiEncodedJson(value: unknown): unknown {
 }
 
 function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
-  // selectedColumn should already be preprocessed by preprocessObjectWithJsonFields
-  // so we can directly use it with JSONPath
+  // JSONPath can only query objects/arrays — return primitives as-is
+  if (typeof selectedColumn !== "object" || selectedColumn === null) {
+    return selectedColumn;
+  }
+
   const result = JSONPath({
     path: jsonSelector,
     json: selectedColumn as any, // JSONPath accepts unknown but types are strict
   });
 
-  return Array.isArray(result) && result.length > 0 ? result[0] : undefined;
+  if (!Array.isArray(result) || result.length === 0) {
+    return undefined;
+  }
+
+  // For single-match queries (e.g. $.name), return the unwrapped value.
+  // For multi-match queries (e.g. $[1:], $[*].name), return the full array.
+  return result.length === 1 ? result[0] : result;
 }
 
 export function extractValueFromObject(
@@ -69,12 +78,7 @@ export function extractValueFromObject(
   jsonSelector?: string,
   parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown,
 ): { value: string; error: Error | null } {
-  let selectedColumn = obj[selectedColumnId];
-
-  // Simple preprocessing: attempt to parse to valid JSON object
-  if (typeof selectedColumn === "string") {
-    selectedColumn = parseMultiEncodedJson(selectedColumn);
-  }
+  const selectedColumn = obj[selectedColumnId];
 
   const jsonParser = parseJson || parseJsonDefault;
 
@@ -82,14 +86,21 @@ export function extractValueFromObject(
   let error: Error | null = null;
 
   if (jsonSelector && selectedColumn) {
+    // Only parse multi-encoded JSON when a selector is present — avoids
+    // mutating formatting (e.g. whitespace) for the no-selector passthrough.
+    const parsed =
+      typeof selectedColumn === "string"
+        ? parseMultiEncodedJson(selectedColumn)
+        : selectedColumn;
+
     try {
-      jsonSelectedColumn = jsonParser(selectedColumn, jsonSelector);
+      jsonSelectedColumn = jsonParser(parsed, jsonSelector);
     } catch (err) {
       error =
         err instanceof Error
           ? err
           : new Error("There was an unknown error parsing the JSON");
-      jsonSelectedColumn = selectedColumn; // Fallback to original value
+      jsonSelectedColumn = selectedColumn; // Fallback to raw original value
     }
   } else {
     jsonSelectedColumn = selectedColumn;

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -11,12 +11,25 @@ export const VERTEXAI_USE_DEFAULT_CREDENTIALS =
 export const BedrockConfigSchema = z.object({ region: z.string() });
 export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
 
-export const BedrockCredentialSchema = z
+export const BedrockAccessKeysSchema = z
   .object({
-    accessKeyId: z.string(),
-    secretAccessKey: z.string(),
+    accessKeyId: z.string().min(1),
+    secretAccessKey: z.string().min(1),
   })
-  .optional();
+  .strict();
+export type BedrockAccessKeys = z.infer<typeof BedrockAccessKeysSchema>;
+
+export const BedrockApiKeySchema = z
+  .object({
+    apiKey: z.string().min(1),
+  })
+  .strict();
+export type BedrockApiKey = z.infer<typeof BedrockApiKeySchema>;
+
+export const BedrockCredentialSchema = z.union([
+  BedrockAccessKeysSchema,
+  BedrockApiKeySchema,
+]);
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
 
 export const VertexAIConfigSchema = z

packages/shared/src/server/cache/index.ts

@@ -0,0 +1 @@
+export * from "./localCache";

packages/shared/src/server/cache/localCache.ts

@@ -0,0 +1,172 @@
+import { LRUCache } from "lru-cache";
+import { logger } from "../logger";
+import { recordGauge, recordIncrement } from "../instrumentation";
+
+export type LocalCacheLoadResult<V> = {
+  value: V | undefined;
+  ttlMs?: number;
+  source?: string;
+};
+
+export type LocalCacheConfig = {
+  namespace: string;
+  enabled: boolean;
+  ttlMs: number;
+  max: number;
+};
+
+export class LocalCache<V extends {}> {
+  private readonly config: LocalCacheConfig;
+  private readonly cache: LRUCache<string, V>;
+
+  constructor(config: LocalCacheConfig) {
+    this.config = config;
+    const dispose: LRUCache.Disposer<string, V> = (_value, _key, reason) => {
+      if (reason === "evict") {
+        this.record("evict");
+        this.recordSizeMetrics();
+      }
+    };
+
+    const baseOptions = {
+      ttlAutopurge: false as const,
+      allowStale: false as const,
+      updateAgeOnGet: false as const,
+      updateAgeOnHas: false as const,
+      dispose,
+    };
+
+    this.cache = new LRUCache<string, V>({
+      ...baseOptions,
+      ttl: config.ttlMs,
+      max: config.max,
+    });
+    this.logInfo("Initialized local cache", {
+      enabled: config.enabled,
+      ttlMs: config.ttlMs,
+      max: config.max,
+    });
+  }
+
+  get(key: string): V | undefined {
+    if (!this.config.enabled) {
+      return undefined;
+    }
+
+    const value = this.cache.get(key);
+    this.record(value === undefined ? "miss" : "hit");
+    this.logDebug(
+      value === undefined ? "Local cache miss" : "Local cache hit",
+      {
+        size: this.cache.size,
+        keyLength: key.length,
+      },
+    );
+
+    return value;
+  }
+
+  set(key: string, value: V): void {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    const ttlMs = this.config.ttlMs;
+
+    try {
+      this.cache.set(key, value, { ttl: ttlMs });
+      this.record("set");
+      this.recordSizeMetrics();
+      this.logDebug("Stored local cache entry", {
+        ttlMs,
+        size: this.cache.size,
+        keyLength: key.length,
+      });
+    } catch (error) {
+      logger.error(
+        `Failed to set local cache entry for namespace ${this.config.namespace}`,
+        error,
+      );
+    }
+  }
+
+  clear(): void {
+    this.cache.clear();
+    this.record("clear");
+    this.recordSizeMetrics();
+    this.logDebug("Cleared local cache");
+  }
+
+  async getOrLoad(
+    key: string,
+    loader: () => Promise<LocalCacheLoadResult<V>>,
+  ): Promise<LocalCacheLoadResult<V>> {
+    const cached = this.get(key);
+    if (cached !== undefined) {
+      return { value: cached, source: "local" };
+    }
+
+    if (!this.config.enabled) {
+      this.logDebug("Bypassing disabled local cache", {
+        keyLength: key.length,
+      });
+      return loader();
+    }
+
+    const result = await loader();
+    this.logDebug("Completed local cache load", {
+      source: result.source ?? "unknown",
+      cacheable: result.value !== undefined,
+      ttlMs: result.ttlMs ?? null,
+      keyLength: key.length,
+    });
+
+    if (result.value !== undefined) {
+      this.set(key, result.value);
+    }
+
+    return result;
+  }
+
+  private record(metric: string): void {
+    recordIncrement(`langfuse.local_cache.${metric}`, 1, {
+      namespace: this.config.namespace,
+    });
+  }
+
+  private recordSizeMetrics(): void {
+    recordGauge("langfuse.local_cache.size_entries", this.cache.size, {
+      namespace: this.config.namespace,
+    });
+  }
+
+  private logDebug(message: string, metadata?: Record<string, unknown>): void {
+    if (!logger.isLevelEnabled("debug")) {
+      return;
+    }
+
+    const formattedMetadata =
+      metadata === undefined ? "" : ` ${safeSerialize(metadata)}`;
+
+    logger.debug(
+      `[LocalCache:${this.config.namespace}] ${message}${formattedMetadata}`,
+    );
+  }
+
+  private logInfo(message: string, metadata?: Record<string, unknown>): void {
+    const formattedMetadata =
+      metadata === undefined ? "" : ` ${safeSerialize(metadata)}`;
+
+    logger.info(
+      `[LocalCache:${this.config.namespace}] ${message}${formattedMetadata}`,
+    );
+  }
+}
+
+const safeSerialize = (value: unknown): string => {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return "[unserializable]";
+  }
+};

packages/shared/src/server/index.ts

@@ -1,4 +1,5 @@
 export * from "./services/StorageService";
+export * from "./cache";
 export * from "./services/BufferedStreamUploader";
 export * from "./services/S3ChunkedUploadStrategy";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
@@ -28,6 +29,7 @@ export * from "./llm/fetchLLMCompletion";
 export * from "./llm/errors";
 export * from "./llm/utils";
 export * from "./llm/types";
+export * from "./llm/internalTraceEvents";
 export * from "./llm/compileChatMessages";
 export * from "./llm/testModelCall";
 export * from "./llm/baseUrlValidation";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -8,6 +8,7 @@ import {
   safeMultiDel,
   scanKeys,
 } from "../";
+import { LocalCache } from "../cache";
 import { env } from "../../env";
 import { Decimal } from "decimal.js";
 import { prisma } from "../../db";
@@ -24,6 +25,22 @@ export type ModelWithPrices = {
 };
 
 const MODEL_MATCH_CACHE_LOCKED_KEY = "LOCK:model-match-clear";
+const DEFAULT_LOCAL_CACHE_MODEL_MATCH_TTL_MS = 10_000;
+const DEFAULT_LOCAL_CACHE_MODEL_MATCH_MAX = 20_000;
+// This L1 cache is intentionally TTL-only. Cross-container consistency continues
+// to come from Redis invalidation plus the short local TTL.
+const modelMatchLocalCache = new LocalCache<ModelWithPrices>({
+  namespace: "model_match",
+  enabled: env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED === "true",
+  ttlMs: getPositiveNumberOrDefault(
+    env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_TTL_MS,
+    DEFAULT_LOCAL_CACHE_MODEL_MATCH_TTL_MS,
+  ),
+  max: getPositiveNumberOrDefault(
+    env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_MAX,
+    DEFAULT_LOCAL_CACHE_MODEL_MATCH_MAX,
+  ),
+});
 
 export async function findModel(p: ModelMatchProps): Promise<ModelWithPrices> {
   return instrumentAsync(
@@ -33,66 +50,132 @@ export async function findModel(p: ModelMatchProps): Promise<ModelWithPrices> {
     },
     async (span) => {
       if (logger.isLevelEnabled("debug")) {
-        logger.debug(`Finding model for ${JSON.stringify(p)}`);
-      }
-      const cachedResult = await getModelWithPricesFromRedis(p);
-      if (cachedResult) {
-        span.setAttribute("model_match_source", "redis");
-
-        if (cachedResult.model === null) {
-          return { model: null, pricingTiers: [] };
-        } else {
-          logger.debug(
-            `Found model name ${cachedResult.model?.modelName} (id: ${cachedResult.model?.id}) for project ${p.projectId} and model ${p.model}`,
-          );
-          span.setAttribute("matched_model_id", cachedResult.model.id);
-        }
-
-        return cachedResult;
-      }
-
-      // try to find model in Postgres
-      const postgresModel = await findModelInPostgres(p);
-
-      if (postgresModel && env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
-        const pricingTiers = await findPricingTiersForModel(postgresModel.id);
-        await addModelWithPricingTiersToRedis(p, postgresModel, pricingTiers);
-
-        span.setAttribute("matched_model_id", postgresModel.id);
-        span.setAttribute("model_match_source", "postgres");
-        span.setAttribute("model_cache_set", "true");
-
         logger.debug(
-          `Found model name ${postgresModel?.modelName} (id: ${postgresModel?.id}) for project ${p.projectId} and model ${p.model}`,
+          formatModelMatchDebugMessage("Resolving model match", {
+            projectId: p.projectId,
+            model: p.model,
+            localCacheEnabled:
+              env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED === "true",
+            redisCacheEnabled:
+              env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true",
+          }),
         );
-        return { model: postgresModel, pricingTiers };
-      } else if (postgresModel) {
-        const pricingTiers = await findPricingTiersForModel(postgresModel.id);
-        span.setAttribute("matched_model_id", postgresModel.id);
-        span.setAttribute("model_match_source", "postgres");
-        span.setAttribute("model_cache_set", "false");
+      }
+      const localCacheKey = getRedisModelKey(p);
+      const { source, value } = await modelMatchLocalCache.getOrLoad(
+        localCacheKey,
+        async () => {
+          const cachedResult = await getModelWithPricesFromRedis(p);
+          if (cachedResult) {
+            return {
+              value: cachedResult,
+              source: "redis",
+            };
+          }
 
-        logger.debug(
-          `Found model name ${postgresModel?.modelName} (id: ${postgresModel?.id}) for project ${p.projectId} and model ${p.model}`,
-        );
-        return { model: postgresModel, pricingTiers };
-      } else {
-        span.setAttribute("model_match_source", "none");
+          const postgresModel = await findModelInPostgres(p);
+          if (postgresModel) {
+            const pricingTiers = await findPricingTiersForModel(
+              postgresModel.id,
+            );
 
-        if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
-          await addModelNotFoundTokenToRedis(p);
+            if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
+              await addModelWithPricingTiersToRedis(
+                p,
+                postgresModel,
+                pricingTiers,
+              );
+            }
+
+            return {
+              value: { model: postgresModel, pricingTiers },
+              source: "postgres",
+            };
+          }
+
+          if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
+            await addModelNotFoundTokenToRedis(p);
+          }
+
+          return {
+            value: { model: null, pricingTiers: [] },
+            source: "none",
+          };
+        },
+      );
+
+      if (!value || value.model === null) {
+        span.setAttribute("model_match_source", source ?? "none");
+        if (
+          source === "none" &&
+          env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true"
+        ) {
           span.setAttribute("model_cache_set", "true");
         }
 
-        logger.debug(
-          `Model not found for project ${p.projectId} and model ${p.model}`,
-        );
+        if (logger.isLevelEnabled("debug")) {
+          logger.debug(
+            formatModelMatchDebugMessage(
+              "Model match resolved without a model",
+              {
+                projectId: p.projectId,
+                model: p.model,
+                source: source ?? "none",
+                pricingTierCount: 0,
+              },
+            ),
+          );
+        }
+
         return { model: null, pricingTiers: [] };
       }
+
+      span.setAttribute("model_match_source", source ?? "unknown");
+      span.setAttribute("matched_model_id", value.model.id);
+      if (source === "postgres") {
+        span.setAttribute(
+          "model_cache_set",
+          String(env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true"),
+        );
+      }
+
+      if (logger.isLevelEnabled("debug")) {
+        logger.debug(
+          formatModelMatchDebugMessage("Model match resolved", {
+            projectId: p.projectId,
+            model: p.model,
+            source: source ?? "unknown",
+            matchedModelId: value.model.id,
+            matchedModelName: value.model.modelName,
+            pricingTierCount: value.pricingTiers.length,
+          }),
+        );
+      }
+      return value;
     },
   );
 }
 
+const formatModelMatchDebugMessage = (
+  message: string,
+  metadata: Record<string, unknown>,
+): string => {
+  try {
+    return `${message} ${JSON.stringify(metadata)}`;
+  } catch {
+    return `${message} [unserializable]`;
+  }
+};
+
+function getPositiveNumberOrDefault(value: unknown, fallback: number): number {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export const clearModelMatchLocalCache = (): void => {
+  modelMatchLocalCache.clear();
+};
+
 const getModelWithPricesFromRedis = async (
   p: ModelMatchProps,
 ): Promise<ModelWithPrices | null> => {

packages/shared/src/server/instrumentation/index.ts

@@ -9,6 +9,31 @@ import { logger } from "../logger";
 
 // type CallbackFn<T> = () => T;
 
+/**
+ * IORedis request hook that records the full Redis command as a span attribute.
+ * Redacts credentials from AUTH/HELLO and values from API key cache operations.
+ */
+export function ioredisRequestHook(
+  span: opentelemetry.Span,
+  { cmdName, cmdArgs }: { cmdName: string; cmdArgs: unknown[] },
+): void {
+  if (!Array.isArray(cmdArgs) || cmdArgs.length === 0) return;
+  const cmd = cmdName.toUpperCase();
+  // AUTH and HELLO carry raw credentials — redact all args
+  if (cmd === "AUTH" || cmd === "HELLO") {
+    span.setAttribute("redis.full_command", `${cmdName} [REDACTED]`);
+    return;
+  }
+  const args = [...cmdArgs].map(String);
+  // Redact API key cache values: SET [prefix:]api-key:{hash} <json>
+  if (args[0]?.includes("api-key:")) {
+    for (let i = 1; i < args.length; i++) {
+      args[i] = "[REDACTED]";
+    }
+  }
+  span.setAttribute("redis.full_command", `${cmdName} ${args.join(" ")}`);
+}
+
 export type TCarrier = {
   traceparent?: string;
   tracestate?: string;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -19,6 +19,7 @@ import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
 import { env } from "../../env";
 import GCPServiceAccountKeySchema, {
+  BedrockAccessKeysSchema,
   BedrockConfigSchema,
   BedrockCredentialSchema,
   VertexAIConfigSchema,
@@ -53,6 +54,14 @@ import { LLMCompletionError } from "./errors";
 
 export type CompletionWithReasoning = { text: string; reasoning?: string };
 
+const NON_RETRYABLE_LLM_ERROR_PATTERNS = [
+  "Request timed out",
+  "is not valid JSON",
+  "Unterminated string in JSON at position",
+  "TypeError",
+  "reached the end of its life",
+] as const;
+
 const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
 
 // Maps adapters to the content block types that represent "thinking".
@@ -90,6 +99,52 @@ const googleProviderOptionsSchema = z
   })
   .optional();
 
+// For using Bedrock API key in Bearer token format
+const createBedrockBearerAuth = (token: string) => ({
+  clientOptions: {
+    token: { token },
+    authSchemePreference: ["httpBearerAuth"],
+  },
+});
+
+export function resolveBedrockAuth(params: {
+  secretKey: string;
+  allowDefaultCredentials: boolean;
+}): {
+  credentials?: z.infer<typeof BedrockAccessKeysSchema>;
+  clientOptions?: {
+    token: { token: string };
+    authSchemePreference: string[];
+  };
+} {
+  const { secretKey, allowDefaultCredentials } = params;
+
+  if (
+    secretKey === BEDROCK_USE_DEFAULT_CREDENTIALS &&
+    allowDefaultCredentials
+  ) {
+    return {};
+  }
+
+  try {
+    const parsedCredential = BedrockCredentialSchema.parse(
+      JSON.parse(secretKey),
+    );
+
+    if ("apiKey" in parsedCredential) {
+      return createBedrockBearerAuth(parsedCredential.apiKey);
+    }
+
+    return {
+      credentials: parsedCredential,
+    };
+  } catch {
+    throw new Error(
+      "Invalid Bedrock credentials. Expected AWS access key JSON or a Bedrock API key.",
+    );
+  }
+}
+
 type ProcessTracedEvents = () => Promise<void>;
 
 type LLMCompletionParams = {
@@ -363,16 +418,16 @@ export async function fetchLLMCompletion(
     // Handle both explicit credentials and default provider chain
     // Only allow default provider chain in self-hosted or internal AI features
     const isSelfHosted = !isLangfuseCloud;
-    const credentials =
-      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS &&
-      (isSelfHosted || shouldUseLangfuseAPIKey)
-        ? undefined // undefined = use AWS SDK default credential provider chain
-        : BedrockCredentialSchema.parse(JSON.parse(apiKey));
+    const { credentials, clientOptions } = resolveBedrockAuth({
+      secretKey: apiKey,
+      allowDefaultCredentials: isSelfHosted || shouldUseLangfuseAPIKey,
+    });
 
     chatModel = new ChatBedrockConverse({
       model: modelParams.model,
       region,
       credentials,
+      clientOptions,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
@@ -577,20 +632,17 @@ export async function fetchLLMCompletion(
     return completion;
   } catch (e) {
     const responseStatusCode =
-      (e as any)?.response?.status ?? (e as any)?.status ?? 500;
+      (e as any)?.response?.status ??
+      (e as any)?.status ??
+      // Bedrock errors have status code in $metadata.httpStatusCode
+      (e as any)?.$metadata?.httpStatusCode ??
+      500;
     const rawMessage = e instanceof Error ? e.message : String(e);
     const message = extractCleanErrorMessage(rawMessage);
 
     // Check for non-retryable error patterns in message
-    const nonRetryablePatterns = [
-      "Request timed out",
-      "is not valid JSON",
-      "Unterminated string in JSON at position",
-      "TypeError",
-    ];
-
-    const hasNonRetryablePattern = nonRetryablePatterns.some((pattern) =>
-      message.includes(pattern),
+    const hasNonRetryablePattern = NON_RETRYABLE_LLM_ERROR_PATTERNS.some(
+      (pattern) => message.includes(pattern),
     );
 
     // Determine retryability:

packages/shared/src/server/llm/getInternalTracingHandler.ts

@@ -1,88 +1,22 @@
 import CallbackHandler from "langfuse-langchain";
-import { GenerationDetails, TraceSinkParams } from "./types";
+import { ProcessedTraceEvent, TraceSinkParams } from "./types";
+import { buildInternalTraceEventInputs } from "./internalTraceEvents";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
 import { traceException } from "../instrumentation";
 
-type TracedEvent = {
-  type: string;
-  body: Record<string, unknown>;
-};
+export function prepareInternalTraceEvents(params: {
+  events: Array<{
+    type: string;
+    timestamp: string;
+    body: Record<string, unknown>;
+  }>;
+  environment: string;
+  prompt?: TraceSinkParams["prompt"];
+}): ProcessedTraceEvent[] {
+  const { events, environment, prompt } = params;
 
-/**
- * Extracts and merges generation details from a list of processed events.
- * Handles multiple generation-create and generation-update events with the same id.
- *
- * Events are merged following the "last non-null value wins" pattern:
- * - generation-create events contain: id, name, input, metadata
- * - generation-update events contain: output, usage, usageDetails
- *
- * @returns GenerationDetails or null if no generation events found
- */
-export function extractGenerationDetails(
-  processedEvents: Array<{ type: string; body: Record<string, unknown> }>,
-): GenerationDetails | null {
-  // 1. Filter to only generation events
-  const generationEvents = processedEvents.filter(
-    (event) =>
-      event.type === "generation-create" || event.type === "generation-update",
-  );
-
-  if (generationEvents.length === 0) {
-    return null;
-  }
-
-  // 2. Get the generation id from first event
-  const generationId = generationEvents[0].body.id as string;
-  if (!generationId) {
-    return null;
-  }
-
-  // 3. Filter to events for this generation id only
-  const eventsForGeneration = generationEvents.filter(
-    (event) => event.body.id === generationId,
-  );
-
-  // 4. Merge event bodies (last non-null/non-undefined value wins)
-  // Similar to IngestionService pattern but simplified for our use case
-  const mergedBody = eventsForGeneration.reduce(
-    (acc: Record<string, unknown>, event) => {
-      for (const [key, value] of Object.entries(event.body)) {
-        if (value !== undefined && value !== null) {
-          // Special handling for metadata: deep merge
-          if (
-            key === "metadata" &&
-            typeof value === "object" &&
-            !Array.isArray(value)
-          ) {
-            acc[key] = {
-              ...((acc[key] as Record<string, unknown>) || {}),
-              ...(value as Record<string, unknown>),
-            };
-          } else {
-            acc[key] = value;
-          }
-        }
-      }
-      return acc;
-    },
-    { id: generationId },
-  );
-
-  return {
-    observationId: generationId,
-    name: (mergedBody.name as string) || "generation",
-    input: mergedBody.input,
-    output: mergedBody.output,
-    metadata: (mergedBody.metadata as Record<string, unknown>) || {},
-  };
-}
-
-export function prepareTracedEventsForIngestion(
-  events: TracedEvent[],
-  { environment, prompt }: Pick<TraceSinkParams, "environment" | "prompt">,
-): TracedEvent[] {
-  const blockedSpanIds = new Set<string>();
+  const blockedSpanIds = new Set();
   const blockedSpanNames = [
     "RunnableLambda",
     "StructuredOutputParser",
@@ -91,29 +25,27 @@ export function prepareTracedEventsForIngestion(
   ];
 
   for (const event of events) {
-    const eventName = event.body.name;
+    const eventName = "name" in event.body ? event.body.name : "";
 
     if (typeof eventName !== "string" || eventName.length === 0) {
       continue;
     }
 
-    if (
-      blockedSpanNames.includes(eventName) &&
-      typeof event.body.id === "string"
-    ) {
+    if (blockedSpanNames.includes(eventName as string) && "id" in event.body) {
       blockedSpanIds.add(event.body.id);
     }
   }
 
   return events
     .filter((event) => {
-      if (typeof event.body.id === "string") {
+      if ("id" in event.body) {
         return !blockedSpanIds.has(event.body.id);
       }
 
       return true;
     })
     .map((event) => {
+      // Inject environment into all events
       return {
         ...event,
         body: {
@@ -142,7 +74,8 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
   handler: CallbackHandler;
   processTracedEvents: () => Promise<void>;
 } {
-  const { prompt, targetProjectId, environment, userId } = traceSinkParams;
+  const { prompt, targetProjectId, environment, userId, eventsWriter } =
+    traceSinkParams;
   const handler = new CallbackHandler({
     _projectId: targetProjectId,
     _isLocalEventExportEnabled: true,
@@ -155,41 +88,52 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
       const events = await handler.langfuse._exportLocalEvents(
         traceSinkParams.targetProjectId,
       );
+      const processedEvents = prepareInternalTraceEvents({
+        events,
+        environment,
+        prompt,
+      });
 
-      const processedEvents = prepareTracedEventsForIngestion(
-        events as TracedEvent[],
-        {
-          environment,
-          prompt,
-        },
-      );
+      // Legacy write to traces/observations tables
+      try {
+        await processEventBatch(
+          JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
+          {
+            validKey: true as const,
+            scope: {
+              projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
+              accessLevel: "project",
+            } as any,
+          },
+          {
+            isLangfuseInternal: true,
+            forwardToEventsTable: eventsWriter ? false : undefined, // Do not dual write when we already direct event write
+          },
+        );
+      } catch (processingError) {
+        traceException(processingError);
+        logger.error("Failed to process traced events via legacy ingestion", {
+          error: processingError,
+        });
+      }
 
-      await processEventBatch(
-        JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
-        {
-          validKey: true as const,
-          scope: {
-            projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
-            accessLevel: "project",
-          } as any,
-        },
-        {
-          isLangfuseInternal: true,
-        },
-      );
-
-      // Extract generation details and invoke callback (if provided)
-      if (traceSinkParams.onGenerationComplete) {
+      // Direct write to events table
+      if (eventsWriter) {
         try {
-          const generationDetails = extractGenerationDetails(processedEvents);
-          if (generationDetails) {
-            traceSinkParams.onGenerationComplete(generationDetails);
+          const { rootSpanId, eventInputs } = buildInternalTraceEventInputs({
+            processedEvents,
+            traceId: traceSinkParams.traceId,
+            projectId: targetProjectId,
+            experimentContext: eventsWriter.experimentContext,
+          });
+
+          if (eventInputs.length > 0) {
+            await eventsWriter.write({ rootSpanId, eventInputs });
           }
-        } catch (extractionError) {
-          // Don't fail the LLM call due to generation detail extraction errors
-          traceException(extractionError);
-          logger.error("Failed to extract generation details from events", {
-            error: extractionError,
+        } catch (writeError) {
+          traceException(writeError);
+          logger.error("Failed to direct-write internal traced events", {
+            error: writeError,
           });
         }
       }

packages/shared/src/server/llm/internalTraceEvents.ts

@@ -0,0 +1,417 @@
+import {
+  asBoolean,
+  asNumberRecord,
+  asRecord,
+  asString,
+  asStringArray,
+} from "../../utils/objects";
+import { stringifyValue } from "../../utils/stringChecks";
+import {
+  convertCallsToArrays,
+  convertDefinitionsToMap,
+  extractToolsFromObservation,
+} from "../ingestion/extractToolsBackend";
+import { flattenJsonToPathArrays } from "../otel/utils";
+import type { ProcessedTraceEvent } from "./types";
+
+export const INTERNAL_TRACE_EVENT_SOURCE = "ingestion-api-dual-write";
+export const INTERNAL_TRACE_EXPERIMENT_EVENT_SOURCE =
+  "ingestion-api-dual-write-experiments";
+
+export type InternalTraceExperimentContext = {
+  id: string;
+  name: string;
+  metadata?: Record<string, unknown>;
+  description?: string | null;
+  datasetId: string;
+  itemId: string;
+  itemVersion: string;
+  itemExpectedOutput?: unknown;
+  itemMetadata?: Record<string, unknown> | null;
+};
+
+/**
+ * Flexible input type for writing events to the events table.
+ * This is intentionally loose to allow for iteration as the events
+ * table schema evolves. Only required fields are enforced.
+ */
+export type InternalTraceEventInput = {
+  projectId: string;
+  traceId: string;
+  spanId: string;
+  startTimeISO: string;
+  orgId?: string;
+  parentSpanId?: string;
+  name?: string;
+  type?: string;
+  environment?: string;
+  version?: string;
+  release?: string;
+  endTimeISO: string;
+  completionStartTime?: string;
+  traceName?: string;
+  tags?: string[];
+  bookmarked?: boolean;
+  public?: boolean;
+  userId?: string;
+  sessionId?: string;
+  level?: string;
+  statusMessage?: string;
+  promptId?: string;
+  promptName?: string;
+  promptVersion?: string;
+  modelId?: string;
+  modelName?: string;
+  modelParameters?: string | Record<string, unknown>;
+  providedUsageDetails?: Record<string, number>;
+  usageDetails?: Record<string, number>;
+  providedCostDetails?: Record<string, number>;
+  costDetails?: Record<string, number>;
+  toolDefinitions?: Record<string, string>;
+  toolCalls?: string[];
+  toolCallNames?: string[];
+  input?: string;
+  output?: string;
+  metadata: Record<string, unknown>;
+  source: string;
+  serviceName?: string;
+  serviceVersion?: string;
+  scopeName?: string;
+  scopeVersion?: string;
+  telemetrySdkLanguage?: string;
+  telemetrySdkName?: string;
+  telemetrySdkVersion?: string;
+  blobStorageFilePath?: string;
+  eventRaw?: string;
+  eventBytes?: number;
+  experimentId?: string;
+  experimentName?: string;
+  experimentMetadataNames?: string[];
+  experimentMetadataValues?: Array<string | null | undefined>;
+  experimentDescription?: string;
+  experimentDatasetId?: string;
+  experimentItemId?: string;
+  experimentItemVersion?: string;
+  experimentItemRootSpanId?: string;
+  experimentItemExpectedOutput?: string;
+  experimentItemMetadataNames?: string[];
+  experimentItemMetadataValues?: Array<string | null | undefined>;
+  [key: string]: any;
+};
+
+type InternalTraceSnapshot = {
+  spanId: string;
+  traceId: string;
+  parentSpanId?: string;
+  name?: string;
+  type: "SPAN" | "GENERATION";
+  environment?: string;
+  version?: string;
+  release?: string;
+  startTimeISO?: string;
+  endTimeISO?: string;
+  completionStartTime?: string;
+  level?: string;
+  statusMessage?: string;
+  promptName?: string;
+  promptVersion?: string;
+  modelName?: string;
+  modelParameters?: Record<string, unknown>;
+  providedUsageDetails?: Record<string, number>;
+  providedCostDetails?: Record<string, number>;
+  input?: unknown;
+  output?: unknown;
+  metadata: Record<string, unknown>;
+  tags?: string[];
+  public?: boolean;
+  bookmarked?: boolean;
+  userId?: string;
+  sessionId?: string;
+};
+
+export type MaterializedInternalTrace = {
+  rootSpanId: string;
+  snapshots: InternalTraceSnapshot[];
+};
+
+function isCreateEvent(type: string): boolean {
+  return type.endsWith("-create");
+}
+
+function getSnapshotType(eventType: string): "SPAN" | "GENERATION" {
+  return eventType.startsWith("generation-") ? "GENERATION" : "SPAN";
+}
+
+function getEventTime(
+  event: ProcessedTraceEvent,
+  body: Record<string, unknown>,
+): number {
+  const candidates = [body.startTime, body.timestamp, event.timestamp];
+
+  for (const candidate of candidates) {
+    if (typeof candidate === "string") {
+      const parsed = new Date(candidate).getTime();
+      if (!Number.isNaN(parsed)) {
+        return parsed;
+      }
+    }
+  }
+
+  return 0;
+}
+
+function getTimestampMs(timestamp?: string): number {
+  if (!timestamp) {
+    return 0;
+  }
+
+  const parsed = new Date(timestamp).getTime();
+  return Number.isNaN(parsed) ? 0 : parsed;
+}
+
+function sortEvents(events: ProcessedTraceEvent[]): ProcessedTraceEvent[] {
+  return [...events].sort((left, right) => {
+    const timeDelta =
+      getEventTime(left, left.body) - getEventTime(right, right.body);
+
+    if (timeDelta !== 0) {
+      return timeDelta;
+    }
+
+    if (isCreateEvent(left.type) === isCreateEvent(right.type)) {
+      return 0;
+    }
+
+    return isCreateEvent(left.type) ? -1 : 1;
+  });
+}
+
+function flattenMetadata(value: unknown): {
+  names: string[];
+  values: Array<string | null | undefined>;
+} {
+  const metadata = asRecord(value);
+  return metadata
+    ? flattenJsonToPathArrays(metadata)
+    : { names: [], values: [] };
+}
+
+function mergeSnapshotEvent(
+  snapshot: InternalTraceSnapshot,
+  event: ProcessedTraceEvent,
+): InternalTraceSnapshot {
+  const { body } = event;
+  const startTime = asString(body.startTime);
+  const timestamp = asString(body.timestamp) ?? asString(event.timestamp);
+  const metadata = asRecord(body.metadata);
+
+  return {
+    ...snapshot,
+    traceId: asString(body.traceId) ?? snapshot.traceId,
+    parentSpanId: asString(body.parentObservationId) ?? snapshot.parentSpanId,
+    type:
+      snapshot.type === "GENERATION"
+        ? snapshot.type
+        : getSnapshotType(event.type),
+    name: asString(body.name) ?? snapshot.name,
+    environment: asString(body.environment) ?? snapshot.environment,
+    version: asString(body.version) ?? snapshot.version,
+    release: asString(body.release) ?? snapshot.release,
+    startTimeISO:
+      startTime ?? snapshot.startTimeISO ?? timestamp ?? snapshot.startTimeISO,
+    endTimeISO: asString(body.endTime) ?? snapshot.endTimeISO,
+    completionStartTime:
+      asString(body.completionStartTime) ?? snapshot.completionStartTime,
+    level: asString(body.level) ?? snapshot.level,
+    statusMessage: asString(body.statusMessage) ?? snapshot.statusMessage,
+    promptName: asString(body.promptName) ?? snapshot.promptName,
+    promptVersion:
+      typeof body.promptVersion === "number"
+        ? body.promptVersion.toString()
+        : (asString(body.promptVersion) ?? snapshot.promptVersion),
+    modelName: asString(body.model) ?? snapshot.modelName,
+    modelParameters: asRecord(body.modelParameters) ?? snapshot.modelParameters,
+    providedUsageDetails:
+      asNumberRecord(body.usageDetails) ??
+      asNumberRecord(body.usage) ??
+      snapshot.providedUsageDetails,
+    providedCostDetails:
+      asNumberRecord(body.costDetails) ?? snapshot.providedCostDetails,
+    input:
+      body.input !== undefined && body.input !== null
+        ? body.input
+        : snapshot.input,
+    output:
+      body.output !== undefined && body.output !== null
+        ? body.output
+        : snapshot.output,
+    metadata: metadata
+      ? { ...snapshot.metadata, ...metadata }
+      : snapshot.metadata,
+    tags: asStringArray(body.tags) ?? snapshot.tags,
+    public: asBoolean(body.public) ?? snapshot.public,
+    bookmarked: asBoolean(body.bookmarked) ?? snapshot.bookmarked,
+    userId: asString(body.userId) ?? snapshot.userId,
+    sessionId: asString(body.sessionId) ?? snapshot.sessionId,
+  };
+}
+
+export function materializeInternalTrace(params: {
+  processedEvents: ProcessedTraceEvent[];
+  traceId: string;
+}): MaterializedInternalTrace {
+  const { processedEvents, traceId } = params;
+  const snapshots = new Map<string, InternalTraceSnapshot>();
+  const traceCreateEvent = processedEvents.find(
+    (e) => e.type === "trace-create",
+  );
+  const rootSpanId = asString(traceCreateEvent?.body.id) ?? traceId;
+
+  for (const event of sortEvents(processedEvents)) {
+    const spanId = asString(event.body.id);
+
+    if (!spanId) {
+      continue;
+    }
+
+    const existingSnapshot =
+      snapshots.get(spanId) ??
+      ({
+        spanId,
+        traceId: asString(event.body.traceId) ?? traceId,
+        type: getSnapshotType(event.type),
+        metadata: {},
+      } satisfies InternalTraceSnapshot);
+
+    snapshots.set(spanId, mergeSnapshotEvent(existingSnapshot, event));
+  }
+
+  const orderedSnapshots = [...snapshots.values()].sort((left, right) => {
+    if (left.spanId === rootSpanId) {
+      return -1;
+    }
+
+    if (right.spanId === rootSpanId) {
+      return 1;
+    }
+
+    return (
+      getTimestampMs(left.startTimeISO) - getTimestampMs(right.startTimeISO)
+    );
+  });
+
+  return { rootSpanId, snapshots: orderedSnapshots };
+}
+
+export function buildInternalTraceEventInputs(params: {
+  processedEvents: ProcessedTraceEvent[];
+  traceId: string;
+  projectId: string;
+  experimentContext?: InternalTraceExperimentContext;
+}): {
+  rootSpanId: string;
+  eventInputs: InternalTraceEventInput[];
+} {
+  const { processedEvents, traceId, projectId, experimentContext } = params;
+  // Direct write uses original IDs (observation.id === trace.id for root).
+  // The experiment backfill job skips traces already in events_core via LEFT ANTI JOIN,
+  // so there's no deduplication concern between direct write and backfill.
+  const { rootSpanId, snapshots } = materializeInternalTrace({
+    processedEvents,
+    traceId,
+  });
+  const rootSnapshot = snapshots.find((s) => s.spanId === rootSpanId);
+
+  if (!rootSnapshot) {
+    return { rootSpanId, eventInputs: [] };
+  }
+
+  const experimentMetadata = flattenMetadata(experimentContext?.metadata);
+  const experimentItemMetadata = flattenMetadata(
+    experimentContext?.itemMetadata,
+  );
+  const source = experimentContext
+    ? INTERNAL_TRACE_EXPERIMENT_EVENT_SOURCE
+    : INTERNAL_TRACE_EVENT_SOURCE;
+
+  const eventInputs = snapshots.map((snapshot) => {
+    const { toolDefinitions, toolArguments } = extractToolsFromObservation(
+      snapshot.input,
+      snapshot.output,
+    );
+    const toolCalls = convertCallsToArrays(toolArguments);
+    const isRoot = snapshot.spanId === rootSpanId;
+
+    return {
+      projectId,
+      traceId,
+      spanId: snapshot.spanId,
+      parentSpanId: isRoot ? undefined : (snapshot.parentSpanId ?? rootSpanId),
+      name:
+        snapshot.name ??
+        (snapshot.type === "GENERATION"
+          ? "generation"
+          : (rootSnapshot.name ?? "span")),
+      type: snapshot.type,
+      environment: snapshot.environment ?? rootSnapshot.environment,
+      version: snapshot.version ?? rootSnapshot.version,
+      release: rootSnapshot.release,
+      startTimeISO:
+        snapshot.startTimeISO ??
+        rootSnapshot.startTimeISO ??
+        new Date().toISOString(),
+      endTimeISO:
+        snapshot.endTimeISO ??
+        snapshot.startTimeISO ??
+        rootSnapshot.endTimeISO ??
+        rootSnapshot.startTimeISO ??
+        new Date().toISOString(),
+      completionStartTime: snapshot.completionStartTime,
+      traceName: rootSnapshot.name,
+      tags: rootSnapshot.tags ?? [],
+      bookmarked: rootSnapshot.bookmarked,
+      public: rootSnapshot.public,
+      userId: rootSnapshot.userId,
+      sessionId: rootSnapshot.sessionId,
+      level: snapshot.level ?? "DEFAULT",
+      statusMessage: snapshot.statusMessage,
+      promptName: snapshot.promptName,
+      promptVersion: snapshot.promptVersion,
+      modelName: snapshot.modelName,
+      modelParameters: snapshot.modelParameters,
+      providedUsageDetails: snapshot.providedUsageDetails,
+      providedCostDetails: snapshot.providedCostDetails,
+      toolDefinitions: convertDefinitionsToMap(toolDefinitions),
+      toolCalls: toolCalls.tool_calls,
+      toolCallNames: toolCalls.tool_call_names,
+      input:
+        snapshot.input !== undefined
+          ? stringifyValue(snapshot.input)
+          : undefined,
+      output:
+        snapshot.output !== undefined
+          ? stringifyValue(snapshot.output)
+          : undefined,
+      metadata: snapshot.metadata,
+      source,
+      experimentId: experimentContext?.id,
+      experimentName: experimentContext?.name,
+      experimentMetadataNames: experimentMetadata.names,
+      experimentMetadataValues: experimentMetadata.values,
+      experimentDescription: experimentContext?.description ?? undefined,
+      experimentDatasetId: experimentContext?.datasetId,
+      experimentItemId: experimentContext?.itemId,
+      experimentItemVersion: experimentContext?.itemVersion,
+      experimentItemRootSpanId: experimentContext ? rootSpanId : undefined,
+      experimentItemExpectedOutput:
+        experimentContext?.itemExpectedOutput !== undefined &&
+        experimentContext?.itemExpectedOutput !== null
+          ? stringifyValue(experimentContext.itemExpectedOutput)
+          : undefined,
+      experimentItemMetadataNames: experimentItemMetadata.names,
+      experimentItemMetadataValues: experimentItemMetadata.values,
+    } satisfies InternalTraceEventInput;
+  });
+
+  return { rootSpanId, eventInputs };
+}

packages/shared/src/server/llm/types.ts

@@ -5,6 +5,10 @@ import {
   VertexAIConfigSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { JSONObjectSchema } from "../../utils/zod";
+import type {
+  InternalTraceEventInput,
+  InternalTraceExperimentContext,
+} from "./internalTraceEvents";
 
 // disable lint as this is exported and used in web/worker
 
@@ -429,6 +433,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 export const anthropicModels = [
   "claude-sonnet-4-5-20250929",
   "claude-haiku-4-5-20251001",
+  "claude-opus-4-7",
   "claude-sonnet-4-6",
   "claude-opus-4-6",
   "claude-opus-4-5-20251101",
@@ -533,16 +538,22 @@ export enum LangfuseInternalTraceEnvironment {
   LLMJudge = "langfuse-llm-as-a-judge",
 }
 
+export type ProcessedTraceEvent = {
+  type: string;
+  timestamp: string;
+  body: Record<string, unknown>;
+};
+
 /**
- * Details of a generation extracted from traced events.
- * Used to pass generation information from internal tracing to callbacks.
+ * Configuration for direct writing of trace events to the events table.
+ * Used by internal tracing (prompt experiments, evaluations).
  */
-export type GenerationDetails = {
-  observationId: string;
-  name: string;
-  input: unknown;
-  output: unknown;
-  metadata: Record<string, unknown>;
+export type InternalEventsWriter = {
+  experimentContext?: InternalTraceExperimentContext;
+  write: (params: {
+    rootSpanId: string;
+    eventInputs: InternalTraceEventInput[];
+  }) => Promise<void>;
 };
 
 export type TraceSinkParams = {
@@ -561,8 +572,10 @@ export type TraceSinkParams = {
     version: number;
   };
   /**
-   * Optional callback invoked after the generation events have been processed.
-   * Called with merged generation details (from create + update events).
+   * When provided, traced events are written directly to the events table,
+   * bypassing the legacy traces/observations ingestion pipeline for the events write.
+   * Used for internal tracing (prompt experiments, LLM-as-a-judge evaluations). Traced
+   * events are still written to the legacy traces/observations tables.
    */
-  onGenerationComplete?: (details: GenerationDetails) => void;
+  eventsWriter?: InternalEventsWriter;
 };

packages/shared/src/server/otel/OtelIngestionProcessor.ts

@@ -1315,18 +1315,25 @@ export class OtelIngestionProcessor {
     const keys = Object.keys(input).map((key) => key.replace(`${prefix}.`, ""));
     const useArray = keys.some((key) => key.match(/^\d+\./));
 
+    // Blocklist to prevent prototype pollution via crafted OTel attribute keys
+    const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
     // Helper function to set a value at a nested path
     const setNestedValue = (obj: any, path: string[], value: unknown): void => {
       let current = obj;
       for (let i = 0; i < path.length - 1; i++) {
         const key = path[i];
+        if (DANGEROUS_KEYS.has(key)) return;
         if (!(key in current)) {
           // Check if next key is a number to decide if we need an array or object
           current[key] = /^\d+$/.test(path[i + 1]) ? [] : {};
         }
         current = current[key];
       }
-      current[path[path.length - 1]] = value;
+      const finalKey = path[path.length - 1];
+      if (!DANGEROUS_KEYS.has(finalKey)) {
+        current[finalKey] = value;
+      }
     };
 
     if (useArray) {
@@ -1335,7 +1342,7 @@ export class OtelIngestionProcessor {
         const pathParts = key.split(".");
         const index = parseInt(pathParts[0], 10);
         if (!result[index]) {
-          result[index] = {};
+          result[index] = Object.create(null);
         }
         if (pathParts.length === 2) {
           // Simple case: 0.content -> result[0].content
@@ -1351,7 +1358,7 @@ export class OtelIngestionProcessor {
       }
       return result;
     } else {
-      const result: Record<string, unknown> = {};
+      const result: Record<string, unknown> = Object.create(null);
       for (const key of keys) {
         const pathParts = key.split(".");
         if (pathParts.length === 1) {
@@ -2476,51 +2483,120 @@ export class OtelIngestionProcessor {
     }
 
     if (instrumentationScopeName === "pydantic-ai") {
-      const inputTokens = attributes["gen_ai.usage.input_tokens"];
-      const outputTokens = attributes["gen_ai.usage.output_tokens"];
-      const cacheReadTokens =
-        attributes["gen_ai.usage.cache_read_tokens"] ??
-        attributes["gen_ai.usage.details.cache_read_input_tokens"];
-      const cacheWriteTokens =
-        attributes["gen_ai.usage.cache_write_tokens"] ??
-        attributes["gen_ai.usage.details.cache_creation_input_tokens"];
-
-      return {
-        input: inputTokens,
-        output: outputTokens,
-        input_cache_read: cacheReadTokens,
-        input_cache_creation: cacheWriteTokens,
-      };
+      const usageDetails = this.extractGenericGenAiUsageDetails(attributes);
+      if (Object.keys(usageDetails).length > 0) return usageDetails;
     }
 
+    return this.extractGenericGenAiUsageDetails(attributes);
+  }
+
+  private extractGenericGenAiUsageDetails(
+    attributes: Record<string, unknown>,
+  ): Record<string, number> {
     const usageDetails = Object.keys(attributes).filter(
       (key) =>
         (key.startsWith("gen_ai.usage.") && key !== "gen_ai.usage.cost") ||
-        key.startsWith("llm.token_count"),
+        key.startsWith("llm.token_count."),
     );
 
-    const usageDetailKeyMapping: Record<string, string> = {
-      prompt_tokens: "input",
-      completion_tokens: "output",
-      total_tokens: "total",
-      input_tokens: "input",
-      output_tokens: "output",
-      prompt: "input",
-      completion: "output",
-    };
+    if (usageDetails.length === 0) return {};
 
-    return usageDetails.reduce((acc: any, key) => {
-      const usageDetailKey = key
-        .replace("gen_ai.usage.", "")
-        .replace("llm.token_count.", "");
-      const mappedUsageDetailKey =
-        usageDetailKeyMapping[usageDetailKey] ?? usageDetailKey;
-      const value = Number(attributes[key]);
-      if (!Number.isNaN(value)) {
-        acc[mappedUsageDetailKey] = value;
-      }
-      return acc;
-    }, {});
+    const rawUsageDetails = usageDetails.reduce(
+      (acc: Record<string, number>, key) => {
+        const usageDetailKey = key
+          .replace("gen_ai.usage.", "")
+          .replace("llm.token_count.", "");
+        const value = Number(attributes[key]);
+
+        if (!Number.isNaN(value)) {
+          acc[usageDetailKey] = value;
+        }
+
+        return acc;
+      },
+      {},
+    );
+
+    const inputTokens =
+      rawUsageDetails["prompt_tokens"] ??
+      rawUsageDetails["input_tokens"] ??
+      rawUsageDetails["prompt"];
+    const outputTokens =
+      rawUsageDetails["completion_tokens"] ??
+      rawUsageDetails["output_tokens"] ??
+      rawUsageDetails["completion"];
+    const totalTokens =
+      rawUsageDetails["total_tokens"] ?? rawUsageDetails["total"];
+    const cacheReadTokens =
+      rawUsageDetails["cache_read.input_tokens"] ??
+      rawUsageDetails["cache_read_tokens"] ??
+      rawUsageDetails["details.cache_read_tokens"] ??
+      rawUsageDetails["details.cache_read_input_tokens"];
+    const cacheCreationTokens =
+      rawUsageDetails["cache_creation.input_tokens"] ??
+      rawUsageDetails["cache_write_tokens"] ??
+      rawUsageDetails["details.cache_write_tokens"] ??
+      rawUsageDetails["details.cache_creation_input_tokens"];
+
+    const normalizedUsageDetails = Object.entries(rawUsageDetails).reduce(
+      (acc: Record<string, number>, [key, value]) => {
+        if (
+          [
+            "prompt_tokens",
+            "input_tokens",
+            "prompt",
+            "completion_tokens",
+            "output_tokens",
+            "completion",
+            "total_tokens",
+            "total",
+            "cache_read.input_tokens",
+            "cache_read_tokens",
+            "details.cache_read_tokens",
+            "details.cache_read_input_tokens",
+            "cache_creation.input_tokens",
+            "cache_write_tokens",
+            "details.cache_write_tokens",
+            "details.cache_creation_input_tokens",
+          ].includes(key)
+        ) {
+          return acc;
+        }
+
+        const normalizedKey = key.startsWith("details.")
+          ? key.replace("details.", "")
+          : key;
+
+        acc[normalizedKey] = value;
+        return acc;
+      },
+      {},
+    );
+
+    if (inputTokens !== undefined) {
+      normalizedUsageDetails.input = Math.max(
+        inputTokens - (cacheReadTokens ?? 0) - (cacheCreationTokens ?? 0),
+        0,
+      );
+    }
+
+    if (outputTokens !== undefined) {
+      normalizedUsageDetails.output = outputTokens;
+    }
+
+    if (totalTokens !== undefined) {
+      normalizedUsageDetails.total = totalTokens;
+    }
+
+    if (cacheReadTokens !== undefined) {
+      normalizedUsageDetails.input_cached_tokens = cacheReadTokens;
+    }
+
+    if (cacheCreationTokens !== undefined) {
+      normalizedUsageDetails.input_cache_creation = cacheCreationTokens;
+    }
+
+    return normalizedUsageDetails;
   }
 
   private extractCostDetails(

packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts

@@ -181,6 +181,7 @@ const FIELD_SETS = {
     "userId",
     "sessionId",
     "traceName",
+    "tags",
     "toolDefinitions",
     "toolCalls",
     "toolCallNames",
@@ -217,6 +218,7 @@ const FIELD_SETS = {
     "userId",
     "sessionId",
     "traceName",
+    "tags",
   ],
   calculated: ["latency", "timeToFirstToken"],
   io: ["input", "output"],
@@ -239,6 +241,12 @@ const FIELD_SETS = {
     "level",
     "statusMessage",
     "version",
+    "userId",
+    "sessionId",
+    "traceName",
+    "tags",
+    "bookmarked",
+    "public",
     "toolDefinitions",
     "toolCalls",
     "toolCallNames",
@@ -1710,6 +1718,11 @@ const EXPERIMENTS_AGGREGATION_FIELDS = {
     "groupUniqArrayIf(tuple(e.prompt_name, e.prompt_version), e.prompt_name != '') AS prompts",
   experimentMetadata:
     "any(mapFromArrays(e.experiment_metadata_names, e.experiment_metadata_values)) AS experiment_metadata",
+
+  // Metrics fields
+  totalCost: "SUM(e.total_cost) AS total_cost",
+  latencyAvg:
+    "avgIf(date_diff('millisecond', e.start_time, e.end_time), e.span_id = e.experiment_item_root_span_id AND e.end_time IS NOT NULL) AS latency_avg",
 } as const;
 
 /**
@@ -1728,6 +1741,7 @@ const EXPERIMENTS_AGGREGATION_FIELD_SETS = {
     "prompts",
     "experimentMetadata",
   ] as const,
+  metrics: ["experimentId", "totalCost", "latencyAvg"] as const,
 } as const;
 
 export type ExperimentsAggregationFieldSetName =
@@ -1736,9 +1750,9 @@ export type ExperimentsAggregationFieldSetName =
 /**
  * ExperimentsAggregationQueryBuilder - Aggregates events by (experiment_id, project_id).
  *
- * For metrics requiring trace-level aggregation first (cost, latency), use CTEQueryBuilder
- * to wrap a trace CTE and re-aggregate at experiment level with selectRaw() + groupBy().
- * selectRaw() is intentionally used for explicit two-level aggregation semantics.
+ * Use the "metrics" field set for cost and latency aggregations:
+ * - Cost: SUM of all event costs for the experiment
+ * - Latency: AVG of root span duration (where span_id = experiment_item_root_span_id)
  */
 export class ExperimentsAggregationQueryBuilder extends BaseEventsQueryBuilder<
   typeof EXPERIMENTS_AGGREGATION_FIELDS

packages/shared/src/server/repositories/definitions.ts

@@ -115,6 +115,7 @@ export const eventsObservationRecordReadSchema =
     user_id: z.string().nullish(),
     session_id: z.string().nullish(),
     trace_name: z.string().nullish(),
+    tags: z.array(z.string()).optional(),
     bookmarked: z.boolean().optional(),
     public: z.boolean().optional(),
   });

packages/shared/src/server/repositories/events.ts

@@ -51,16 +51,16 @@ import {
   queryClickhouse,
   queryClickhouseStream,
 } from "./clickhouse";
-import { ObservationRecordReadType, TraceRecordReadType } from "./definitions";
+import {
+  EventsObservationRecordReadType,
+  TraceRecordReadType,
+} from "./definitions";
 import type { AnalyticsObservationEvent } from "../analytics-integrations/types";
 import {
   ObservationsTableQueryResult,
   ObservationTableQuery,
 } from "./observations";
-import {
-  convertEventsObservation,
-  convertObservation,
-} from "./observations_converters";
+import { convertEventsObservation } from "./observations_converters";
 import {
   EventsQueryBuilder,
   CTEQueryBuilder,
@@ -193,18 +193,22 @@ async function enrichObservationsWithModelData(
 async function enrichObservationsWithTraceFields(
   observationRecords: Array<EventsObservation & ObservationPriceFields>,
 ): Promise<FullEventsObservations> {
-  return observationRecords.map((o) => {
+  return observationRecords.map((observation) => {
+    // Remove raw tags field as this is re-mapped to traceTags
+    const { tags: _tags, ...observationWithoutRawTags } = observation;
     return {
-      ...o,
-      traceTags: [], // TODO pull from PG
+      ...observationWithoutRawTags,
+      traceTags: observation.tags ?? [],
       traceTimestamp: null,
-      toolDefinitions: o.toolDefinitions ?? null,
-      toolCalls: o.toolCalls ?? null,
+      toolDefinitions: observation.toolDefinitions ?? null,
+      toolCalls: observation.toolCalls ?? null,
       // Compute counts from actual data for events table
-      toolDefinitionsCount: o.toolDefinitions
-        ? Object.keys(o.toolDefinitions).length
+      toolDefinitionsCount: observation.toolDefinitions
+        ? Object.keys(observation.toolDefinitions).length
+        : null,
+      toolCallsCount: observation.toolCalls
+        ? observation.toolCalls.length
         : null,
-      toolCallsCount: o.toolCalls ? o.toolCalls.length : null,
     };
   });
 }
@@ -609,9 +613,19 @@ export const getObservationByIdFromEventsTable = async ({
     renderingProps,
     preferredClickhouseService: preferredClickhouseService ?? "EventsReadOnly",
   });
-  const mapped = records.map((record) =>
-    convertObservation(record, renderingProps),
-  );
+  const mapped = records.map((record) => {
+    // Remove raw tags field as this is re-mapped to traceTags
+    const { tags, ...converted } = convertEventsObservation(
+      record,
+      renderingProps,
+      true,
+    );
+
+    return {
+      ...converted,
+      traceTags: tags ?? [],
+    };
+  });
 
   mapped.forEach((observation) => {
     recordDistribution(
@@ -682,7 +696,7 @@ async function getObservationByIdFromEventsTableInternal({
 
   const { query, params } = queryBuilder.buildWithParams();
 
-  return await queryClickhouse<ObservationRecordReadType>({
+  return await queryClickhouse<EventsObservationRecordReadType>({
     query,
     params,
     tags: {
@@ -1072,7 +1086,7 @@ async function getObservationsCountFromEventsTableForPublicApiInternal(
  */
 export const getObservationsFromEventsTableForPublicApi = async (
   opts: Omit<PublicApiObservationsQuery, "fields">,
-): Promise<Array<Observation & ObservationPriceFields>> => {
+): Promise<Array<EventsObservation & ObservationPriceFields>> => {
   const { projectId } = opts;
 
   // Build query with filters and common CTEs
@@ -1090,12 +1104,15 @@ export const getObservationsFromEventsTableForPublicApi = async (
       projectId,
       queryBuilder,
     );
-  return await enrichObservationsWithModelData(
+
+  const observations = await enrichObservationsWithModelData(
     observationRecords,
     opts.projectId,
     opts.parseIoAsJson ?? true, // V1 API: default to parsing JSON (backwards compatibility)
     null, // V1 API: no field groups, return complete observations
   );
+
+  return observations;
 };
 
 /**

packages/shared/src/server/repositories/experiments.ts

@@ -19,7 +19,6 @@ import {
   eventsExperimentsAggregation,
   eventsScoresAggregation,
   eventsTracesScoresAggregation,
-  eventsTracesAggregation,
 } from "../queries/clickhouse-sql/query-fragments";
 import { extractTimeFilter, queryClickhouse } from "../repositories";
 import { parseClickhouseUTCDateTimeFormat } from "../repositories/clickhouse";
@@ -174,27 +173,17 @@ export const getExperimentMetricsFromEvents = async (props: {
     return [];
   }
 
-  const tracesBuilder = eventsTracesAggregation({
+  // Use eventsExperimentsAggregation with "metrics" field set for simplified aggregation
+  const queryBuilder = eventsExperimentsAggregation({
     projectId: props.projectId,
-  }).whereRaw("e.experiment_id IN ({experimentIds: Array(String)})", {
+    fieldSet: "metrics",
     experimentIds: props.experimentIds,
   });
 
-  // Build the final query
-  const queryBuilder = new CTEQueryBuilder()
-    .withCTEFromBuilder("traces_agg", tracesBuilder)
-    .from("traces_agg", "ta")
-    .select(
-      "ta.experiment_id AS experiment_id",
-      "SUM(ta.total_cost) AS total_cost",
-      "AVG(ta.latency_milliseconds) AS latency_avg",
-    )
-    .groupBy("ta.project_id", "ta.experiment_id");
-
   const { query, params } = queryBuilder.buildWithParams();
 
   const res = await measureAndReturn({
-    operationName: "getExperimentsFromEventsGeneric",
+    operationName: "getExperimentMetricsFromEvents",
     projectId: props.projectId,
     input: {
       params,
@@ -202,7 +191,7 @@ export const getExperimentMetricsFromEvents = async (props: {
         feature: "experiments",
         type: "experiments-table",
         projectId: props.projectId,
-        operation_name: `getExperimentMetricsFromEvents`,
+        operation_name: "getExperimentMetricsFromEvents",
       },
     },
     fn: async (input) => {

packages/shared/src/server/repositories/observations_converters.ts

@@ -411,6 +411,7 @@ export function convertEventsObservation(
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     traceName: record.trace_name ?? null,
+    tags: record.tags ?? [],
     bookmarked: record.bookmarked,
     public: record.public,
   };

packages/shared/src/server/services/StorageService.ts

@@ -22,6 +22,10 @@ import { backOff } from "exponential-backoff";
 import { ServiceUnavailableError } from "../../errors";
 import { BufferedStreamUploader } from "./BufferedStreamUploader";
 import { S3ChunkedUploadStrategy } from "./S3ChunkedUploadStrategy";
+import * as objectstorage from "oci-objectstorage";
+import * as common from "oci-common";
+import { UploadManager as OciUploadManager } from "oci-objectstorage";
+import { URL } from "node:url";
 
 export interface S3SseConfig {
   serverSideEncryption?: string;
@@ -127,6 +131,7 @@ export class StorageServiceFactory {
    * @param params.region - Region in which the bucket resides
    * @param params.forcePathStyle - Add bucket name into the path instead of the domain name. Mainly used for MinIO.
    * @param params.useAzureBlob - Use Azure Blob Storage instead of S3
+   * @param params.useOCIObjectStorage - Use OCI Object Storage instead of S3
    * @param params.useGoogleCloudStorage - Use Google Cloud Storage instead of S3
    * @param params.googleCloudCredentials - Google Cloud Storage credentials JSON string or path to credentials file
    * @param params.awsSse - Server-side encryption method (e.g., "aws:kms")
@@ -142,6 +147,7 @@ export class StorageServiceFactory {
     forcePathStyle: boolean;
     useAzureBlob?: boolean;
     useGoogleCloudStorage?: boolean;
+    useOCIObjectStorage?: boolean;
     googleCloudCredentials?: string;
     awsSse: string | undefined;
     awsSseKmsKeyId: string | undefined;
@@ -167,6 +173,13 @@ export class StorageServiceFactory {
       };
       return new GoogleCloudStorageService(googleParams);
     }
+    if (
+      params.useOCIObjectStorage !== undefined
+        ? params.useOCIObjectStorage
+        : env.LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE === "true"
+    ) {
+      return new OCIObjectStorageService(params);
+    }
     return new S3StorageService(params);
   }
 }
@@ -1009,3 +1022,492 @@ class GoogleCloudStorageService implements StorageService {
     }
   }
 }
+
+class OCIObjectStorageService implements StorageService {
+  private client?: objectstorage.ObjectStorageClient;
+  private clientInit: Promise<void>;
+  private bucketName: string;
+  private externalEndpoint?: string;
+  private namespaceName: string = "";
+
+  constructor(params: {
+    bucketName: string;
+    endpoint: string | undefined;
+    externalEndpoint?: string | undefined;
+    region: string | undefined;
+  }) {
+    this.bucketName = params.bucketName;
+    this.externalEndpoint = params.externalEndpoint;
+    this.clientInit = this.initClient(params);
+  }
+
+  private async initClient(params: { endpoint?: string; region?: string }) {
+    let provider: common.AuthenticationDetailsProvider;
+    switch (env.LANGFUSE_OCI_AUTH_TYPE) {
+      case "workload_identity": {
+        provider =
+          new common.OkeWorkloadIdentityAuthenticationDetailsProvider.OkeWorkloadIdentityAuthenticationDetailsProviderBuilder().build();
+
+        break;
+      }
+
+      case "instance_principal": {
+        provider =
+          await new common.InstancePrincipalsAuthenticationDetailsProviderBuilder().build();
+
+        break;
+      }
+
+      case "resource_principal": {
+        provider =
+          common.ResourcePrincipalAuthenticationDetailsProvider.builder();
+
+        break;
+      }
+
+      case "oci_profile": {
+        provider = new common.ConfigFileAuthenticationDetailsProvider(
+          env.LANGFUSE_OCI_CONFIG_FILE,
+          env.LANGFUSE_OCI_CONFIG_PROFILE,
+        );
+
+        break;
+      }
+
+      case "session_token": {
+        provider = new common.SessionAuthDetailProvider(
+          env.LANGFUSE_OCI_CONFIG_FILE,
+          env.LANGFUSE_OCI_CONFIG_PROFILE,
+        );
+
+        break;
+      }
+
+      default:
+        throw new Error(
+          "OCI auth not configured: set LANGFUSE_OCI_AUTH_TYPE to " +
+            "'workload_identity' | 'instance_principal' | 'resource_principal' | 'oci_profile' | 'session_token'",
+        );
+    }
+
+    this.client = new objectstorage.ObjectStorageClient({
+      authenticationDetailsProvider: provider,
+    });
+
+    const regionId = params.region?.trim();
+    if (regionId) this.client.region = common.Region.fromRegionId(regionId);
+    const endpoint = params.endpoint?.trim();
+    if (endpoint) this.client.endpoint = endpoint;
+  }
+
+  private async ensureClient() {
+    await this.clientInit;
+    if (!this.client)
+      throw new Error("OCI ObjectStorage client failed to initialize");
+    return this.client;
+  }
+
+  private async ensureNamespace(): Promise<string> {
+    if (this.namespaceName) return this.namespaceName;
+    const client = await this.ensureClient();
+    const nsResp = await client.getNamespace({});
+    this.namespaceName = nsResp.value ?? "";
+    return this.namespaceName;
+  }
+
+  private async getClientAndNamespace(): Promise<{
+    client: objectstorage.ObjectStorageClient;
+    namespaceName: string;
+  }> {
+    const client = await this.ensureClient();
+    const namespaceName = await this.ensureNamespace(); // uses the same client init + cached namespace
+    return { client, namespaceName };
+  }
+
+  private async streamToString(
+    readable: any, // could be many shapes, so use `any`
+  ): Promise<string> {
+    if (!readable) return "";
+
+    // Helper: convert many chunk shapes to Buffer
+    const toBuffer = (chunk: any): Buffer => {
+      if (Buffer.isBuffer(chunk)) return chunk;
+      if (typeof chunk === "string") return Buffer.from(chunk, "utf8");
+      if (chunk instanceof ArrayBuffer) return Buffer.from(chunk);
+      // TypedArray / DataView
+      if (ArrayBuffer.isView(chunk)) {
+        return Buffer.from(
+          (chunk as Uint8Array).buffer,
+          (chunk as any).byteOffset ?? 0,
+          (chunk as any).byteLength ?? undefined,
+        );
+      }
+      // Fallback: try Buffer.from (may throw)
+      return Buffer.from(chunk);
+    };
+
+    // 1) Node.js Readable (EventEmitter style)
+    if (
+      typeof readable.on === "function" &&
+      typeof readable.read !== "undefined"
+    ) {
+      return await new Promise<string>((resolve, reject) => {
+        const chunks: Buffer[] = [];
+        readable.on("data", (chunk: any) => {
+          try {
+            chunks.push(toBuffer(chunk));
+          } catch (_err) {
+            // if conversion fails, push as Buffer of stringified chunk
+            chunks.push(Buffer.from(String(chunk)));
+          }
+        });
+        readable.on("error", (err: any) => reject(err));
+        readable.on("end", () => {
+          resolve(Buffer.concat(chunks).toString("utf8"));
+        });
+      });
+    }
+
+    // 2) WHATWG ReadableStream (browser / some fetch-like APIs)
+    if (typeof readable.getReader === "function") {
+      const reader = readable.getReader();
+      const chunks: Buffer[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(toBuffer(value));
+        }
+        return Buffer.concat(chunks).toString("utf8");
+      } finally {
+        // safe to close reader if available
+        try {
+          if (reader.releaseLock) reader.releaseLock();
+        } catch (_err) {
+          // intentionally ignore releaseLock errors
+        }
+      }
+    }
+
+    // 3) Buffer / Uint8Array / ArrayBuffer direct
+    if (Buffer.isBuffer(readable)) return readable.toString("utf8");
+    if (readable instanceof Uint8Array)
+      return Buffer.from(readable).toString("utf8");
+    if (readable instanceof ArrayBuffer)
+      return Buffer.from(readable).toString("utf8");
+
+    // 4) Blob (browser)
+    if (typeof Blob !== "undefined" && readable instanceof Blob) {
+      const ab = await readable.arrayBuffer();
+      return Buffer.from(ab).toString("utf8");
+    }
+
+    // 5) Async iterable (some stream implementations)
+    if (typeof readable[Symbol.asyncIterator] === "function") {
+      const chunks: Buffer[] = [];
+      for await (const chunk of readable) {
+        chunks.push(toBuffer(chunk));
+      }
+      return Buffer.concat(chunks).toString("utf8");
+    }
+
+    // 6) Synchronous iterable
+    if (typeof readable[Symbol.iterator] === "function") {
+      const chunks: Buffer[] = [];
+      for (const chunk of readable) {
+        chunks.push(toBuffer(chunk));
+      }
+      return Buffer.concat(chunks).toString("utf8");
+    }
+
+    // 7) Fallback: try string conversion
+    try {
+      return String(readable);
+    } catch (_err) {
+      // If all else fails, throw a helpful error
+      throw new TypeError("Unsupported body type passed to streamToString");
+    }
+  }
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    partSize,
+    queueSize,
+  }: UploadFile): Promise<void> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const uploadManager = new OciUploadManager(client, {
+        partSize: partSize ?? 20 * 1024 * 1024,
+        maxConcurrentUploads: queueSize ?? 5,
+      });
+
+      // UploadManager in the OCI SDK expects content shaped as one of:
+      // { blob }, { filePath }, or { stream }.
+      // To work reliably in Node, always provide { stream }.
+      const stream =
+        typeof data === "string"
+          ? Readable.from([data])
+          : data instanceof Readable
+            ? data
+            : Buffer.isBuffer(data as any)
+              ? Readable.from([data as any])
+              : Readable.from([String(data)]);
+
+      const contentLength =
+        typeof data === "string"
+          ? Buffer.byteLength(data)
+          : Buffer.isBuffer(data as any)
+            ? (data as any).byteLength
+            : undefined;
+
+      await uploadManager.upload({
+        requestDetails: {
+          namespaceName,
+          bucketName: this.bucketName,
+          objectName: fileName,
+          contentType: fileType,
+          ...(contentLength ? { contentLength } : {}),
+        },
+        content: { stream },
+      });
+
+      return;
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(err, "upload file to OCI Object Storage ");
+    }
+  }
+
+  public async uploadFileBuffered({
+    fileName,
+    fileType,
+    data,
+    partSizeBytes,
+  }: UploadFileBuffered): Promise<void> {
+    await this.uploadFile({
+      fileName,
+      fileType,
+      data,
+      partSize: partSizeBytes,
+    });
+  }
+
+  public async uploadWithSignedUrl({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+    partSize,
+    queueSize,
+  }: UploadWithSignedUrl): Promise<{ signedUrl: string }> {
+    try {
+      await this.uploadFile({ fileName, data, fileType, partSize, queueSize });
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(
+        err,
+        "upload file to OCI Object Storage  or generate signed URL",
+      );
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const jsonString = JSON.stringify(body);
+      const req: objectstorage.requests.PutObjectRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        objectName: path,
+        contentLength: Buffer.byteLength(jsonString),
+        putObjectBody: Readable.from([jsonString]),
+        contentType: "application/json",
+      };
+      await client.putObject(req);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to OCI Object Storage ${path}`, err);
+      handleStorageError(err, "upload JSON to OCI Object Storage ");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const req: objectstorage.requests.GetObjectRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        objectName: path,
+      };
+      const response = await client.getObject(req);
+      const bodyStream = (response as any).value as
+        | NodeJS.ReadableStream
+        | undefined;
+      return await this.streamToString(bodyStream);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from OCI Object Storage  ${path}`,
+        err,
+      );
+      handleStorageError(err, "download file from OCI Object Storage ");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const req: objectstorage.requests.ListObjectsRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        prefix,
+      };
+      const resp = await client.listObjects(req);
+      const objects = ((resp as any).listObjects?.objects ?? []) as Array<{
+        name?: string;
+        timeCreated?: Date | string;
+      }>;
+      return (
+        objects.flatMap((obj) =>
+          obj.name
+            ? [
+                {
+                  file: obj.name,
+                  createdAt: obj.timeCreated
+                    ? new Date(obj.timeCreated as any)
+                    : new Date(),
+                },
+              ]
+            : [],
+        ) ?? []
+      );
+    } catch (err) {
+      logger.error(
+        `Failed to list files from OCI Object Storage  ${prefix}`,
+        err,
+      );
+      handleStorageError(err, "list files from OCI Object Storage ");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const expiresOn = new Date(Date.now() + ttlSeconds * 1000);
+      const req: objectstorage.requests.CreatePreauthenticatedRequestRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        createPreauthenticatedRequestDetails: {
+          name: `read-${fileName}-${Date.now()}`,
+          accessType: "ObjectRead" as any,
+          objectName: fileName,
+          timeExpires: expiresOn as any,
+        } as any,
+      };
+      const resp = await client.createPreauthenticatedRequest(req);
+      const accessUri = (resp.preauthenticatedRequest as any)
+        .accessUri as string;
+      const base = this.externalEndpoint ?? client.endpoint;
+
+      if (!base) {
+        throw new Error(
+          "Cannot build PAR URL: no externalEndpoint configured and client.endpoint is empty",
+        );
+      }
+      const baseUrl = new URL(base);
+      const parUrl = new URL(accessUri, baseUrl);
+      if (asAttachment) {
+        parUrl.searchParams.set("download", "1");
+      }
+      const url = parUrl.toString();
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL (PAR) for OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(err, "generate signed URL for OCI Object Storage ");
+    }
+  }
+
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      for (const p of paths) {
+        const req: objectstorage.requests.DeleteObjectRequest = {
+          namespaceName,
+          bucketName: this.bucketName,
+          objectName: p,
+        } as any;
+        await client.deleteObject(req as any);
+      }
+    } catch (err) {
+      logger.error(`Failed to delete files from OCI Object Storage `, {
+        error: err,
+        files: paths,
+      });
+      handleStorageError(err, "delete files from OCI Object Storage ");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds } = params;
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const expiresOn = new Date(Date.now() + ttlSeconds * 1000);
+      const req: objectstorage.requests.CreatePreauthenticatedRequestRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        createPreauthenticatedRequestDetails: {
+          name: `write-${path}-${Date.now()}`,
+          accessType: "ObjectWrite" as any,
+          objectName: path,
+          timeExpires: expiresOn as any,
+        } as any,
+      };
+      const resp = await client.createPreauthenticatedRequest(req);
+      const accessUri = (resp.preauthenticatedRequest as any)
+        .accessUri as string;
+      const base = this.externalEndpoint ?? client.endpoint;
+
+      if (!base) {
+        throw new Error(
+          "Cannot build PAR URL: no externalEndpoint configured and client.endpoint is empty",
+        );
+      }
+      const baseUrl = new URL(base);
+      let url = new URL(accessUri, baseUrl).toString();
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL (PAR) for OCI Object Storage  ${path}`,
+        err,
+      );
+      handleStorageError(
+        err,
+        "generate presigned upload URL for OCI Object Storage ",
+      );
+    }
+  }
+}

packages/shared/src/server/services/blockEvaluatorConfigs.ts

@@ -227,6 +227,22 @@ export async function notifyBlockedEvaluatorConfigs({
     return;
   }
 
+  const project = await prisma.project.findUnique({
+    where: {
+      id: projectId,
+    },
+    select: {
+      name: true,
+    },
+  });
+
+  if (!project) {
+    logger.warn(
+      `[EVALUATOR BLOCK] Project ${projectId} not found. Skipping notifications.`,
+    );
+    return;
+  }
+
   const blockedConfigs = await prisma.jobConfiguration.findMany({
     where: {
       projectId,
@@ -254,6 +270,7 @@ export async function notifyBlockedEvaluatorConfigs({
     adminEmails.map((receiverEmail) =>
       sendEvaluatorBlockedEmail({
         env: emailEnv,
+        projectName: project.name,
         evaluatorName: config.evalTemplate?.name ?? config.scoreName,
         blockReason,
         blockMessage,

packages/shared/src/server/services/email/evaluatorBlocked/EvaluatorBlockedEmailTemplate.tsx

@@ -16,6 +16,7 @@ import {
 import { EvaluatorBlockReason } from "@prisma/client";
 
 type EvaluatorBlockedEmailTemplateProps = {
+  projectName: string;
   evaluatorName: string;
   blockReason: EvaluatorBlockReason;
   blockMessage: string;
@@ -104,6 +105,7 @@ const getResolutionSteps = (blockReason: EvaluatorBlockReason) => {
 };
 
 export const EvaluatorBlockedEmailTemplate = ({
+  projectName,
   evaluatorName,
   blockReason,
   blockMessage,
@@ -114,8 +116,8 @@ export const EvaluatorBlockedEmailTemplate = ({
     <Html>
       <Head />
       <Preview>
-        LLM evaluator &quot;{evaluatorName}&quot; paused:{" "}
-        {getReasonSummary(blockReason)}
+        LLM evaluator &quot;{evaluatorName}&quot; in project &quot;
+        {projectName}&quot; paused: {getReasonSummary(blockReason)}
       </Preview>
       <Tailwind>
         <Body className="bg-background my-auto mx-auto font-sans">
@@ -135,8 +137,9 @@ export const EvaluatorBlockedEmailTemplate = ({
                 ⚠️ Evaluator Paused
               </Heading>
               <Text className="text-gray-700 text-sm leading-6">
-                The LLM evaluator &quot;{evaluatorName}&quot; was automatically
-                paused because {getReasonSummary(blockReason).toLowerCase()}.
+                The LLM evaluator &quot;{evaluatorName}&quot; in project &quot;
+                {projectName}&quot; was automatically paused because{" "}
+                {getReasonSummary(blockReason).toLowerCase()}.
               </Text>
             </Section>
 
@@ -174,7 +177,8 @@ export const EvaluatorBlockedEmailTemplate = ({
             <Section>
               <Text className="text-[#666666] text-[12px] leading-[24px]">
                 This notification was sent to {receiverEmail} regarding the
-                paused evaluator &quot;{evaluatorName}&quot;.
+                paused evaluator &quot;{evaluatorName}&quot; in project &quot;
+                {projectName}&quot;.
               </Text>
             </Section>
           </Container>

packages/shared/src/server/services/email/evaluatorBlocked/sendEvaluatorBlockedEmail.ts

@@ -17,6 +17,7 @@ export type SendEvaluatorBlockedEmailParams = {
       string | undefined
     >
   >;
+  projectName: string;
   evaluatorName: string;
   blockReason: EvaluatorBlockReason;
   blockMessage: string;
@@ -26,6 +27,7 @@ export type SendEvaluatorBlockedEmailParams = {
 
 export const sendEvaluatorBlockedEmail = async ({
   env,
+  projectName,
   evaluatorName,
   blockReason,
   blockMessage,
@@ -42,9 +44,11 @@ export const sendEvaluatorBlockedEmail = async ({
   try {
     const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
     const safeEvaluatorName = sanitizeEmailSubject(evaluatorName);
+    const safeProjectName = sanitizeEmailSubject(projectName);
     const subject = `⚠️ LLM evaluator "${safeEvaluatorName}" paused - action required`;
     const html = await render(
       EvaluatorBlockedEmailTemplate({
+        projectName: safeProjectName,
         evaluatorName: safeEvaluatorName,
         blockReason,
         blockMessage,

packages/shared/src/utils/objects.ts

@@ -3,7 +3,6 @@ type OmitKeys<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
 /**
  * Removes specified keys from an object and returns a new object without those keys.
  */
-
 export function removeObjectKeys<T, K extends keyof T>(
   obj: T,
   keys: K[],
@@ -14,3 +13,62 @@ export function removeObjectKeys<T, K extends keyof T>(
   }
   return result;
 }
+
+/**
+ * Safely coerces a value to a Record if it's a plain object.
+ * Returns undefined for null, undefined, arrays, and non-objects.
+ */
+export function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+/**
+ * Safely coerces a value to a string if it's a non-empty string.
+ * Returns undefined for empty strings and non-strings.
+ */
+export function asString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+/**
+ * Safely coerces a value to a boolean if it's a boolean.
+ * Returns undefined for non-booleans.
+ */
+export function asBoolean(value: unknown): boolean | undefined {
+  return typeof value === "boolean" ? value : undefined;
+}
+
+/**
+ * Safely coerces a value to a string array if it's an array of strings.
+ * Returns undefined for non-arrays or arrays with non-string elements.
+ */
+export function asStringArray(value: unknown): string[] | undefined {
+  return Array.isArray(value) &&
+    value.every((entry) => typeof entry === "string")
+    ? value
+    : undefined;
+}
+
+/**
+ * Safely coerces a value to a Record<string, number> if it's an object with numeric values.
+ * Filters out non-finite numbers. Returns undefined if result is empty or input is not an object.
+ */
+export function asNumberRecord(
+  value: unknown,
+): Record<string, number> | undefined {
+  const record = asRecord(value);
+
+  if (!record) {
+    return undefined;
+  }
+
+  const normalized = Object.fromEntries(
+    Object.entries(record).filter(
+      ([, entry]) => typeof entry === "number" && Number.isFinite(entry),
+    ),
+  ) as Record<string, number>;
+
+  return Object.keys(normalized).length > 0 ? normalized : undefined;
+}

scripts/codex/cloud_services.sh

@@ -29,6 +29,9 @@ MINIO_SHA256_AMD64="${MINIO_SHA256_AMD64:-7c5bd8512c6e966455b1d198209358b2d191c7
 MINIO_SHA256_ARM64="${MINIO_SHA256_ARM64:-5c83cd2cf151717ba0243f73e1c7802ff36e272b67144bdd7f1f7d684fd6f03d}"
 MC_SHA256_AMD64="${MC_SHA256_AMD64:-01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891}"
 MC_SHA256_ARM64="${MC_SHA256_ARM64:-14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c}"
+MIGRATE_RELEASE_TAG="${MIGRATE_RELEASE_TAG:-v4.19.1}"
+MIGRATE_SHA256_AMD64="${MIGRATE_SHA256_AMD64:-2ac648fbd1b127b69ab5a7b33cf96212178f71e22379fc50573630c6f4c7ce18}"
+MIGRATE_SHA256_ARM64="${MIGRATE_SHA256_ARM64:-2fea2455c0f3f07cc3f4b98471c951ad1a716059574b20b6416bd1e9058751c5}"
 
 export DEBIAN_FRONTEND=noninteractive
 
@@ -114,6 +117,55 @@ ensure_clickhouse_binaries() {
   stop_service_if_running clickhouse-server
 }
 
+detect_migrate_arch() {
+  local machine_arch
+  machine_arch="$(uname -m)"
+
+  case "$machine_arch" in
+    x86_64|amd64)
+      echo "amd64"
+      ;;
+    aarch64|arm64)
+      echo "arm64"
+      ;;
+    *)
+      echo "Unsupported architecture for golang-migrate binary: $machine_arch" >&2
+      exit 1
+      ;;
+  esac
+}
+
+ensure_migrate_binary() {
+  if command -v migrate >/dev/null 2>&1; then
+    return 0
+  fi
+
+  ensure_apt_package ca-certificates
+  ensure_apt_package curl
+
+  local migrate_arch
+  local migrate_sha256
+  local tmp_dir
+  migrate_arch="$(detect_migrate_arch)"
+  case "$migrate_arch" in
+    amd64)
+      migrate_sha256="$MIGRATE_SHA256_AMD64"
+      ;;
+    arm64)
+      migrate_sha256="$MIGRATE_SHA256_ARM64"
+      ;;
+  esac
+  tmp_dir="$(mktemp -d)"
+  trap 'rm -rf "$tmp_dir"' RETURN
+
+  download_and_verify_sha256 \
+    "https://github.com/golang-migrate/migrate/releases/download/${MIGRATE_RELEASE_TAG}/migrate.linux-${migrate_arch}.tar.gz" \
+    "$tmp_dir/migrate.tar.gz" \
+    "$migrate_sha256"
+  tar -xzf "$tmp_dir/migrate.tar.gz" -C "$tmp_dir" migrate
+  install -m 0755 "$tmp_dir/migrate" /usr/local/bin/migrate
+}
+
 detect_minio_arch() {
   local machine_arch
   machine_arch="$(uname -m)"
@@ -454,6 +506,7 @@ ensure_minio_running() {
 ensure_cloud_dependencies() {
   mkdir -p "$CODEX_SERVICES_ROOT"
 
+  ensure_migrate_binary
   ensure_postgres_running
   ensure_redis_running
   ensure_clickhouse_running

scripts/codex/maintenance_cloud.sh

@@ -18,3 +18,9 @@ pnpm install --frozen-lockfile
 
 # Keep generated Prisma artifacts aligned after dependency or schema updates.
 pnpm run db:generate
+
+# Keep local databases initialized for worker/web tests during maintenance runs.
+pnpm --filter=shared run db:reset:test
+pnpm --filter=shared run db:reset -f
+SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset
+pnpm --filter=shared run db:seed:examples

scripts/codex/setup_cloud.sh

@@ -40,3 +40,10 @@ pnpm --filter=shared run db:generate
 
 # Prisma client generation is needed for typecheck/build tasks in Codex.
 pnpm run db:generate
+
+# Initialize local databases so worker/web tests can run immediately after
+# bootstrap without "table does not exist" failures.
+pnpm --filter=shared run db:reset:test
+pnpm --filter=shared run db:reset -f
+SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset
+pnpm --filter=shared run db:seed:examples

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.167.4",
+  "version": "3.168.0",
   "private": true,
   "license": "MIT",
   "engines": {

web/src/__tests__/experiments-access.clienttest.ts

@@ -1,42 +1,37 @@
 import { getExperimentsAccess } from "@/src/features/experiments/utils/experimentsAccess";
 
 describe("getExperimentsAccess", () => {
-  it("returns enabled only when cloud, v4 beta, and admin/flag gate all pass", () => {
-    const enabledViaAdmin = getExperimentsAccess({
-      isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: true,
-      isFeatureEnabledOnUser: false,
-    });
-
-    const enabledViaFlag = getExperimentsAccess({
-      isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: false,
-      isFeatureEnabledOnUser: true,
-    });
-
-    expect(enabledViaAdmin.isEnabled).toBe(true);
-    expect(enabledViaFlag.isEnabled).toBe(true);
-  });
-
-  it("returns disabled when v4 beta is off even for eligible users", () => {
+  it("returns enabled when cloud and v4 beta are both enabled", () => {
     const access = getExperimentsAccess({
       isLangfuseCloud: true,
-      isV4BetaEnabled: false,
-      isAdmin: true,
-      isFeatureEnabledOnUser: true,
+      isV4BetaEnabled: true,
+    });
+
+    expect(access.isEnabled).toBe(true);
+  });
+
+  it("returns disabled when not on cloud", () => {
+    const access = getExperimentsAccess({
+      isLangfuseCloud: false,
+      isV4BetaEnabled: true,
     });
 
     expect(access.isEnabled).toBe(false);
   });
 
-  it("returns disabled when user is neither admin nor flagged", () => {
+  it("returns disabled when v4 beta is off", () => {
     const access = getExperimentsAccess({
       isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: false,
-      isFeatureEnabledOnUser: false,
+      isV4BetaEnabled: false,
+    });
+
+    expect(access.isEnabled).toBe(false);
+  });
+
+  it("returns disabled when both cloud and v4 beta are off", () => {
+    const access = getExperimentsAccess({
+      isLangfuseCloud: false,
+      isV4BetaEnabled: false,
     });
 
     expect(access.isEnabled).toBe(false);

web/src/__tests__/server/api/otel/otelMapping.servertest.ts

@@ -1885,6 +1885,106 @@ describe("OTel Resource Span Mapping", () => {
       expect(metadataAttributes).not.toHaveProperty("pydantic_ai.all_messages");
     });
 
+    it("should normalize current Pydantic AI cache usage fields into Langfuse usage details", async () => {
+      const traceId = "abcdef1234567890abcdef1234567891";
+
+      const pydanticAiRootSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "telemetry.sdk.language",
+              value: { stringValue: "python" },
+            },
+            {
+              key: "telemetry.sdk.name",
+              value: { stringValue: "opentelemetry" },
+            },
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "pydantic-ai",
+              version: "1.66.0",
+              attributes: [],
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex"),
+                spanId: Buffer.from("80854cd6bd218bf6", "hex"),
+                name: "pydantic-cache-test",
+                kind: 1,
+                startTimeUnixNano: {
+                  low: 1000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                endTimeUnixNano: {
+                  low: 2000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                attributes: [
+                  {
+                    key: "gen_ai.usage.input_tokens",
+                    value: { intValue: { low: 120, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_tokens",
+                    value: { intValue: { low: 40, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_read.input_tokens",
+                    value: { intValue: { low: 30, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_creation.input_tokens",
+                    value: { intValue: { low: 10, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.input_audio_tokens",
+                    value: { intValue: { low: 5, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.cache_audio_read_tokens",
+                    value: { intValue: { low: 2, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.output_audio_tokens",
+                    value: { intValue: { low: 7, high: 0, unsigned: false } },
+                  },
+                ],
+                events: [],
+                status: { code: 1 },
+              },
+            ],
+          },
+        ],
+      };
+
+      const events = await convertOtelSpanToIngestionEvent(
+        pydanticAiRootSpan,
+        new Set(),
+      );
+
+      const observationEvent = events.find((e) => e.type === "span-create");
+
+      expect(observationEvent).toBeDefined();
+      expect(observationEvent?.body.usageDetails.input).toBe(80);
+      expect(observationEvent?.body.usageDetails.output).toBe(40);
+      expect(observationEvent?.body.usageDetails.input_cached_tokens).toBe(30);
+      expect(observationEvent?.body.usageDetails.input_cache_creation).toBe(10);
+      expect(observationEvent?.body.usageDetails.input_audio_tokens).toBe(5);
+      expect(observationEvent?.body.usageDetails.cache_audio_read_tokens).toBe(
+        2,
+      );
+      expect(observationEvent?.body.usageDetails.output_audio_tokens).toBe(7);
+    });
+
     it("should prepend gen_ai.system_instructions to pydantic_ai.all_messages input when system message is absent", async () => {
       const traceId = "9d7aa9a729def1eadc0b2063ca4ebeb4";
 
@@ -6202,6 +6302,92 @@ describe("OTel Resource Span Mapping", () => {
     });
   });
 
+  describe("GenAI usage normalization", () => {
+    it("should normalize official gen_ai cache usage into Langfuse canonical keys", async () => {
+      const traceId = "abcdef1234567890abcdef1234567892";
+
+      const genAiSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "gen_ai",
+              version: "1.0.0",
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex"),
+                spanId: Buffer.from("1234567890abcde1", "hex"),
+                name: "normalized-genai-usage",
+                kind: 1,
+                startTimeUnixNano: {
+                  low: 1000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                endTimeUnixNano: {
+                  low: 2000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                attributes: [
+                  {
+                    key: "gen_ai.usage.input_tokens",
+                    value: { intValue: { low: 100, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_tokens",
+                    value: { intValue: { low: 40, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.total_tokens",
+                    value: { intValue: { low: 140, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_read.input_tokens",
+                    value: { intValue: { low: 20, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_creation.input_tokens",
+                    value: { intValue: { low: 10, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_audio_tokens",
+                    value: { intValue: { low: 5, high: 0, unsigned: false } },
+                  },
+                ],
+                status: {},
+              },
+            ],
+          },
+        ],
+      };
+
+      const events = await convertOtelSpanToIngestionEvent(
+        genAiSpan,
+        new Set(),
+      );
+      const observationEvent = events.find(
+        (e) => e.type === "generation-create" || e.type === "span-create",
+      );
+
+      expect(observationEvent).toBeDefined();
+      expect(observationEvent?.body.usageDetails.input).toBe(70);
+      expect(observationEvent?.body.usageDetails.output).toBe(40);
+      expect(observationEvent?.body.usageDetails.total).toBe(140);
+      expect(observationEvent?.body.usageDetails.input_cached_tokens).toBe(20);
+      expect(observationEvent?.body.usageDetails.input_cache_creation).toBe(10);
+      expect(observationEvent?.body.usageDetails.output_audio_tokens).toBe(5);
+    });
+  });
+
   describe("Vercel AI SDK Usage details", () => {
     it("should extract usage details from both provider metadata and 'ai.usage'", async () => {
       const traceId = "abcdef1234567890abcdef1234567890";
@@ -7279,4 +7465,73 @@ describe("OTel Resource Span Mapping", () => {
       );
     });
   });
+
+  describe("Prototype pollution protection", () => {
+    const publicKey = "pk-lf-1234567890";
+
+    it("should not pollute Object.prototype via __proto__ in gen_ai.prompt attributes", async () => {
+      const traceId = "abcdef1234567890abcdef1234567890";
+      const spanId = "abcdef1234567890";
+
+      const resourceSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "langfuse-sdk",
+              version: "2.0.0",
+              attributes: [
+                {
+                  key: "public_key",
+                  value: { stringValue: publicKey },
+                },
+              ],
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex").toJSON(),
+                spanId: Buffer.from(spanId, "hex").toJSON(),
+                name: "pollution-test",
+                kind: 1,
+                startTimeUnixNano: { low: 1000000000, high: 0, unsigned: true },
+                endTimeUnixNano: { low: 2000000000, high: 0, unsigned: true },
+                attributes: [
+                  {
+                    key: "gen_ai.prompt.role",
+                    value: { stringValue: "user" },
+                  },
+                  {
+                    key: "gen_ai.prompt.content",
+                    value: { stringValue: "hello" },
+                  },
+                  {
+                    key: "gen_ai.prompt.__proto__.POLLUTED",
+                    value: { stringValue: "SUCCESS" },
+                  },
+                ],
+                status: {},
+              },
+            ],
+          },
+        ],
+      };
+
+      await convertOtelSpanToIngestionEvent(
+        resourceSpan,
+        new Set([traceId]),
+        publicKey,
+      );
+
+      // Verify Object.prototype was NOT polluted
+      expect(({} as any).POLLUTED).toBeUndefined();
+      expect(Object.prototype.hasOwnProperty("POLLUTED" as any)).toBe(false);
+    });
+  });
 });

web/src/__tests__/server/llm-api-key.servertest.ts

@@ -9,11 +9,13 @@ jest.mock("@langfuse/shared/src/server", () => {
 });
 
 import type { Session } from "next-auth";
-import { LLMAdapter } from "@langfuse/shared";
+import { BEDROCK_USE_DEFAULT_CREDENTIALS, LLMAdapter } from "@langfuse/shared";
+import { env } from "@/src/env.mjs";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
 import { decrypt, encrypt } from "@langfuse/shared/encryption";
+import { AuthMethod } from "@/src/features/llm-api-key/types";
 import {
   createOrgProjectAndApiKey,
   fetchLLMCompletion,
@@ -140,6 +142,49 @@ describe("llmApiKey.all RPC", () => {
     expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
   });
 
+  it("should create a Bedrock llm api key with a Bedrock API key", async () => {
+    const secret = "bedrock-api-key-1234";
+
+    await caller.llmApiKey.create({
+      projectId,
+      secretKey: JSON.stringify({ apiKey: secret }),
+      provider: "bedrock",
+      adapter: LLMAdapter.Bedrock,
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const llmApiKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: {
+        projectId,
+        provider: "bedrock",
+      },
+    });
+
+    expect(decrypt(llmApiKey.secretKey)).toBe(
+      JSON.stringify({
+        apiKey: secret,
+      }),
+    );
+    expect(llmApiKey.displaySecretKey).toBe("...1234");
+    expect(llmApiKey.config).toEqual({ region: "us-east-1" });
+  });
+
+  it("should reject creating a Bedrock key with invalid secret key JSON", async () => {
+    await expect(
+      caller.llmApiKey.create({
+        projectId,
+        secretKey: JSON.stringify({ unknownField: "value" }),
+        provider: "bedrock",
+        adapter: LLMAdapter.Bedrock,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      }),
+    ).rejects.toThrow("Invalid Bedrock credentials");
+  });
+
   it("should block creating an llm api key with a localhost base URL", async () => {
     await expect(
       caller.llmApiKey.create({
@@ -199,6 +244,87 @@ describe("llmApiKey.all RPC", () => {
     expect(secretKey).toBeUndefined();
   });
 
+  it("should derive the Bedrock auth method in llmApiKey.all without returning secrets", async () => {
+    await prisma.llmApiKeys.createMany({
+      data: [
+        {
+          projectId,
+          provider: "bedrock-access",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(
+            JSON.stringify({
+              accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+              secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+            }),
+          ),
+          displaySecretKey: "...MPLE",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "bedrock-api",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(
+            JSON.stringify({
+              apiKey: "bedrock-api-key-1234",
+            }),
+          ),
+          displaySecretKey: "...1234",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "bedrock-default",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
+          displaySecretKey: "Default AWS credentials",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "openai",
+          adapter: LLMAdapter.OpenAI,
+          secretKey: encrypt("sk-test"),
+          displaySecretKey: "...test",
+          customModels: [],
+          withDefaultModels: true,
+          extraHeaderKeys: [],
+        },
+      ],
+    });
+
+    const { data: llmApiKeys } = await caller.llmApiKey.all({
+      projectId,
+    });
+
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-access")?.authMethod,
+    ).toBe(AuthMethod.AccessKeys);
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-api")?.authMethod,
+    ).toBe(AuthMethod.ApiKey);
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-default")?.authMethod,
+    ).toBe(AuthMethod.DefaultCredentials);
+    expect(
+      llmApiKeys.find((key) => key.provider === "openai")?.authMethod,
+    ).toBeUndefined();
+    expect(
+      llmApiKeys.every(
+        (key) => key.secretKey === undefined && key.extraHeaders === undefined,
+      ),
+    ).toBe(true);
+  });
+
   it("should require llmApiKeys:create access for testing a new llm api key", async () => {
     const memberCaller = createCallerForProjectRole("MEMBER");
 
@@ -448,6 +574,287 @@ describe("llmApiKey.all RPC", () => {
     expect(updatedKeys[0].withDefaultModels).toBe(newWithDefaultModels);
   });
 
+  it("should update a Bedrock Access key auth to a Bedrock API key", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: {
+        projectId,
+        provider,
+      },
+    });
+
+    await caller.llmApiKey.update({
+      id: existingKey.id,
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-5678" }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "eu-west-1" },
+    });
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(
+      JSON.stringify({
+        apiKey: "bedrock-api-key-5678",
+      }),
+    );
+    expect(updatedKey.displaySecretKey).toBe("...5678");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
+  it("should update a Bedrock API key auth to Access keys", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    await caller.llmApiKey.update({
+      id: existingKey.id,
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "eu-west-1" },
+    });
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(
+      JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+    );
+    expect(updatedKey.displaySecretKey).toBe("...EKEY");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
+  it("should update a Bedrock DefaultCredentials key to explicit Access keys", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    try {
+      // Simulate self-hosted to allow default credentials
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.create({
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      });
+
+      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+        where: { projectId, provider },
+      });
+
+      expect(decrypt(existingKey.secretKey)).toBe(
+        BEDROCK_USE_DEFAULT_CREDENTIALS,
+      );
+      expect(existingKey.displaySecretKey).toBe("Default AWS credentials");
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: JSON.stringify({
+          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        }),
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+
+      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+        where: { id: existingKey.id },
+      });
+
+      expect(decrypt(updatedKey.secretKey)).toBe(
+        JSON.stringify({
+          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        }),
+      );
+      expect(updatedKey.displaySecretKey).toBe("...EKEY");
+      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+  });
+
+  it("should update a Bedrock DefaultCredentials key to a Bedrock API key", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    try {
+      // Simulate self-hosted to allow default credentials
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.create({
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      });
+
+      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+        where: { projectId, provider },
+      });
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+
+      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+        where: { id: existingKey.id },
+      });
+
+      expect(decrypt(updatedKey.secretKey)).toBe(
+        JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
+      );
+      expect(updatedKey.displaySecretKey).toBe("...9999");
+      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+  });
+
+  it("should reject updating a Bedrock key back to DefaultCredentials on cloud", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    await expect(
+      caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      }),
+    ).rejects.toThrow(
+      "Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
+    );
+  });
+
+  it("should update a Bedrock Access key auth back to DefaultCredentials (self-hosted)", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    try {
+      // Simulate self-hosted deployment where default credentials are allowed
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(BEDROCK_USE_DEFAULT_CREDENTIALS);
+    expect(updatedKey.displaySecretKey).toBe("Default AWS credentials");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
   it("should update only the secret key", async () => {
     const secret = "test-secret";
     const provider = "openai";

web/src/__tests__/server/llm-connections-api.servertest.ts

@@ -577,6 +577,26 @@ describe("/api/public/llm-connections API Endpoints", () => {
       expect(bedrockResponse.body.adapter).toBe(LLMAdapter.Bedrock);
       expect(bedrockResponse.body.config).toEqual({ region: "us-east-1" });
 
+      const bedrockApiKeyResponse = await makeZodVerifiedAPICall(
+        PutLlmConnectionV1Response,
+        "PUT",
+        "/api/public/llm-connections",
+        {
+          provider: generateUniqueProvider("test-bedrock-api-key"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
+          config: { region: "us-east-1" },
+        },
+        auth,
+        201,
+      );
+      expect(bedrockApiKeyResponse.status).toBe(201);
+      expect(bedrockApiKeyResponse.body.adapter).toBe(LLMAdapter.Bedrock);
+      expect(bedrockApiKeyResponse.body.displaySecretKey).toBe("...1234");
+      expect(bedrockApiKeyResponse.body.config).toEqual({
+        region: "us-east-1",
+      });
+
       // VertexAI works with or without config
       const vertexResponse = await makeZodVerifiedAPICall(
         PutLlmConnectionV1Response,
@@ -904,6 +924,41 @@ describe("/api/public/llm-connections API Endpoints", () => {
         expect(dbConnection?.config).toEqual({ region: "us-east-1" });
       });
 
+      it("should create Bedrock connection with a Bedrock API key", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-api-key-config-test"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9876" }),
+          config: {
+            region: "us-west-2",
+          },
+        };
+
+        const response = await makeZodVerifiedAPICall(
+          PutLlmConnectionV1Response,
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+          201,
+        );
+
+        expect(response.status).toBe(201);
+        expect(response.body.displaySecretKey).toBe("...9876");
+        expect(response.body.config).toEqual({ region: "us-west-2" });
+
+        const dbConnection = await prisma.llmApiKeys.findUnique({
+          where: {
+            projectId_provider: {
+              projectId,
+              provider: createData.provider,
+            },
+          },
+        });
+
+        expect(dbConnection?.displaySecretKey).toBe("...9876");
+      });
+
       it("should reject Bedrock connection without config", async () => {
         const createData = {
           provider: generateUniqueProvider("bedrock-no-config"),
@@ -954,6 +1009,48 @@ describe("/api/public/llm-connections API Endpoints", () => {
         );
       });
 
+      it("should reject Bedrock connection with default credentials sentinel on cloud", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-default-creds"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: "__BEDROCK_DEFAULT_CREDENTIALS__",
+          config: { region: "us-east-1" },
+        };
+
+        const response = await makeAPICall(
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+        );
+
+        expect(response.status).toBe(400);
+        expect(JSON.stringify(response.body)).toContain(
+          "Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
+        );
+      });
+
+      it("should reject Bedrock connection with invalid credential JSON", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-invalid-creds"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ unknownField: "value" }),
+          config: { region: "us-east-1" },
+        };
+
+        const response = await makeAPICall(
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+        );
+
+        expect(response.status).toBe(400);
+        expect(JSON.stringify(response.body)).toContain(
+          "Invalid Bedrock credentials",
+        );
+      });
+
       it("should create VertexAI connection with location config", async () => {
         const createData = {
           provider: generateUniqueProvider("vertexai-config-test"),

web/src/__tests__/server/repositories/event-repository.servertest.ts

@@ -39,6 +39,34 @@ describe("Clickhouse Events Repository Test", () => {
   });
 
   maybe("getObservationsWithModelDataFromEventsTable", () => {
+    it("should return trace tags for events table observations", async () => {
+      const traceId = randomUUID();
+      const observationId = randomUUID();
+
+      await createEventsCh([
+        createEvent({
+          id: observationId,
+          span_id: observationId,
+          project_id: projectId,
+          trace_id: traceId,
+          type: "SPAN",
+          name: "tagged-event",
+          tags: ["chat", "prod"],
+        }),
+      ]);
+
+      const result = await getObservationsWithModelDataFromEventsTable({
+        projectId,
+        filter: [idFilter(observationId)],
+        limit: 1000,
+        offset: 0,
+      });
+
+      const observation = result.find((o) => o.id === observationId);
+      expect(observation).toBeDefined();
+      expect(observation?.traceTags).toEqual(["chat", "prod"]);
+    });
+
     it("should return observations with model data", async () => {
       const traceId = randomUUID();
       const generationId = randomUUID();
@@ -1219,6 +1247,37 @@ describe("Clickhouse Events Repository Test", () => {
   });
 
   maybe("getObservationByIdFromEventsTable", () => {
+    it("should return trace-level fields for event observations", async () => {
+      const traceId = randomUUID();
+      const spanId = randomUUID();
+
+      await createEventsCh([
+        createEvent({
+          id: spanId,
+          span_id: spanId,
+          project_id: projectId,
+          trace_id: traceId,
+          type: "SPAN",
+          name: "test-trace-fields-byid",
+          trace_name: "trace-with-tags",
+          tags: ["chat", "prod"],
+          user_id: "user-123",
+          session_id: "session-123",
+        }),
+      ]);
+
+      const observation = await getObservationByIdFromEventsTable({
+        id: spanId,
+        projectId,
+      });
+
+      expect(observation).toBeDefined();
+      expect(observation?.traceName).toBe("trace-with-tags");
+      expect(observation?.traceTags).toEqual(["chat", "prod"]);
+      expect(observation?.userId).toBe("user-123");
+      expect(observation?.sessionId).toBe("session-123");
+    });
+
     it("should return observation by id with input and output", async () => {
       const traceId = randomUUID();
       const generationId = randomUUID();

web/src/__tests__/server/repositories/experiment-repository.servertest.ts

@@ -333,7 +333,7 @@ describe("Clickhouse Experiment Repository Test", () => {
       const now = new Date().getTime();
 
       // Trace 1: Multiple events with timing data to test latency calculation
-      // Latency should be: earliest start_time to latest end_time
+      // Latency is calculated from ROOT SPAN only (span_id = experiment_item_root_span_id)
       const trace1Id = randomUUID();
       const rootSpanId = randomUUID();
       const event1 = createEvent({
@@ -351,8 +351,8 @@ describe("Clickhouse Experiment Repository Test", () => {
         experiment_item_id: randomUUID(),
         experiment_item_version: null,
         experiment_item_root_span_id: rootSpanId,
-        start_time: (now - 3500) * 1000, // Earliest start: now - 3500ms (convert to microseconds)
-        end_time: (now - 2500) * 1000, // End: now - 2500ms (convert to microseconds)
+        start_time: (now - 3500) * 1000, // Root span start (convert to microseconds)
+        end_time: (now - 2500) * 1000, // Root span end: latency = 1000ms (convert to microseconds)
       });
 
       const childSpan1Id = randomUUID();
@@ -394,7 +394,7 @@ describe("Clickhouse Experiment Repository Test", () => {
         experiment_item_version: null,
         experiment_item_root_span_id: event1.experiment_item_root_span_id,
         start_time: (now - 3000) * 1000,
-        end_time: (now - 1500) * 1000, // Latest end: now - 1500ms (convert to microseconds)
+        end_time: (now - 1500) * 1000, // Child spans are NOT included in latency calculation
       });
 
       // Trace 2: Single event with known latency (1000ms)
@@ -415,8 +415,8 @@ describe("Clickhouse Experiment Repository Test", () => {
         experiment_item_id: randomUUID(),
         experiment_item_version: null,
         experiment_item_root_span_id: rootSpan2Id,
-        start_time: (now - 2500) * 1000, // Start: now - 2500ms (convert to microseconds)
-        end_time: (now - 1500) * 1000, // End: now - 1500ms (latency = 1000ms, convert to microseconds)
+        start_time: (now - 2500) * 1000, // Root span start (convert to microseconds)
+        end_time: (now - 1500) * 1000, // Root span end: latency = 1000ms (convert to microseconds)
       });
 
       await createEventsCh([event1, event2, event3, event4]);
@@ -433,7 +433,8 @@ describe("Clickhouse Experiment Repository Test", () => {
       expect(metric.latencyAvg).toBeDefined();
       expect(typeof metric.latencyAvg).toBe("number");
 
-      expect(metric.latencyAvg).toBeCloseTo(1500, -1); // Within 10ms tolerance
+      // Latency avg = (1000ms + 1000ms) / 2 = 1000ms (only root spans count)
+      expect(metric.latencyAvg).toBeCloseTo(1000, -1); // Within 10ms tolerance
     });
 
     it("should handle cost calculations correctly", async () => {

web/src/__tests__/widget-default-view.clienttest.ts

@@ -17,7 +17,7 @@ describe("shouldUseWidgetSSE", () => {
   it("should enable SSE on the v4 beta v2 path", () => {
     expect(
       shouldUseWidgetSSE({
-        isV4BetaEnabled: true,
+        isV4Enabled: true,
         version: "v2",
       }),
     ).toBe(true);
@@ -26,7 +26,7 @@ describe("shouldUseWidgetSSE", () => {
   it("should disable SSE when v4 beta is off", () => {
     expect(
       shouldUseWidgetSSE({
-        isV4BetaEnabled: false,
+        isV4Enabled: false,
         version: "v2",
       }),
     ).toBe(false);
@@ -35,7 +35,7 @@ describe("shouldUseWidgetSSE", () => {
   it("should disable SSE for non-v4 query versions", () => {
     expect(
       shouldUseWidgetSSE({
-        isV4BetaEnabled: true,
+        isV4Enabled: true,
         version: "v1",
       }),
     ).toBe(false);

web/src/components/ChatMessages/messageSearch/context.tsx

@@ -142,6 +142,7 @@ export function useMessageSearch() {
     openSearch: controller.openSearch,
     closeSearch: controller.closeSearch,
     setQueryInput: controller.setQueryInput,
+    blurQueryInput: controller.blurQueryInput,
     nextMatch: controller.nextMatch,
     previousMatch: controller.previousMatch,
   };

web/src/components/ChatMessages/messageSearch/controller.clienttest.ts

@@ -123,4 +123,60 @@ describe("message search controller", () => {
       expect.objectContaining({ from: 5, to: 6 }),
     ]);
   });
+
+  it("preserves leading and trailing whitespace in literal queries", () => {
+    const controller = createMessageSearchController(["page-1"]);
+
+    controller.registerPageMessages("page-1", [
+      {
+        id: "message-1",
+        type: ChatMessageType.System,
+        role: ChatMessageRole.System,
+        content: " foo foo ",
+      },
+    ]);
+
+    expect(commitQuery(controller, " foo")).toEqual([
+      expect.objectContaining({ from: 0, to: 4 }),
+      expect.objectContaining({ from: 4, to: 8 }),
+    ]);
+    expect(controller.getSnapshot().query).toBe(" foo");
+
+    controller.setQueryInput("foo ");
+    controller.nextMatch();
+
+    expect(controller.getSnapshot().query).toBe("foo ");
+    expect(controller.getSnapshot().matches).toEqual([
+      expect.objectContaining({ from: 1, to: 5 }),
+      expect.objectContaining({ from: 5, to: 9 }),
+    ]);
+  });
+
+  it("clears whitespace-only input on blur", () => {
+    const controller = createMessageSearchController(["page-1"]);
+
+    controller.registerPageMessages("page-1", [
+      {
+        id: "message-1",
+        type: ChatMessageType.System,
+        role: ChatMessageRole.System,
+        content: "a   b",
+      },
+    ]);
+
+    controller.setQueryInput("   ");
+    jest.runAllTimers();
+
+    expect(controller.getSnapshot().queryInput).toBe("   ");
+    expect(controller.getSnapshot().query).toBe("   ");
+    expect(controller.getSnapshot().matches).toEqual([
+      expect.objectContaining({ from: 1, to: 4 }),
+    ]);
+
+    controller.blurQueryInput();
+
+    expect(controller.getSnapshot().queryInput).toBe("");
+    expect(controller.getSnapshot().query).toBe("");
+    expect(controller.getSnapshot().matches).toEqual([]);
+  });
 });

web/src/components/ChatMessages/messageSearch/controller.ts

@@ -67,6 +67,7 @@ export type MessageSearchController = {
   openSearch: () => void;
   closeSearch: () => void;
   setQueryInput: (value: string) => void;
+  blurQueryInput: () => void;
   nextMatch: () => void;
   previousMatch: () => void;
   setPageIds: (pageIds: string[]) => void;
@@ -376,7 +377,7 @@ export function createMessageSearchController(
 
     clearPendingQueryTimeout();
 
-    const queryChanged = commitSearchQuery(state.queryInput.trim());
+    const queryChanged = commitSearchQuery(state.queryInput);
     if (queryChanged) {
       emit();
     }
@@ -465,8 +466,7 @@ export function createMessageSearchController(
       state.queryInput = value;
       clearPendingQueryTimeout();
 
-      const nextSearchQuery = value.trim();
-      if (nextSearchQuery === "") {
+      if (value === "") {
         commitSearchQuery("");
         emit();
         return;
@@ -474,20 +474,38 @@ export function createMessageSearchController(
 
       emit();
 
-      if (nextSearchQuery === state.searchQuery) {
+      if (value === state.searchQuery) {
         return;
       }
 
       pendingQueryTimeout = window.setTimeout(() => {
         pendingQueryTimeout = null;
 
-        const queryChanged = commitSearchQuery(nextSearchQuery);
+        const queryChanged = commitSearchQuery(value);
         if (queryChanged) {
           emit();
         }
       }, SEARCH_INPUT_DEBOUNCE_MS);
     },
 
+    blurQueryInput() {
+      if (state.queryInput.trim() !== "") {
+        return;
+      }
+
+      clearPendingQueryTimeout();
+
+      const queryChanged = commitSearchQuery("");
+      const inputChanged = state.queryInput !== "";
+      if (inputChanged) {
+        state.queryInput = "";
+      }
+
+      if (queryChanged || inputChanged) {
+        emit();
+      }
+    },
+
     nextMatch() {
       moveActiveMatch(1);
     },

web/src/components/ChatMessages/messageSearch/toolbar.tsx

@@ -25,6 +25,7 @@ export function MessageSearchToolbar({ className }: { className?: string }) {
     openSearch,
     closeSearch,
     setQueryInput,
+    blurQueryInput,
     nextMatch,
     previousMatch,
   } = useMessageSearch();
@@ -71,6 +72,7 @@ export function MessageSearchToolbar({ className }: { className?: string }) {
         ref={inputRef}
         value={queryInput}
         onChange={(event) => setQueryInput(event.target.value)}
+        onBlur={blurQueryInput}
         placeholder="Find in messages"
         className="h-6 min-w-40 border-0 px-1 text-xs shadow-none focus-visible:ring-0 sm:min-w-56"
         onKeyDown={(event) => {

web/src/components/editor/CodeMirrorEditor.tsx

@@ -6,7 +6,12 @@ import CodeMirror, {
   ViewPlugin,
   type ViewUpdate,
 } from "@uiw/react-codemirror";
-import { RangeSetBuilder, StateEffect, StateField } from "@codemirror/state";
+import {
+  EditorState,
+  RangeSetBuilder,
+  StateEffect,
+  StateField,
+} from "@codemirror/state";
 import { SearchQuery, search, setSearchQuery } from "@codemirror/search";
 import { json, jsonParseLinter } from "@codemirror/lang-json";
 import { linter, type Diagnostic } from "@codemirror/lint";
@@ -428,6 +433,10 @@ export function CodeMirrorEditor({
       }}
       lang={mode === "json" ? "json" : undefined}
       extensions={[
+        // Block document changes (including paste) when not editable; the
+        // `editable` DOM facet alone does not always prevent paste (see CM6
+        // EditorState.readOnly vs EditorView.editable).
+        ...(!editable ? [EditorState.readOnly.of(true)] : []),
         searchHighlightingSupport,
         search(),
         // RTL/bidi support - must be early for proper line decoration

web/src/components/layouts/app-layout/utils/navigationFilters.ts

@@ -8,15 +8,10 @@ import type { NavigationFilterContext } from "./navigationFilters.types";
 import { hasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { hasOrganizationAccess } from "@/src/features/rbac/utils/checkOrganizationAccess";
 import type { User } from "next-auth";
-import type { Flag } from "@/src/features/feature-flags/types";
-import { getExperimentsAccess } from "@/src/features/experiments/utils/experimentsAccess";
 
 /** Organization type from user session (can be null when not in project/org context) */
 type Organization = User["organizations"][number] | null | undefined;
 
-// Admin-only flags that don't respect experimental features
-const adminOnlyFlags: Flag[] = ["experimentsV4Enabled"];
-
 /**
  * Individual filter functions - each handles one concern
  * Exported for testing and composition
@@ -73,32 +68,20 @@ export const filters = {
    * - Experimental features enabled
    * - User is cloud admin
    * - User has specific feature flag
-   * - For v4Beta: show to all cloud users and keep it visible for opted-in users outside cloud
    */
   featureFlags: (route: Route, ctx: NavigationFilterContext): Route | null => {
     if (route.featureFlag === undefined) return route;
 
-    if (route.featureFlag && adminOnlyFlags.includes(route.featureFlag)) {
-      const access = getExperimentsAccess({
-        isLangfuseCloud: ctx.isLangfuseCloud,
-        isV4BetaEnabled: ctx.session?.user?.v4BetaEnabled === true,
-        isAdmin: ctx.cloudAdmin,
-        isFeatureEnabledOnUser:
-          ctx.session?.user?.featureFlags?.[route.featureFlag] === true,
-      });
-
-      return access.isEnabled ? route : null;
+    if (route.featureFlag === "experimentsV4Enabled") {
+      return ctx.isLangfuseCloud && ctx.session?.user?.v4BetaEnabled === true
+        ? route
+        : null;
     }
 
     if (route.featureFlag === "v4BetaToggleVisible") {
-      const hasOptedIn = ctx.session?.user?.v4BetaEnabled === true;
+      const canToggleV4 = ctx.session?.user?.canToggleV4 === true;
 
-      return ctx.isLangfuseCloud ||
-        ctx.enableExperimentalFeatures ||
-        ctx.cloudAdmin ||
-        hasOptedIn
-        ? route
-        : null;
+      return canToggleV4 && ctx.isLangfuseCloud ? route : null;
     }
 
     const hasFlag =

web/src/components/layouts/app-layout/variants/AuthenticatedLayout.tsx

@@ -37,25 +37,21 @@ const PaymentBanner = dynamic(
   },
 );
 
-const V4BetaEnabledBanner = dynamic(
+const V4EnabledBanner = dynamic(
   () =>
-    import("@/src/features/events/components/V4BetaEnabledBanner").then(
-      (mod) => ({
-        default: mod.V4BetaEnabledBanner,
-      }),
-    ),
+    import("@/src/features/events/components/V4EnabledBanner").then((mod) => ({
+      default: mod.V4EnabledBanner,
+    })),
   {
     ssr: false,
   },
 );
 
-const V4BetaPromoBanner = dynamic(
+const V4PromoBanner = dynamic(
   () =>
-    import("@/src/features/events/components/V4BetaPromoBanner").then(
-      (mod) => ({
-        default: mod.V4BetaPromoBanner,
-      }),
-    ),
+    import("@/src/features/events/components/V4PromoBanner").then((mod) => ({
+      default: mod.V4PromoBanner,
+    })),
   {
     ssr: false,
   },
@@ -140,8 +136,8 @@ export function AuthenticatedLayout({
         <SidebarProvider>
           <div className="flex h-dvh w-full flex-col">
             <PaymentBanner />
-            <V4BetaEnabledBanner />
-            <V4BetaPromoBanner />
+            <V4EnabledBanner />
+            <V4PromoBanner />
             <div className="pt-banner-offset flex min-h-0 flex-1">
               <AppSidebar
                 navItems={navigation.mainNavigation}

web/src/components/layouts/routes.tsx

@@ -26,7 +26,7 @@ import { type User } from "next-auth";
 import { type OrganizationScope } from "@/src/features/rbac/constants/organizationAccessRights";
 import { SupportButton } from "@/src/components/nav/support-button";
 import { BookACallButton } from "@/src/components/nav/book-a-call-button";
-import { V4BetaSidebarToggle } from "@/src/features/events/components/V4BetaSidebarToggle";
+import { V4SidebarToggle } from "@/src/features/events/components/V4SidebarToggle";
 import { SidebarMenuButton } from "@/src/components/ui/sidebar";
 import { useCommandMenu } from "@/src/features/command-k-menu/CommandMenuProvider";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -209,7 +209,7 @@ export const ROUTES: Route[] = [
     pathname: "",
     section: RouteSection.Secondary,
     featureFlag: "v4BetaToggleVisible",
-    menuNode: <V4BetaSidebarToggle />,
+    menuNode: <V4SidebarToggle />,
   },
   {
     title: "Settings",

web/src/components/onboarding/TracesOnboarding.tsx

@@ -1,6 +1,5 @@
 import React from "react";
-import { SplashScreen } from "@/src/components/ui/splash-screen";
-import { TracingSetup } from "@/src/pages/project/[projectId]/traces/setup";
+import { TracesSetupOnboardingCard } from "@/src/features/setup/components/TracesSetupOnboardingCard";
 
 interface TracesOnboardingProps {
   projectId: string;
@@ -8,15 +7,8 @@ interface TracesOnboardingProps {
 
 export function TracesOnboarding({ projectId }: TracesOnboardingProps) {
   return (
-    <SplashScreen
-      title="You don't have any traces yet"
-      description="Traces show you how your LLM calls behave in your application: what they cost, how they perform, and where things go wrong. It's the first step towards improving the behavior of your app."
-      videoSrc="https://static.langfuse.com/prod-assets/onboarding/tracing-overview-v1.mp4"
-    >
-      <div className="mt-8">
-        <h3 className="mb-8 text-2xl font-semibold">Get started</h3>
-        <TracingSetup projectId={projectId} hasTracingConfigured={false} />
-      </div>
-    </SplashScreen>
+    <div className="space-y-10">
+      <TracesSetupOnboardingCard projectId={projectId} />
+    </div>
   );
 }

web/src/components/session/TraceEventsRow.tsx

@@ -33,6 +33,7 @@ export const TraceEventsRow = React.memo(
     traceCommentCounts,
     showCorrections,
     filterState,
+    hideTracePanel = false,
   }: {
     trace: RouterOutputs["sessions"]["tracesFromEvents"][number];
     projectId: string;
@@ -41,6 +42,7 @@ export const TraceEventsRow = React.memo(
     traceCommentCounts: Map<string, number> | undefined;
     showCorrections: boolean;
     filterState: FilterState;
+    hideTracePanel?: boolean;
   }) => {
     const observationsQuery =
       api.sessions.observationsForTraceFromEvents.useQuery(
@@ -59,10 +61,20 @@ export const TraceEventsRow = React.memo(
 
     return (
       <Card className="border-border shadow-none">
-        <div className="grid md:grid-cols-[1fr_1px_358px] lg:grid-cols-[1fr_1px_30rem]">
+        <div
+          className={
+            hideTracePanel
+              ? "grid"
+              : "grid md:grid-cols-[1fr_1px_358px] lg:grid-cols-[1fr_1px_30rem]"
+          }
+        >
           <div className="overflow-hidden py-4 pr-4 pl-4">
             {observationsQuery.isLoading ? (
               <JsonSkeleton className="h-full w-full" numRows={8} />
+            ) : observationsQuery.isError ? (
+              <div className="text-destructive p-2 text-xs">
+                Failed to load observations.
+              </div>
             ) : observationsQuery.data && observationsQuery.data.length > 0 ? (
               <div className="flex flex-col gap-4">
                 {observationsQuery.data.map((observation) => (
@@ -110,78 +122,82 @@ export const TraceEventsRow = React.memo(
               </div>
             )}
           </div>
-          <div className="bg-border hidden md:block"></div>
-          <div className="flex flex-col border-t py-4 pr-4 pl-4 md:border-0">
-            <div className="mb-4 flex flex-col gap-2">
-              <Link
-                href={`/project/${projectId}/traces/${trace.id}`}
-                className="hover:bg-accent flex items-start gap-2 rounded-lg border p-2 transition-colors"
-                onClick={(e) => {
-                  if (!e.metaKey && !e.ctrlKey && !e.shiftKey) {
-                    e.preventDefault();
-                    openPeek(trace.id, trace);
-                  }
-                }}
-              >
-                <ItemBadge type="TRACE" isSmall />
-                <div className="flex flex-col">
-                  <span className="text-xs font-medium">
-                    {trace.name ?? "Trace"} ({trace.id})&nbsp;↗
-                  </span>
-                  <span className="text-muted-foreground text-xs">
-                    {trace.timestamp.toLocaleString()}
-                  </span>
+          {!hideTracePanel && (
+            <>
+              <div className="bg-border hidden md:block"></div>
+              <div className="flex flex-col border-t py-4 pr-4 pl-4 md:border-0">
+                <div className="mb-4 flex flex-col gap-2">
+                  <Link
+                    href={`/project/${projectId}/traces/${trace.id}`}
+                    className="hover:bg-accent flex items-start gap-2 rounded-lg border p-2 transition-colors"
+                    onClick={(e) => {
+                      if (!e.metaKey && !e.ctrlKey && !e.shiftKey) {
+                        e.preventDefault();
+                        openPeek(trace.id, trace);
+                      }
+                    }}
+                  >
+                    <ItemBadge type="TRACE" isSmall />
+                    <div className="flex flex-col">
+                      <span className="text-xs font-medium">
+                        {trace.name ?? "Trace"} ({trace.id})&nbsp;↗
+                      </span>
+                      <span className="text-muted-foreground text-xs">
+                        {trace.timestamp.toLocaleString()}
+                      </span>
+                    </div>
+                  </Link>
+                  <div className="flex flex-wrap gap-2">
+                    <NewDatasetItemFromTraceId
+                      projectId={projectId}
+                      traceId={trace.id}
+                      timestamp={new Date(trace.timestamp)}
+                      buttonVariant="outline"
+                    />
+                    <div className="flex items-start">
+                      <AnnotateDrawer
+                        key={"annotation-drawer" + trace.id}
+                        projectId={projectId}
+                        scoreTarget={{
+                          type: "trace",
+                          traceId: trace.id,
+                        }}
+                        scores={trace.scores}
+                        buttonVariant="outline"
+                        analyticsData={{
+                          type: "trace",
+                          source: "SessionDetail",
+                        }}
+                        scoreMetadata={{
+                          projectId: projectId,
+                          environment: trace.environment ?? undefined,
+                        }}
+                      />
+                      <CreateNewAnnotationQueueItem
+                        projectId={projectId}
+                        objectId={trace.id}
+                        objectType={AnnotationQueueObjectType.TRACE}
+                        variant="outline"
+                      />
+                    </div>
+                    <CommentDrawerButton
+                      projectId={projectId}
+                      variant="outline"
+                      objectId={trace.id}
+                      objectType="TRACE"
+                      count={getNumberFromMap(traceCommentCounts, trace.id)}
+                    />
+                  </div>
                 </div>
-              </Link>
-              <div className="flex flex-wrap gap-2">
-                <NewDatasetItemFromTraceId
-                  projectId={projectId}
-                  traceId={trace.id}
-                  timestamp={new Date(trace.timestamp)}
-                  buttonVariant="outline"
-                />
-                <div className="flex items-start">
-                  <AnnotateDrawer
-                    key={"annotation-drawer" + trace.id}
-                    projectId={projectId}
-                    scoreTarget={{
-                      type: "trace",
-                      traceId: trace.id,
-                    }}
-                    scores={trace.scores}
-                    buttonVariant="outline"
-                    analyticsData={{
-                      type: "trace",
-                      source: "SessionDetail",
-                    }}
-                    scoreMetadata={{
-                      projectId: projectId,
-                      environment: trace.environment ?? undefined,
-                    }}
-                  />
-                  <CreateNewAnnotationQueueItem
-                    projectId={projectId}
-                    objectId={trace.id}
-                    objectType={AnnotationQueueObjectType.TRACE}
-                    variant="outline"
-                  />
+                <div className="flex-1">
+                  <p className="mb-1 font-medium">Scores</p>
+                  <div className="flex flex-wrap content-start items-start gap-1">
+                    <GroupedScoreBadges scores={trace.scores} />
+                  </div>
                 </div>
-                <CommentDrawerButton
-                  projectId={projectId}
-                  variant="outline"
-                  objectId={trace.id}
-                  objectType="TRACE"
-                  count={getNumberFromMap(traceCommentCounts, trace.id)}
-                />
               </div>
-            </div>
-            <div className="flex-1">
-              <p className="mb-1 font-medium">Scores</p>
-              <div className="flex flex-wrap content-start items-start gap-1">
-                <GroupedScoreBadges scores={trace.scores} />
-              </div>
-            </div>
-          </div>
+            </>
+          )}
         </div>
       </Card>
     );
@@ -201,6 +217,7 @@ export const LazyTraceEventsRow = React.forwardRef<
     traceCommentCounts: Map<string, number> | undefined;
     showCorrections: boolean;
     filterState: FilterState;
+    hideTracePanel?: boolean;
     onLoad?: (index: number) => void;
   }
 >((props, measureRef) => {

web/src/components/table/table-view-presets/hooks/useTableViewManager.ts

@@ -6,7 +6,6 @@ import {
   type TableViewPresetState,
   type ColumnDefinition,
 } from "@langfuse/shared";
-import { type DefaultViewScope } from "@langfuse/shared/src/server";
 import { useRouter } from "next/router";
 import { useEffect, useCallback, useState, useRef } from "react";
 import { type VisibilityState } from "@tanstack/react-table";
@@ -31,6 +30,7 @@ interface TableStateUpdaters {
 interface UseTableStateProps {
   tableName: TableViewPresetTableName;
   projectId: string;
+  viewPersistenceKey?: string;
   stateUpdaters: TableStateUpdaters;
   validationContext?: {
     columns?: LangfuseColumnDef<any, any>[];
@@ -46,6 +46,7 @@ interface UseTableStateProps {
 export function useTableViewManager({
   projectId,
   tableName,
+  viewPersistenceKey,
   stateUpdaters,
   validationContext = {},
   currentFilterState,
@@ -58,9 +59,14 @@ export function useTableViewManager({
   const capture = usePostHogClientCapture();
   const pendingFiltersRef = useRef<FilterState | null>(null);
   const pendingFiltersPreviousStateRef = useRef<FilterState | null>(null);
+  // Session storage needs a mode-specific key because the same tableName can be
+  // rendered by different route variants (for example legacy vs v4 pages) with
+  // distinct saved-view IDs. Reusing tableName would restore stale IDs across
+  // modes and boot the user into an incompatible saved view.
+  const resolvedViewPersistenceKey = viewPersistenceKey ?? tableName;
 
   const [storedViewId, setStoredViewId] = useSessionStorage<string | null>(
-    `${tableName}-${projectId}-viewId`,
+    `${resolvedViewPersistenceKey}-${projectId}-viewId`,
     null,
   );
   const [selectedViewIdParam, setSelectedViewId] = useQueryParam(
@@ -288,6 +294,10 @@ export function useTableViewManager({
     if (isInitializedRef.current) return;
     if (selectedViewIdRef.current !== requestedViewId) return;
     if (selectedViewData.id !== requestedViewId) return;
+    if (selectedViewData.tableName !== tableName) {
+      handleSetViewId(null);
+      return;
+    }
 
     // Track permalink visit
     capture("saved_views:permalink_visit", {
@@ -304,6 +314,7 @@ export function useTableViewManager({
     isSelectedViewSuccess,
     selectedViewData,
     selectedViewId,
+    handleSetViewId,
     capture,
     tableName,
     applyViewState,
@@ -368,6 +379,6 @@ export function useTableViewManager({
     applyViewState,
     handleSetViewId,
     selectedViewId,
-    defaultViewScope: resolvedDefault?.scope as DefaultViewScope | null,
+    defaultViewScope: resolvedDefault?.scope ?? null,
   };
 }

web/src/components/table/use-cases/observations.tsx

@@ -12,8 +12,9 @@ import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState
 import { usePaginationState } from "@/src/hooks/usePaginationState";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
-  observationFilterConfig,
+  getObservationsFilterConfig,
   OBSERVATION_COLUMN_TO_BACKEND_KEY,
+  type ObservationsOmittableFilterColumn,
 } from "@/src/features/filters/config/observations-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { transformFiltersForBackend } from "@/src/features/filters/lib/filter-transform";
@@ -134,9 +135,10 @@ export type ObservationsTableProps = {
   promptName?: string;
   promptVersion?: number;
   modelId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: ObservationsOmittableFilterColumn[];
   // External control props for embedded preview tables
   hideControls?: boolean;
+  viewPersistenceKey?: string;
   externalFilterState?: FilterState;
   externalDateRange?: TableDateRange;
   limitRows?: number;
@@ -147,11 +149,17 @@ export default function ObservationsTable({
   promptName,
   promptVersion,
   modelId,
+  omittedFilter = [],
   hideControls = false,
+  viewPersistenceKey,
   externalFilterState,
   externalDateRange,
   limitRows,
 }: ObservationsTableProps) {
+  const observationsFilterConfig = useMemo(
+    () => getObservationsFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
   const router = useRouter();
   const { viewId } = router.query;
   const utils = api.useUtils();
@@ -435,7 +443,7 @@ export default function ObservationsTable({
   }, [environmentFilterOptions.data, filterOptions.data]);
 
   const queryFilter = useSidebarFilterState(
-    observationFilterConfig,
+    observationsFilterConfig,
     newFilterOptions,
     {
       loading: filterOptions.isPending || environmentFilterOptions.isPending,
@@ -468,7 +476,7 @@ export default function ObservationsTable({
   const backendFilterState = transformFiltersForBackend(
     filterState,
     OBSERVATION_COLUMN_TO_BACKEND_KEY,
-    observationFilterConfig.columnDefinitions,
+    observationsFilterConfig.columnDefinitions,
   );
 
   const getCountPayload = {
@@ -1215,6 +1223,7 @@ export default function ObservationsTable({
   const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
     tableName: TableViewPresetTableName.Observations,
     projectId,
+    viewPersistenceKey,
     stateUpdaters: {
       setOrderBy: setOrderByState,
       setFilters: setFiltersWrapper,
@@ -1224,7 +1233,7 @@ export default function ObservationsTable({
     },
     validationContext: {
       columns,
-      filterColumnDefinition: observationFilterConfig.columnDefinitions,
+      filterColumnDefinition: observationsFilterConfig.columnDefinitions,
     },
     currentFilterState: queryFilter.explicitFilterState,
     disabled: hideControls,
@@ -1287,7 +1296,7 @@ export default function ObservationsTable({
   }, [generations]);
 
   return (
-    <DataTableControlsProvider tableName={observationFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={observationsFilterConfig.tableName}>
       <div className="flex h-full w-full flex-col">
         {/* Toolbar spanning full width */}
         {!hideControls && (

web/src/components/table/use-cases/sessions.tsx

@@ -12,8 +12,9 @@ import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
-  sessionFilterConfig,
+  getSessionFilterConfig,
   SESSION_COLUMN_TO_BACKEND_KEY,
+  type SessionOmittableFilterColumn,
 } from "@/src/features/filters/config/sessions-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { transformFiltersForBackend } from "@/src/features/filters/lib/filter-transform";
@@ -78,7 +79,7 @@ export type SessionTableRow = {
 export type SessionTableProps = {
   projectId: string;
   userId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: SessionOmittableFilterColumn[];
   isBetaEnabled?: boolean;
 };
 
@@ -88,6 +89,10 @@ export default function SessionsTable({
   omittedFilter = [],
   isBetaEnabled = false,
 }: SessionTableProps) {
+  const sessionsFilterConfig = useMemo(
+    () => getSessionFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
   const { setDetailPageList } = useDetailPageLists();
   const { timeRange, setTimeRange } = useTableDateRange(projectId);
 
@@ -239,7 +244,7 @@ export default function SessionsTable({
   }, [environmentOptions, filterOptions.data]);
 
   const queryFilter = useSidebarFilterState(
-    sessionFilterConfig,
+    sessionsFilterConfig,
     newFilterOptions,
     {
       loading: filterOptions.isPending || environmentFilterOptions.isPending,
@@ -266,7 +271,7 @@ export default function SessionsTable({
   const backendFilterState = transformFiltersForBackend(
     combinedFilterState,
     SESSION_COLUMN_TO_BACKEND_KEY,
-    sessionFilterConfig.columnDefinitions,
+    sessionsFilterConfig.columnDefinitions,
   );
 
   const payloadCount = {
@@ -528,7 +533,7 @@ export default function SessionsTable({
     },
     {
       accessorKey: "userIds",
-      enableColumnFilter: !omittedFilter.find((f) => f === "userIds"),
+      enableColumnFilter: !omittedFilter.includes("userIds"),
       id: "userIds",
       header: "User IDs",
       size: 200,
@@ -754,13 +759,13 @@ export default function SessionsTable({
     },
     validationContext: {
       columns,
-      filterColumnDefinition: sessionFilterConfig.columnDefinitions,
+      filterColumnDefinition: sessionsFilterConfig.columnDefinitions,
     },
     currentFilterState: queryFilter.explicitFilterState,
   });
 
   return (
-    <DataTableControlsProvider tableName={sessionFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={sessionsFilterConfig.tableName}>
       <div className="flex h-full w-full flex-col">
         {/* Toolbar spanning full width */}
         <DataTableToolbar

web/src/components/table/use-cases/traces.tsx

@@ -73,7 +73,10 @@ import {
 import { Button } from "@/src/components/ui/button";
 import TableIdOrName from "@/src/components/table/table-id";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
-import { traceFilterConfig } from "@/src/features/filters/config/traces-config";
+import {
+  getTraceFilterConfig,
+  type TraceOmittableFilterColumn,
+} from "@/src/features/filters/config/traces-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { PeekViewTraceDetail } from "@/src/components/table/peek/peek-trace-detail";
 import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNavigation";
@@ -135,8 +138,9 @@ export type TracesTableRow = {
 export type TracesTableProps = {
   projectId: string;
   userId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: TraceOmittableFilterColumn[];
   hideControls?: boolean;
+  viewPersistenceKey?: string;
   externalFilterState?: FilterState;
   externalDateRange?: TableDateRange;
   limitRows?: number;
@@ -147,10 +151,15 @@ export default function TracesTable({
   userId,
   omittedFilter = [],
   hideControls = false,
+  viewPersistenceKey,
   externalFilterState,
   externalDateRange,
   limitRows,
 }: TracesTableProps) {
+  const tracesFilterConfig = useMemo(
+    () => getTraceFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
   const utils = api.useUtils();
   const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
   const [rawRefreshInterval, setRawRefreshInterval] =
@@ -326,7 +335,7 @@ export default function TracesTable({
     };
   }, [environmentFilterOptions.data, traceFilterOptionsResponse.data]);
 
-  const queryFilter = useSidebarFilterState(traceFilterConfig, filterOptions, {
+  const queryFilter = useSidebarFilterState(tracesFilterConfig, filterOptions, {
     loading:
       traceFilterOptionsResponse.isPending ||
       environmentFilterOptions.isPending,
@@ -863,7 +872,7 @@ export default function TracesTable({
         ]),
     {
       accessorKey: "sessionId",
-      enableColumnFilter: !omittedFilter.find((f) => f === "sessionId"),
+      enableColumnFilter: !omittedFilter.includes("sessionId"),
       id: "sessionId",
       header: "Session",
       size: 150,
@@ -1240,6 +1249,7 @@ export default function TracesTable({
   const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
     tableName: TableViewPresetTableName.Traces,
     projectId,
+    viewPersistenceKey,
     stateUpdaters: {
       setOrderBy: setOrderByState,
       setFilters: setFiltersWrapper,
@@ -1249,7 +1259,7 @@ export default function TracesTable({
     },
     validationContext: {
       columns,
-      filterColumnDefinition: traceFilterConfig.columnDefinitions,
+      filterColumnDefinition: tracesFilterConfig.columnDefinitions,
     },
     currentFilterState: queryFilter.explicitFilterState,
     disabled: hideControls,
@@ -1302,7 +1312,7 @@ export default function TracesTable({
   }, [traces.isSuccess, traceRowData?.rows]);
 
   return (
-    <DataTableControlsProvider tableName={traceFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={tracesFilterConfig.tableName}>
       <div className="flex h-full w-full flex-col">
         {/* Toolbar spanning full width */}
         {!hideControls && (

web/src/components/trace2/components/ObservationDetailView/ObservationDetailView.tsx

@@ -63,6 +63,7 @@ import {
   aggregateTraceMetrics,
   getDescendantIds,
 } from "@/src/components/trace2/lib/trace-aggregation";
+import TagList from "@/src/features/tag/components/TagList";
 
 export interface ObservationDetailViewProps {
   observation: ObservationReturnTypeWithMetadata;
@@ -82,9 +83,9 @@ export function ObservationDetailView({
   } = useSelection();
 
   // V4 beta mode and observations for log tab
-  const { isBetaEnabled: isV4BetaEnabled } = useV4Beta();
+  const { isBetaEnabled: isV4Enabled } = useV4Beta();
   const { observations, roots, nodeMap } = useTraceData();
-  const showLogViewTab = isV4BetaEnabled && observations.length > 0;
+  const showLogViewTab = isV4Enabled && observations.length > 0;
   const isLogViewVirtualized =
     observations.length >= TRACE_VIEW_CONFIG.logView.virtualizationThreshold;
 
@@ -397,6 +398,25 @@ export function ObservationDetailView({
                 : "overflow-auto pb-4"
             }`}
           >
+            {isRoot &&
+              observation.traceTags &&
+              observation.traceTags.length > 0 && (
+                <>
+                  <div
+                    className={`px-2 pt-2 text-sm font-medium ${currentView !== "pretty" ? "shrink-0" : ""}`}
+                  >
+                    Tags
+                  </div>
+                  <div
+                    className={`flex flex-wrap gap-x-1 gap-y-1 px-2 pb-2 ${currentView !== "pretty" ? "shrink-0" : ""}`}
+                  >
+                    <TagList
+                      selectedTags={observation.traceTags}
+                      isLoading={false}
+                    />
+                  </div>
+                </>
+              )}
             <IOPreview
               key={observation.id}
               observationName={observation.name ?? undefined}

web/src/components/ui/CodeJsonViewer.tsx

@@ -1,4 +1,4 @@
-import { useMemo, useRef, useState } from "react";
+import { useMemo, useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import {
   Check,
@@ -24,6 +24,7 @@ import {
   usePromptReferenceProjectId,
 } from "@/src/components/ui/PromptReferences";
 import { copyTextToClipboard } from "@/src/utils/clipboard";
+import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
 
 export const IO_TABLE_CHAR_LIMIT = 10000;
 
@@ -221,32 +222,32 @@ export function CodeView(props: {
 }) {
   const { copiedToClipboardMessage } = props;
 
-  const [isCopied, setIsCopied] = useState(false);
   const [isCollapsed, setCollapsed] = useState(props.defaultCollapsed);
 
-  const copySuccessStateDuration = copiedToClipboardMessage ? 3_000 : 1_000;
-  const copySuccessTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(
-    null,
-  );
+  const { copy, isCopied } = useCopyToClipboard({
+    successDuration: copiedToClipboardMessage ? 3_000 : 1_000,
+  });
 
-  const handleCopy = (event: React.MouseEvent<HTMLButtonElement>) => {
+  const handleCopy = async (event: React.MouseEvent<HTMLButtonElement>) => {
     event.preventDefault();
-    setIsCopied(true);
+    const button = event.currentTarget;
     const content =
       props.originalContent ??
       (typeof props.content === "string"
         ? props.content
         : (props.content?.join("\n") ?? ""));
-    void copyTextToClipboard(content);
-    if (copySuccessTimeoutRef.current)
-      clearTimeout(copySuccessTimeoutRef.current);
-    copySuccessTimeoutRef.current = setTimeout(
-      () => setIsCopied(false),
-      copySuccessStateDuration,
-    );
 
-    // Keep focus on the copy button to prevent focus shifting
-    event.currentTarget.focus();
+    try {
+      await copy(content);
+    } catch {
+      // Clipboard writes can be rejected when the browser denies permission.
+    }
+
+    if (button) {
+      // Keep focus on the copy button to prevent focus shifting
+      // Note: the original button might no longer be in the DOM if React re-rendered the component after the state update.
+      button.focus();
+    }
   };
 
   const handleShowAll = () => setCollapsed(!isCollapsed);
@@ -339,10 +340,13 @@ export const JsonSkeleton = ({
   borderless?: boolean;
   className?: string;
 }) => {
+  const isSingleLine = numRows === 1;
+
   return (
     <div
       className={cn(
-        "w-[400px] rounded-md",
+        isSingleLine ? "w-full" : "w-[400px]",
+        "rounded-md",
         borderless ? "" : "border",
         className,
       )}
@@ -352,7 +356,7 @@ export const JsonSkeleton = ({
           <Skeleton
             className={cn(
               "h-4 w-full",
-              i === numRows - 1 ? "w-3/4" : undefined,
+              !isSingleLine && i === numRows - 1 ? "w-3/4" : undefined,
             )}
             key={i}
           />

web/src/components/ui/IOTableCell.tsx

@@ -87,6 +87,7 @@ export const IOTableCell = ({
     return (
       <JsonSkeleton
         borderless
+        numRows={singleLine ? 1 : undefined}
         className="h-full w-full overflow-hidden px-2 py-1"
       />
     );

web/src/components/ui/splash-screen.tsx

@@ -4,6 +4,7 @@ import Image from "next/image";
 import { InfoIcon } from "lucide-react";
 import { ActionButton } from "@/src/components/ActionButton";
 import { Alert, AlertTitle, AlertDescription } from "@/src/components/ui/alert";
+import { StatusBadge } from "@/src/components/layouts/status-badge";
 
 export interface ValueProposition {
   title: string;
@@ -18,9 +19,17 @@ export interface ActionConfig {
   component?: React.ReactNode;
 }
 
+export interface Step {
+  title: string;
+  description?: string;
+  badge?: React.ReactNode;
+  content?: React.ReactNode;
+}
+
 export interface SplashScreenProps {
   title: string;
   description: string;
+  waitingFor?: string;
   image?: {
     src: string;
     alt: string;
@@ -28,19 +37,17 @@ export interface SplashScreenProps {
     height: number;
   };
   videoSrc?: string;
+  steps?: Step[];
   valuePropositions?: ValueProposition[];
   primaryAction?: ActionConfig;
   secondaryAction?: ActionConfig;
   gettingStarted?: string | React.ReactNode;
   children?: React.ReactNode;
-  className?: string;
+  /** Where to render the video. Defaults to "top" (after header, before content) */
+  videoPosition?: "top" | "bottom";
 }
 
-interface VideoPlayerProps {
-  videoSrc: string;
-}
-
-function VideoPlayer({ videoSrc }: VideoPlayerProps) {
+function VideoPlayer({ videoSrc }: { videoSrc: string }) {
   const [hasError, setHasError] = useState(false);
   const [isLoaded, setIsLoaded] = useState(false);
 
@@ -48,9 +55,7 @@ function VideoPlayer({ videoSrc }: VideoPlayerProps) {
     <div
       className={cn(
         "border-border my-6 w-full max-w-3xl overflow-hidden rounded-lg border",
-        {
-          hidden: !isLoaded || hasError,
-        },
+        { hidden: !isLoaded || hasError },
       )}
     >
       <video
@@ -72,17 +77,46 @@ function VideoPlayer({ videoSrc }: VideoPlayerProps) {
 export function SplashScreen({
   title,
   description,
+  waitingFor,
   image,
   videoSrc,
+  steps,
   valuePropositions = [],
   primaryAction,
   secondaryAction,
   gettingStarted,
   children,
+  videoPosition = "top",
 }: SplashScreenProps) {
+  const mediaBlock = (
+    <>
+      {videoSrc && <VideoPlayer videoSrc={videoSrc} />}
+      {!videoSrc && image && (
+        <div className="my-6 w-full max-w-3xl">
+          <Image
+            src={image.src}
+            alt={image.alt}
+            width={image.width}
+            height={image.height}
+            className="rounded-md"
+          />
+        </div>
+      )}
+    </>
+  );
+
   return (
-    <div className={cn("mx-auto flex max-w-4xl flex-col items-center p-8")}>
+    <div className="mx-auto flex max-w-4xl flex-col items-center p-8">
       <div className="mb-6 text-center">
+        {waitingFor && (
+          <StatusBadge
+            type="waiting"
+            showText={false}
+            className="mb-3 px-3 py-1 text-sm"
+          >
+            {waitingFor}
+          </StatusBadge>
+        )}
         <h2 className="mb-2 text-2xl font-bold">{title}</h2>
         <p className="text-muted-foreground">{description}</p>
       </div>
@@ -98,7 +132,6 @@ export function SplashScreen({
               {primaryAction.label}
             </ActionButton>
           ))}
-
         {secondaryAction &&
           (secondaryAction.component || (
             <ActionButton
@@ -120,17 +153,42 @@ export function SplashScreen({
         </Alert>
       )}
 
-      {videoSrc && <VideoPlayer videoSrc={videoSrc} />}
+      {videoPosition === "top" && mediaBlock}
 
-      {!videoSrc && image && (
-        <div className="my-6 w-full max-w-3xl">
-          <Image
-            src={image.src}
-            alt={image.alt}
-            width={image.width}
-            height={image.height}
-            className="rounded-md"
-          />
+      {/* Numbered steps */}
+      {steps && steps.length > 0 && (
+        <div className="w-full max-w-4xl">
+          {steps.map((step, index) => (
+            <div key={index} className="flex gap-4">
+              {/* Left: circle + connecting line */}
+              <div className="flex flex-col items-center">
+                <div className="bg-foreground text-background flex h-8 w-8 shrink-0 items-center justify-center rounded-full text-sm font-semibold">
+                  {index + 1}
+                </div>
+                {index < steps.length - 1 && (
+                  <div className="bg-border mt-1 w-px flex-1" />
+                )}
+              </div>
+              {/* Right: title + content */}
+              <div
+                className={cn(
+                  "min-w-0 flex-1 pt-1",
+                  index < steps.length - 1 ? "pb-8" : "pb-0",
+                )}
+              >
+                <div className="mb-2 flex items-center gap-3">
+                  <h3 className="text-xl font-semibold">{step.title}</h3>
+                  {step.badge}
+                </div>
+                {step.description && (
+                  <p className="text-muted-foreground text-sm leading-6">
+                    {step.description}
+                  </p>
+                )}
+                {step.content && <div className="mt-3">{step.content}</div>}
+              </div>
+            </div>
+          ))}
         </div>
       )}
 
@@ -147,6 +205,8 @@ export function SplashScreen({
           ))}
         </div>
       )}
+
+      {videoPosition === "bottom" && mediaBlock}
     </div>
   );
 }

web/src/components/ui/table.tsx

@@ -1,6 +1,9 @@
 import * as React from "react";
 
+import { Button } from "@/src/components/ui/button";
+import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
 import { cn } from "@/src/utils/tailwind";
+import { Check, Copy } from "lucide-react";
 
 type TableDensity = "compact" | "comfortable";
 
@@ -101,6 +104,59 @@ const TableCell = React.forwardRef<
 ));
 TableCell.displayName = "TableCell";
 
+type TableCellWithCopyButtonProps =
+  React.TdHTMLAttributes<HTMLTableCellElement> & {
+    text: string;
+    density?: TableDensity;
+    copyButtonLabel?: string;
+  };
+
+const TableCellWithCopyButton = React.forwardRef<
+  HTMLTableCellElement,
+  TableCellWithCopyButtonProps
+>(({ text, copyButtonLabel, className, ...props }, ref) => {
+  const { copy, isCopied } = useCopyToClipboard();
+
+  return (
+    <TableCell
+      ref={ref}
+      className={cn("relative min-w-0 pr-10", className)}
+      title={text}
+      {...props}
+    >
+      {text}
+      <Button
+        variant="ghost"
+        size="icon-xs"
+        className="absolute top-1/2 right-2 -translate-y-1/2"
+        title={copyButtonLabel ?? "Copy to clipboard"}
+        aria-label={copyButtonLabel ?? "Copy to clipboard"}
+        onClick={async (event) => {
+          event.preventDefault();
+          const button = event.currentTarget;
+          try {
+            await copy(text);
+          } catch {
+            // Clipboard writes can be rejected when the browser denies permission.
+          }
+
+          if (button) {
+            // The original button might no longer be in the DOM if React re-rendered the component after the state update.
+            button.focus();
+          }
+        }}
+      >
+        {isCopied ? (
+          <Check className="h-3 w-3" />
+        ) : (
+          <Copy className="h-3 w-3" />
+        )}
+      </Button>
+    </TableCell>
+  );
+});
+TableCellWithCopyButton.displayName = "TableCellWithCopyButton";
+
 const TableCaption = React.forwardRef<
   HTMLTableCaptionElement,
   React.HTMLAttributes<HTMLTableCaptionElement>
@@ -121,5 +177,6 @@ export {
   TableHead,
   TableRow,
   TableCell,
+  TableCellWithCopyButton,
   TableCaption,
 };

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.167.4";
+export const VERSION = "v3.168.0";

web/src/ee/features/multi-tenant-sso/utils.ts

@@ -188,6 +188,7 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
+      issuer: "https://github.com/login/oauth",
       ...getClientConfig(provider.authConfig),
     });
   else if (provider.authProvider === "gitlab")
@@ -264,6 +265,8 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       enterprise: {
         baseUrl: provider.authConfig.enterprise.baseUrl,
       },
+      issuer: new URL("/login/oauth", provider.authConfig.enterprise.baseUrl)
+        .href,
       ...getClientConfig(provider.authConfig),
     });
   else if (provider.authProvider === "jumpcloud")

web/src/env.mjs

@@ -106,7 +106,7 @@ export const env = createEnv({
     AUTH_GITHUB_CHECKS: zAuthChecks,
     AUTH_GITHUB_ENTERPRISE_CLIENT_ID: z.string().optional(),
     AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET: z.string().optional(),
-    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().url().optional(),
     AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING: z
       .enum(["true", "false"])
       .optional(),

web/src/features/annotation-queues/components/processors/SessionAnnotationProcessor.tsx

@@ -5,7 +5,8 @@ import {
 import { AnnotationDrawerSection } from "../shared/AnnotationDrawerSection";
 import { AnnotationProcessingLayout } from "../shared/AnnotationProcessingLayout";
 import { SessionIO } from "@/src/components/session";
-import { useState, useEffect } from "react";
+import { LazyTraceEventsRow } from "@/src/components/session/TraceEventsRow";
+import { useState, useMemo, useCallback } from "react";
 import { Button } from "@/src/components/ui/button";
 import { ItemBadge } from "@/src/components/ItemBadge";
 import { CopyIdsPopover } from "@/src/components/trace2/components/_shared/CopyIdsPopover";
@@ -13,6 +14,9 @@ import { Badge } from "@/src/components/ui/badge";
 import { Separator } from "@/src/components/ui/separator";
 import Link from "next/link";
 import { Card } from "@/src/components/ui/card";
+import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
+import { api } from "@/src/utils/api";
+import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";
 
 interface SessionAnnotationProcessorProps {
   item: AnnotationQueueItem & {
@@ -27,39 +31,67 @@ interface SessionAnnotationProcessorProps {
 // some projects have thousands of traces in a session, paginate to avoid rendering all at once
 const PAGE_SIZE = 10;
 
+// Stable empty array to avoid creating new references on every render (defeats React.memo)
+const EMPTY_FILTER_STATE: [] = [];
+
 export const SessionAnnotationProcessor: React.FC<
   SessionAnnotationProcessorProps
 > = ({ item, data, configs, projectId }) => {
   const [visibleTraces, setVisibleTraces] = useState(PAGE_SIZE);
-  const [currentTraceIndex, setCurrentTraceIndex] = useState(1);
+  const { isBetaEnabled } = useV4Beta();
 
-  // Intersection observer to which trace is currently in view
-  useEffect(() => {
-    if (!data?.traces) return;
-
-    const observer = new IntersectionObserver(
-      (entries) => {
-        entries.forEach((entry) => {
-          if (entry.isIntersecting && entry.intersectionRatio > 0.5) {
-            const index = parseInt(
-              entry.target.getAttribute("data-trace-index") || "0",
-            );
-            setCurrentTraceIndex(index + 1);
-          }
-        });
+  // Fetch traces separately when v4 beta is enabled (events table path)
+  // The byIdWithScoresFromEvents endpoint doesn't include traces array
+  const tracesFromEventsQuery = api.sessions.tracesFromEvents.useQuery(
+    { projectId, sessionId: item.objectId },
+    {
+      enabled: isBetaEnabled,
+      retry(failureCount, error) {
+        if (
+          error.data?.code === "UNAUTHORIZED" ||
+          error.data?.code === "NOT_FOUND"
+        )
+          return false;
+        return failureCount < 3;
       },
+    },
+  );
+
+  const traceCommentCounts =
+    api.comments.getTraceCommentCountsBySessionId.useQuery(
       {
-        threshold: 0.5, // Trigger when 50% of trace is visible
-        rootMargin: "-25% 0px -25% 0px", // Focus on center area
+        projectId,
+        sessionId: item.objectId,
       },
+      { enabled: isBetaEnabled },
     );
 
-    // Observe all trace cards
-    const traceCards = document.querySelectorAll("[data-trace-index]");
-    traceCards.forEach((card) => observer.observe(card));
+  // Unify traces from both paths:
+  // - v4 beta OFF: traces come from data.traces (byIdWithScores endpoint)
+  // - v4 beta ON: traces come from separate tracesFromEvents query
+  const traces = useMemo(() => {
+    if (isBetaEnabled) {
+      return tracesFromEventsQuery.data ?? [];
+    }
+    return data?.traces ?? [];
+  }, [isBetaEnabled, tracesFromEventsQuery.data, data?.traces]);
 
-    return () => observer.disconnect();
-  }, [data?.traces, visibleTraces]);
+  // For the "Total traces" badge, show countTraces from session metadata when available (v4),
+  // or fall back to loaded traces length
+  const totalTracesForBadge = useMemo(() => {
+    if (isBetaEnabled) {
+      return data?.countTraces ?? traces.length;
+    }
+    return traces.length;
+  }, [isBetaEnabled, data?.countTraces, traces.length]);
+
+  // Stable callback to avoid creating new function reference on every render (defeats React.memo)
+  const openPeek = useCallback(
+    (traceId: string) => {
+      window.open(`/project/${projectId}/traces/${traceId}`, "_blank");
+    },
+    [projectId],
+  );
 
   const leftPanel = (
     <div className="flex h-full flex-col overflow-hidden">
@@ -77,22 +109,15 @@ export const SessionAnnotationProcessor: React.FC<
               idItems={[{ id: item.objectId, name: "Session ID" }]}
             />
           </div>
-          {data?.traces && (
-            <div className="flex items-center">
-              <Badge variant="outline" className="text-xs">
-                Trace {currentTraceIndex} / {data.traces.length}
-              </Badge>
-            </div>
-          )}
         </div>
         <div className="mt-2 mb-4 grid w-full min-w-0 items-center justify-between px-4">
           <div className="flex max-w-full min-w-0 shrink flex-col">
             <div className="flex max-w-full min-w-0 flex-wrap items-center gap-1">
-              {data.environment && (
+              {data?.environment && (
                 <Badge variant="tertiary">Env: {data.environment}</Badge>
               )}
               <Badge variant="outline">
-                Total traces: {data?.traces.length}
+                Total traces: {totalTracesForBadge}
               </Badge>
             </div>
           </div>
@@ -103,13 +128,53 @@ export const SessionAnnotationProcessor: React.FC<
       {/* Scrollable Content */}
       <div className="flex-1 overflow-y-auto">
         <div className="p-4">
-          {data?.traces
-            .slice(0, visibleTraces)
-            .map((trace: any, index: number) => (
+          {/* Loading state for v4 beta traces */}
+          {isBetaEnabled && tracesFromEventsQuery.isLoading && (
+            <div className="space-y-4">
+              {Array.from({ length: 3 }).map((_, i) => (
+                <Card
+                  key={i}
+                  className="border-border mb-2 grid gap-2 p-2 shadow-none"
+                >
+                  <JsonSkeleton
+                    className="h-full w-full overflow-hidden"
+                    numRows={4}
+                  />
+                </Card>
+              ))}
+            </div>
+          )}
+          {/* Error state for v4 beta traces */}
+          {isBetaEnabled && tracesFromEventsQuery.isError && (
+            <div className="text-destructive p-2 text-sm">
+              Failed to load traces for this session.
+            </div>
+          )}
+          {/* Trace list - v4 path uses LazyTraceEventsRow for deferred loading */}
+          {isBetaEnabled &&
+            tracesFromEventsQuery.isSuccess &&
+            traces
+              .slice(0, visibleTraces)
+              .map((trace: any, index: number) => (
+                <LazyTraceEventsRow
+                  key={trace.id}
+                  trace={trace}
+                  projectId={projectId}
+                  sessionId={item.objectId}
+                  openPeek={openPeek}
+                  traceCommentCounts={traceCommentCounts.data ?? undefined}
+                  showCorrections
+                  filterState={EMPTY_FILTER_STATE}
+                  hideTracePanel
+                  index={index}
+                />
+              ))}
+          {/* Trace list - v3 path uses SessionIO */}
+          {!isBetaEnabled &&
+            traces.slice(0, visibleTraces).map((trace: any) => (
               <Card
                 className="border-border hover:border-ring group mb-2 grid gap-2 p-2 shadow-none"
                 key={trace.id}
-                data-trace-index={index}
               >
                 <div className="-mt-1 p-1 pt-0 opacity-50 transition-opacity group-hover:opacity-100">
                   <Link
@@ -130,16 +195,17 @@ export const SessionAnnotationProcessor: React.FC<
                 />
               </Card>
             ))}
-          {data?.traces && data.traces.length > visibleTraces && (
-            <div className="flex justify-center py-4">
-              <Button
-                onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
-                variant="ghost"
-              >
-                {`Load ${Math.min(data.traces.length - visibleTraces, PAGE_SIZE)} More`}
-              </Button>
-            </div>
-          )}
+          {(!isBetaEnabled || tracesFromEventsQuery.isSuccess) &&
+            traces.length > visibleTraces && (
+              <div className="flex justify-center py-4">
+                <Button
+                  onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
+                  variant="ghost"
+                >
+                  {`Load ${Math.min(traces.length - visibleTraces, PAGE_SIZE)} More`}
+                </Button>
+              </div>
+            )}
         </div>
       </div>
     </div>

web/src/features/auth-credentials/lib/credentialsServerUtils.ts

@@ -42,7 +42,7 @@ export async function createUserEmailPassword(
     },
   });
 
-  await createProjectMembershipsOnSignup(newUser);
+  await createProjectMembershipsOnSignup(newUser, { userWasJustCreated: true });
 
   return newUser.id;
 }

web/src/features/auth/lib/createProjectMembershipsOnSignup.ts

@@ -4,12 +4,18 @@ import { logger } from "@langfuse/shared/src/server";
 import { ServerPosthog } from "@/src/features/posthog-analytics/ServerPosthog";
 import { hasEntitlementBasedOnPlan } from "@/src/features/entitlements/server/hasEntitlement";
 import { getOrganizationPlanServerSide } from "@/src/features/entitlements/server/getPlan";
+import { shouldAutoEnableV4 } from "@/src/features/events/lib/v4Rollout";
 
-export async function createProjectMembershipsOnSignup(user: {
-  id: string;
-  email: string | null;
-}) {
+export async function createProjectMembershipsOnSignup(
+  user: {
+    id: string;
+    email: string | null;
+  },
+  options?: { userWasJustCreated?: boolean },
+) {
   try {
+    const isCloudDeployment = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
+
     // in no case do we want to send duplicate sign up events to posthog
     const isNewUser = !(await prisma.organizationMembership.findFirst({
       where: { userId: user.id },
@@ -150,6 +156,60 @@ export async function createProjectMembershipsOnSignup(user: {
     // Invites do not work for users without emails (some future SSO users)
     if (user.email) await processMembershipInvitations(user.email, user.id);
 
+    if (isCloudDeployment && (options?.userWasJustCreated || isNewUser)) {
+      const userRolloutState = await prisma.user.findUnique({
+        where: { id: user.id },
+        select: {
+          createdAt: true,
+          v4BetaEnabled: true,
+          organizationMemberships: {
+            select: {
+              organization: {
+                select: {
+                  id: true,
+                  createdAt: true,
+                },
+              },
+            },
+          },
+        },
+      });
+
+      if (userRolloutState) {
+        const shouldAutoEnableV4ForUser = shouldAutoEnableV4({
+          userCreatedAt: userRolloutState.createdAt,
+          organizations: userRolloutState.organizationMemberships.map(
+            (membership) => ({
+              id: membership.organization.id,
+              createdAt: membership.organization.createdAt,
+            }),
+          ),
+          excludedOrganizationIds: env.NEXT_PUBLIC_DEMO_ORG_ID
+            ? [env.NEXT_PUBLIC_DEMO_ORG_ID]
+            : [],
+        });
+        const shouldInitializeForNewUser =
+          options?.userWasJustCreated &&
+          !userRolloutState.v4BetaEnabled &&
+          shouldAutoEnableV4ForUser;
+        const shouldInitializeForFirstOrganization =
+          !options?.userWasJustCreated &&
+          isNewUser &&
+          !userRolloutState.v4BetaEnabled &&
+          shouldAutoEnableV4ForUser;
+
+        if (
+          shouldInitializeForNewUser ||
+          shouldInitializeForFirstOrganization
+        ) {
+          await prisma.user.update({
+            where: { id: user.id },
+            data: { v4BetaEnabled: true },
+          });
+        }
+      }
+    }
+
     // for conversion metric tracking in posthog: did a new user sign up?
     if (
       isNewUser &&

web/src/features/blobstorage-integration/blobstorage-integration-router.ts

@@ -274,6 +274,7 @@ export const blobStorageIntegrationRouter = createTRPCRouter({
           forcePathStyle: forcePathStyle || false,
           useAzureBlob: type === BlobStorageIntegrationType.AZURE_BLOB_STORAGE,
           useGoogleCloudStorage: false, // Not supported in blob storage integration
+          useOCIObjectStorage: false, // Not supported in blob storage integration
           googleCloudCredentials: undefined,
           awsSse: undefined,
           awsSseKmsKeyId: undefined,

web/src/features/evals/hooks/useEvalCapabilities.ts

@@ -1,4 +1,6 @@
+import { useSession } from "next-auth/react";
 import { api } from "@/src/utils/api";
+import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
 export interface EvalCapabilities {
   isNewCompatible: boolean;
   allowLegacy: boolean;
@@ -18,6 +20,9 @@ const mockOtelStatus = {
  * @returns Capabilities object indicating which eval features are allowed
  */
 export function useEvalCapabilities(projectId: string): EvalCapabilities {
+  const { data: session, status: sessionStatus } = useSession();
+  const isSessionLoading = sessionStatus === "loading";
+
   // Query OTEL SDK status
   const { isOtel, isPropagating } = mockOtelStatus;
 
@@ -25,13 +30,20 @@ export function useEvalCapabilities(projectId: string): EvalCapabilities {
   const evalCounts = api.evals.counts.useQuery({ projectId });
   const hasLegacyEvals = (evalCounts.data?.legacyConfigCount ?? 0) > 0;
 
+  // Only hide legacy options for new cloud users (canToggleV4 = false)
+  // Non-cloud deployments always see legacy options
+  // Use === true to default to false while session is loading, preventing flash of legacy options
+  // New users (canToggleV4 = false) default to observation-level evals regardless of v3/v4
+  const { isLangfuseCloud } = useLangfuseCloudRegion();
+  const canToggleV4 = session?.user?.canToggleV4 === true;
+
   return {
     isNewCompatible: isOtel,
-    // Allow legacy evals if user already has them OR if not using OTEL
-    allowLegacy: hasLegacyEvals || !isOtel,
+    // Allow legacy if: not cloud OR user has legacy evals OR user can toggle v4 (existing user)
+    allowLegacy: !isLangfuseCloud || hasLegacyEvals || canToggleV4,
     // Allow propagation filters only when using OTEL and spans are propagating
     allowPropagationFilters: isOtel && isPropagating,
-    isLoading: evalCounts.isLoading,
+    isLoading: evalCounts.isLoading || isSessionLoading,
     hasLegacyEvals,
   };
 }

web/src/features/events/components/EventsTable.tsx

@@ -11,7 +11,8 @@ import { usePaginationState } from "@/src/hooks/usePaginationState";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
   getEventsColumnName,
-  observationEventsFilterConfig,
+  getObservationEventsFilterConfig,
+  type ObservationEventsOmittableFilterColumn,
 } from "../config/filter-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { formatIntervalSeconds } from "@/src/utils/dates";
@@ -165,7 +166,9 @@ export type EventsTableRow = {
 export type EventsTableProps = {
   projectId: string;
   userId?: string;
+  omittedFilter?: ObservationEventsOmittableFilterColumn[];
   hideControls?: boolean;
+  viewPersistenceKey?: string;
   // External control props for embedded preview tables
   externalFilterState?: FilterState;
   externalDateRange?: TableDateRange;
@@ -176,7 +179,9 @@ export type EventsTableProps = {
 export default function ObservationsEventsTable({
   projectId,
   userId,
+  omittedFilter = [],
   hideControls = false,
+  viewPersistenceKey,
   externalFilterState,
   externalDateRange,
   limitRows,
@@ -184,6 +189,10 @@ export default function ObservationsEventsTable({
 }: EventsTableProps) {
   const router = useRouter();
   const { viewId } = router.query;
+  const eventsFilterConfig = useMemo(
+    () => getObservationEventsFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
 
   const { setDetailPageList } = useDetailPageLists();
   const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
@@ -353,17 +362,13 @@ export default function ObservationsEventsTable({
     oldFilterState,
   });
 
-  const queryFilter = useSidebarFilterState(
-    observationEventsFilterConfig,
-    filterOptions,
-    {
-      loading: isFilterOptionsPending,
-      disableUrlPersistence: hideControls, // Disable URL persistence for embedded preview tables
-      sessionFilterContextId: projectId,
-      // Sidebar-only implicit environment defaults
-      implicitDefaultConfig: DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG,
-    },
-  );
+  const queryFilter = useSidebarFilterState(eventsFilterConfig, filterOptions, {
+    loading: isFilterOptionsPending,
+    disableUrlPersistence: hideControls, // Disable URL persistence for embedded preview tables
+    sessionFilterContextId: projectId,
+    // Sidebar-only implicit environment defaults
+    implicitDefaultConfig: DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG,
+  });
 
   // Create ref-based wrapper to avoid stale closure when queryFilter updates
   const queryFilterRef = useRef(queryFilter);
@@ -580,6 +585,7 @@ export default function ObservationsEventsTable({
         if (ioLoading) {
           return (
             <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
               borderless
               className="h-full w-full overflow-hidden px-2 py-1"
             />
@@ -605,6 +611,7 @@ export default function ObservationsEventsTable({
         if (ioLoading) {
           return (
             <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
               borderless
               className="h-full w-full overflow-hidden px-2 py-1"
             />
@@ -634,6 +641,7 @@ export default function ObservationsEventsTable({
         if (ioLoading) {
           return (
             <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
               borderless
               className="h-full w-full overflow-hidden px-2 py-1"
             />
@@ -1017,7 +1025,8 @@ export default function ObservationsEventsTable({
       cell: ({ row }) => {
         const traceTags: string[] | undefined = row.getValue("traceTags");
         return (
-          traceTags && (
+          traceTags &&
+          traceTags.length > 0 && (
             <div
               className={cn(
                 "flex gap-x-2 gap-y-1",
@@ -1146,6 +1155,7 @@ export default function ObservationsEventsTable({
   const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
     tableName: TableViewPresetTableName.Observations,
     projectId,
+    viewPersistenceKey,
     stateUpdaters: {
       setOrderBy: setOrderByState,
       setFilters: setFiltersWrapper,
@@ -1155,7 +1165,7 @@ export default function ObservationsEventsTable({
     },
     validationContext: {
       columns,
-      filterColumnDefinition: observationEventsFilterConfig.columnDefinitions,
+      filterColumnDefinition: eventsFilterConfig.columnDefinitions,
     },
     currentFilterState: queryFilter.explicitFilterState,
     disabled: hideControls,
@@ -1210,7 +1220,7 @@ export default function ObservationsEventsTable({
               promptId: observation.promptId ?? undefined,
               promptName: observation.promptName ?? undefined,
               promptVersion: observation.promptVersion?.toString() ?? undefined,
-              traceTags: undefined, // TODO: traceTags not available in EventsObservation
+              traceTags: observation.traceTags ?? undefined,
               traceName: observation.traceName ?? undefined,
               timestamp: observation.startTime ?? undefined,
               usageDetails: observation.usageDetails ?? {},
@@ -1262,9 +1272,7 @@ export default function ObservationsEventsTable({
   }, [selectedObservationIds, observations.rows]);
 
   return (
-    <DataTableControlsProvider
-      tableName={observationEventsFilterConfig.tableName}
-    >
+    <DataTableControlsProvider tableName={eventsFilterConfig.tableName}>
       <div className="flex h-full w-full flex-col">
         {/* Toolbar spanning full width */}
         {!hideControls && (

web/src/features/events/components/V4EnabledBanner.tsx

@@ -16,7 +16,7 @@ const DISMISSED_STORAGE_KEY = "v4-beta-enabled-banner:v1:dismissed";
 const V4_BETA_BANNER_ID = "v4-beta-enabled-banner";
 const V4_BETA_BANNER_ORDER = 20;
 
-export function V4BetaEnabledBanner() {
+export function V4EnabledBanner() {
   const session = useSession();
   const { isBetaEnabled } = useV4Beta();
   const { getTopBannerOffset } = useTopBanner();

web/src/features/events/components/V4IntroDialog.tsx

@@ -6,15 +6,17 @@ import {
   DialogFooter,
 } from "@/src/components/ui/dialog";
 
-export function V4BetaIntroDialog({
+export function V4IntroDialog({
   open,
   onConfirm,
+  onDismiss,
 }: {
   open: boolean;
   onConfirm: () => void;
+  onDismiss: () => void;
 }) {
   return (
-    <Dialog open={open} onOpenChange={(o) => !o && onConfirm()}>
+    <Dialog open={open} onOpenChange={(o) => !o && onDismiss()}>
       <DialogContent
         className="[&>div:last-child]:hidden"
         aria-label="Welcome to a faster Langfuse"

web/src/features/events/components/V4PromoBanner.tsx

@@ -10,7 +10,7 @@ import {
   useTopBannerRegistration,
 } from "@/src/features/top-banner";
 import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
-import { V4BetaIntroDialog } from "@/src/features/events/components/V4BetaIntroDialog";
+import { V4IntroDialog } from "@/src/features/events/components/V4IntroDialog";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
 
@@ -29,14 +29,16 @@ const PAGE_MESSAGES: Record<string, string> = {
   "/project/[projectId]/traces/[traceId]": "Faster trace UI available.",
 };
 
-export function V4BetaPromoBanner() {
+export function V4PromoBanner() {
   const router = useRouter();
   const session = useSession();
   const {
     isBetaEnabled,
+    canToggleV4,
     enableWithIntro,
     showIntroDialog,
     confirmIntroDialog,
+    dismissIntroDialog,
     isLoading,
   } = useV4Beta();
   const capture = usePostHogClientCapture();
@@ -49,12 +51,9 @@ export function V4BetaPromoBanner() {
   const bannerRef = useRef<HTMLDivElement>(null);
 
   const isAuthenticated = session.status === "authenticated";
-  const enableExperimentalFeatures =
-    session.data?.environment?.enableExperimentalFeatures ?? false;
 
-  // Match the v4BetaToggleVisible logic from navigationFilters.ts
-  // cloudAdmin = isLangfuseCloud && isAdmin (already covered by isLangfuseCloud)
-  const isToggleVisible = isLangfuseCloud || enableExperimentalFeatures;
+  // Match the v4BetaToggleVisible logic from navigationFilters.ts.
+  const isToggleVisible = canToggleV4 && isLangfuseCloud;
   const pageMessage = PAGE_MESSAGES[router.pathname];
 
   const isVisible =
@@ -75,9 +74,10 @@ export function V4BetaPromoBanner() {
 
   if (!isVisible) {
     return (
-      <V4BetaIntroDialog
+      <V4IntroDialog
         open={showIntroDialog}
         onConfirm={confirmIntroDialog}
+        onDismiss={dismissIntroDialog}
       />
     );
   }
@@ -137,9 +137,10 @@ export function V4BetaPromoBanner() {
           <X className="h-4 w-4 shrink-0" />
         </Button>
       </div>
-      <V4BetaIntroDialog
+      <V4IntroDialog
         open={showIntroDialog}
         onConfirm={confirmIntroDialog}
+        onDismiss={dismissIntroDialog}
       />
     </div>
   );

web/src/features/events/components/V4SidebarToggle.tsx

@@ -7,7 +7,7 @@ import {
   TooltipTrigger,
 } from "@/src/components/ui/tooltip";
 import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
-import { V4BetaIntroDialog } from "@/src/features/events/components/V4BetaIntroDialog";
+import { V4IntroDialog } from "@/src/features/events/components/V4IntroDialog";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { ZapIcon } from "lucide-react";
 
@@ -15,17 +15,23 @@ const PREVIEW_FAST_DESCRIPTION =
   "Get a more performant Langfuse experience. Upgrade SDKs to the latest major for real-time data. This is a personal setting.";
 const PREVIEW_FAST_DESCRIPTION_ID = "preview-fast-toggle-description";
 
-export function V4BetaSidebarToggle() {
+export function V4SidebarToggle() {
   const {
     isBetaEnabled,
+    canToggleV4,
     setBetaEnabled,
     enableWithIntro,
     showIntroDialog,
     confirmIntroDialog,
+    dismissIntroDialog,
     isLoading,
   } = useV4Beta();
   const capture = usePostHogClientCapture();
 
+  if (!canToggleV4) {
+    return null;
+  }
+
   const handleToggle = (enabled: boolean) => {
     if (enabled) {
       enableWithIntro({
@@ -82,9 +88,10 @@ export function V4BetaSidebarToggle() {
           </span>
         </div>
       </SidebarMenuButton>
-      <V4BetaIntroDialog
+      <V4IntroDialog
         open={showIntroDialog}
         onConfirm={confirmIntroDialog}
+        onDismiss={dismissIntroDialog}
       />
     </>
   );

web/src/features/events/config/filter-config.ts

@@ -1,5 +1,8 @@
 import { eventsTableCols } from "@langfuse/shared";
-import type { FilterConfig } from "@/src/features/filters/lib/filter-config";
+import {
+  omitFilterFacets,
+  type FilterConfig,
+} from "@/src/features/filters/lib/filter-config";
 import type { ColumnToBackendKeyMap } from "@/src/features/filters/lib/filter-transform";
 import { renderFilterIcon } from "@/src/components/ItemBadge";
 
@@ -9,7 +12,7 @@ export const getEventsColumnName = (id: string): string => {
   if (!column) {
     throw new Error(`Column ${id} not found in eventsTableCols`);
   }
-  return column?.name;
+  return column.name;
 };
 
 /**
@@ -20,6 +23,8 @@ export const OBSERVATION_EVENTS_COLUMN_TO_BACKEND_KEY: ColumnToBackendKeyMap = {
   // No mapping needed currently - events table column names align with UI
 };
 
+export type ObservationEventsOmittableFilterColumn = "sessionId" | "userId";
+
 export const observationEventsFilterConfig: FilterConfig = {
   tableName: "observations-events",
 
@@ -246,3 +251,9 @@ export const observationEventsFilterConfig: FilterConfig = {
     },
   ],
 };
+
+export function getObservationEventsFilterConfig(
+  omittedFilter: ObservationEventsOmittableFilterColumn[] = [],
+): FilterConfig {
+  return omitFilterFacets(observationEventsFilterConfig, omittedFilter);
+}

web/src/features/events/hooks/useV4Beta.ts

@@ -4,24 +4,30 @@ import { useCallback, useState } from "react";
 import posthog from "posthog-js";
 import { V4_BETA_ENABLED_POSTHOG_PROPERTY } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
-type SetV4BetaEnabledOptions = {
+type SetV4EnabledOptions = {
   onSuccess?: () => void | Promise<void>;
 };
 
 const INTRO_DIALOG_SEEN_KEY = "v4-beta-intro-dialog-seen";
 
 export function useV4Beta() {
-  const { data: session, update: updateSession } = useSession();
+  const {
+    data: session,
+    update: updateSession,
+    status: sessionStatus,
+  } = useSession();
 
   const mutation = api.userAccount.setV4BetaEnabled.useMutation();
 
   const isBetaEnabled = session?.user?.v4BetaEnabled ?? false;
+  const canToggleV4 = session?.user?.canToggleV4 === true;
+  const isInitializing = sessionStatus === "loading";
   const [showIntroDialog, setShowIntroDialog] = useState(false);
   const [pendingOnSuccess, setPendingOnSuccess] =
-    useState<SetV4BetaEnabledOptions["onSuccess"]>();
+    useState<SetV4EnabledOptions["onSuccess"]>();
 
   const setBetaEnabled = useCallback(
-    (enabled: boolean, options?: SetV4BetaEnabledOptions) => {
+    (enabled: boolean, options?: SetV4EnabledOptions) => {
       mutation.mutate(
         { enabled },
         {
@@ -42,7 +48,7 @@ export function useV4Beta() {
   );
 
   const enableWithIntro = useCallback(
-    (options?: SetV4BetaEnabledOptions) => {
+    (options?: SetV4EnabledOptions) => {
       if (
         typeof window !== "undefined" &&
         !localStorage.getItem(INTRO_DIALOG_SEEN_KEY)
@@ -71,6 +77,8 @@ export function useV4Beta() {
 
   return {
     isBetaEnabled,
+    canToggleV4,
+    isInitializing,
     setBetaEnabled,
     enableWithIntro,
     showIntroDialog,

web/src/features/events/lib/eventsToTraceAdapter.ts

@@ -1,4 +1,5 @@
-import { type EventsObservation, type TraceDomain } from "@langfuse/shared";
+import { type TraceDomain } from "@langfuse/shared";
+import { type FullEventsObservations } from "@langfuse/shared/src/server";
 import { type ObservationReturnTypeWithMetadata } from "@/src/server/api/routers/traces";
 import { type WithStringifiedMetadata } from "@/src/utils/clientSideDomainTypes";
 
@@ -24,7 +25,7 @@ export interface AdaptedTraceData {
  * The root observation (no parentObservationId) provides trace-level properties like name.
  */
 export function adaptEventsToTraceFormat(params: {
-  events: EventsObservation[];
+  events: FullEventsObservations;
   traceId: string;
   rootIO?: { input: unknown; output: unknown; metadata?: unknown } | null;
 }): AdaptedTraceData {
@@ -43,6 +44,22 @@ export function adaptEventsToTraceFormat(params: {
   // TODO: think, how to determine root span?
   const root = events.find((e) => !e.parentObservationId);
 
+  const latestTaggedEvent = events.reduce<
+    FullEventsObservations[number] | null
+  >((latest, event) => {
+    if (event.traceTags.length === 0) return latest;
+    if (!latest) return event;
+
+    if (event.updatedAt.getTime() > latest.updatedAt.getTime()) return event;
+    if (event.updatedAt.getTime() < latest.updatedAt.getTime()) return latest;
+
+    return event.createdAt.getTime() > latest.createdAt.getTime()
+      ? event
+      : latest;
+  }, null);
+
+  const traceTags = latestTaggedEvent?.traceTags;
+
   const endTimes = events
     .map((e) => e.endTime)
     .filter((t): t is Date => t !== null);
@@ -61,7 +78,7 @@ export function adaptEventsToTraceFormat(params: {
     input: rootIO?.input ? JSON.stringify(rootIO.input) : null,
     output: rootIO?.output ? JSON.stringify(rootIO.output) : null,
     metadata: JSON.stringify(rootIO?.metadata ?? root?.metadata ?? {}),
-    tags: [], // Events have tags on each observation, not trace-level
+    tags: traceTags ?? [],
     bookmarked: root?.bookmarked ?? false,
     public: root?.public ?? false,
     release: earliest.version ?? null,