chore: release v3.168.0

fix(web): Hide irrelevant filters in subtables (#13136 )
fix(evals): return full JSONPath slice result and deduplicate eval JSONPath logic (#13200 )
2026-04-17 10:36:00 +02:00 · 2026-04-17 08:26:56 +00:00 · 2026-04-17 07:49:55 +00:00 · 2026-04-16 16:53:44 +00:00 · 2026-04-16 15:47:13 +00:00 · 2026-04-16 15:09:44 +00:00
195 changed files with 8979 additions and 4111 deletions
@@ -40,6 +40,9 @@ evaluating, and debugging AI applications.
  [`skills/clickhouse-best-practices/SKILL.md`](skills/clickhouse-best-practices/SKILL.md)
 - Monorepo/Turbo task graph changes:
  [`skills/turborepo/SKILL.md`](skills/turborepo/SKILL.md)
+- pnpm dependency upgrades, package-version bumps, or `minimumReleaseAgeExclude`
+  decisions in `pnpm-workspace.yaml`:
+  [`skills/pnpm-upgrade-package/SKILL.md`](skills/pnpm-upgrade-package/SKILL.md)
 - User-visible frontend changes, Playwright review, or browser signoff:
  [`skills/frontend-browser-review/SKILL.md`](skills/frontend-browser-review/SKILL.md)
 - Web UI and frontend entry points:
@@ -81,6 +81,17 @@ Use for:

 Open: [changelog-writing/SKILL.md](changelog-writing/SKILL.md)

+### pnpm-upgrade-package
+
+Use for:
+- pnpm dependency bumps that need a specific target version
+- interactive upgrades where the package name or version may be missing
+- checking whether `pnpm-workspace.yaml` `minimumReleaseAgeExclude` must change
+- comparing registry latest with the latest version installable under the
+  current release-age gate
+
+Open: [pnpm-upgrade-package/SKILL.md](pnpm-upgrade-package/SKILL.md)
+
 ## Adding a New Shared Skill

 1. Codex may create or refine shared skills under `.agents/skills/` when a
@@ -0,0 +1,72 @@
+# PNPM Upgrade Package
+
+Use this workflow when a user wants to upgrade a dependency in the Langfuse
+pnpm workspace.
+
+## Workflow
+
+1. Collect missing inputs.
+   - Ask for the package if missing.
+   - Ask for the target version if missing.
+   - If the user says `latest`, resolve the real registry latest first.
+
+2. Run the main helper once.
+   - Run
+     `node .agents/skills/pnpm-upgrade-package/scripts/check-release-age-window.mjs <package> <targetVersion>`.
+   - Treat this as the single source of truth for:
+     - direct workspace references
+     - root `pnpm.overrides` / `pnpm.patchedDependencies`
+     - latest registry version
+     - latest version installable under the current release-age rules
+     - existing matching `minimumReleaseAgeExclude` entries
+     - exact dependency companions from `dependencies` and `optionalDependencies`
+     - exact peer dependencies that are actually installed in the workspace
+
+3. Handle the transitive-only case before editing anything.
+   - If the helper shows no direct workspace references, run `pnpm why -r <package>`.
+   - Identify the current top-level parent that pulls the package in.
+   - Check whether that parent's current dependency range already permits the
+     requested transitive version.
+   - If the current parent range already covers the requested version, prefer a
+     lock refresh / reinstall path before changing `package.json`.
+   - If the current parent range does not cover the requested version, upgrade
+     the direct parent dependency that pulls the package in.
+   - If a compatible transitive package still stays pinned after the normal
+     refresh path, you may suggest `pnpm dedupe` to the user as an optional
+     manual follow-up, but do not run it automatically and do not require it.
+   - Do not add the transitive package directly unless the user explicitly asks.
+
+4. Ask before changing `minimumReleaseAgeExclude`.
+   - Prefer `package@version` entries.
+   - Only use bare `package` entries after explicit approval.
+   - Ask about exact companion packages only when the helper says they still
+     need a new exclusion.
+   - Treat range-based dependency or peer entries as manual review.
+
+5. Bump at the narrowest useful scope.
+   - `pnpm -w up <package>@<version>` for root-only changes.
+   - `pnpm --filter <workspace> up <package>@<version>` for one workspace.
+   - `pnpm -r up <package>@<version>` only when every current reference should move.
+   - Do not hand-edit `pnpm-lock.yaml`.
+
+6. Validate.
+   - Use the nearest package `AGENTS.md` plus the root verification matrix.
+   - Finish with `pnpm why -r <package>`.
+   - If companions moved too, run `pnpm why -r <companion-package>` for them as well.
+
+## Quick Commands
+
+- Run the single analysis pass:
+  `node .agents/skills/pnpm-upgrade-package/scripts/check-release-age-window.mjs <package> <targetVersion>`
+- Transitive provenance check:
+  `pnpm why -r <package>`
+- Inspect the current parent manifest on the registry:
+  `npm view <parent>@<installedVersion> dependencies peerDependencies optionalDependencies --json`
+- Final graph verification:
+  `pnpm why -r <package>`
+- Bump in the root workspace:
+  `pnpm -w up <package>@<version>`
+- Bump in one workspace:
+  `pnpm --filter web up <package>@<version>`
+- Bump everywhere that should move together:
+  `pnpm -r up <package>@<version>`
@@ -0,0 +1,42 @@
+---
+name: pnpm-upgrade-package
+description: Use when upgrading a dependency in this pnpm workspace, including requests to bump a package to a specific version, compare the registry latest version with the latest version installable under the current minimum-release-age window, or decide whether minimumReleaseAgeExclude in pnpm-workspace.yaml must change. Ask the user for the package name or target version when either is missing.
+---
+
+# PNPM Upgrade Package
+
+Use this skill for interactive dependency bumps in Langfuse.
+
+## Read Order
+
+- Start with [AGENTS.md](AGENTS.md) for the end-to-end workflow.
+- Run the main helper once at the start of the upgrade:
+  `node .agents/skills/pnpm-upgrade-package/scripts/check-release-age-window.mjs <package> [targetVersion]`
+
+## Apply This Skill
+
+- Ask for the package name if the user did not provide one.
+- Ask for the target version if the user did not provide one.
+- Run the main helper once as the first analysis step and use that single
+  output for scope, exclusion decisions, and the final bump.
+- If the target package is not directly declared anywhere, run
+  `pnpm why -r <package>` to find which direct dependency brings it in, then
+  inspect whether the current top-level parent already allows the requested
+  transitive version via its dependency range.
+- If the current parent range already covers the requested transitive version,
+  prefer a lockfile refresh / reinstall path over bumping the parent manifest.
+- If the current parent range does not cover the requested transitive version,
+  upgrade that parent dependency instead of adding the target package directly
+  unless the user explicitly wants that.
+- If a compatible transitive package still stays pinned after the normal
+  refresh path, you may suggest `pnpm dedupe` to the user as an optional manual
+  follow-up, but do not run it automatically and do not require it.
+- Resolve the registry latest version, but do not silently upgrade to latest
+  unless the user asked for latest.
+- Compare the target version with the latest version installable under the
+  current `minimumReleaseAge` window.
+- Ask before adding `minimumReleaseAgeExclude` entries for the target package,
+  exact dependency companions from `dependencies` or `optionalDependencies`, or
+  locally installed exact peer dependencies.
+- Finish with `pnpm why -r <package>` to confirm that only the intended version
+  remains in the workspace.
@@ -0,0 +1,4 @@
+interface:
+  display_name: "PNPM Upgrade Package"
+  short_description: "Interactive pnpm package bump workflow"
+  default_prompt: "Use $pnpm-upgrade-package to upgrade a package in this pnpm workspace, asking me for the package or version if I did not provide them."
@@ -0,0 +1,418 @@
+#!/usr/bin/env node
+
+import { join } from "node:path";
+import {
+  entryCoversVersion,
+  findLocalPackageReferences,
+  formatWorkspaceReference,
+  getRootPnpmControls,
+  readWorkspaceConfig,
+} from "./lib/workspace-utils.mjs";
+
+const args = process.argv.slice(2);
+const asJson = args.includes("--json");
+const positional = args.filter((arg) => !arg.startsWith("--"));
+const packageName = positional[0];
+const requestedTargetVersion = positional[1] ?? null;
+
+if (!packageName) {
+  console.error(
+    "Usage: node .agents/skills/pnpm-upgrade-package/scripts/check-release-age-window.mjs <package> [targetVersion] [--json]",
+  );
+  process.exit(1);
+}
+
+const repoRoot = process.cwd();
+const workspaceConfig = readWorkspaceConfig(join(repoRoot, "pnpm-workspace.yaml"));
+const minimumReleaseAgeMinutes = workspaceConfig.minimumReleaseAge ?? 0;
+const thresholdMs = Date.now() - minimumReleaseAgeMinutes * 60 * 1000;
+const REGISTRY_FETCH_TIMEOUT_MS = 30_000;
+const registryCache = new Map();
+const workspaceReferenceCache = new Map();
+
+const getWorkspaceReferences = (name) => {
+  if (!workspaceReferenceCache.has(name)) {
+    workspaceReferenceCache.set(name, findLocalPackageReferences(repoRoot, name));
+  }
+
+  return workspaceReferenceCache.get(name);
+};
+
+function printSectionHeader(title) {
+  console.log("");
+  console.log(title);
+}
+
+function isPrerelease(version) {
+  return version.includes("-");
+}
+
+function isExactVersion(spec) {
+  return /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(spec.trim());
+}
+
+function getMatchingExcludeEntries(name, version) {
+  return workspaceConfig.minimumReleaseAgeExclude.filter((entry) =>
+    entryCoversVersion(entry, name, version),
+  );
+}
+
+async function fetchRegistryPackage(name) {
+  if (registryCache.has(name)) return registryCache.get(name);
+
+  const abortController = new AbortController();
+  const timeoutId = setTimeout(
+    () => abortController.abort(),
+    REGISTRY_FETCH_TIMEOUT_MS,
+  );
+  timeoutId.unref?.();
+
+  try {
+    const response = await fetch(
+      `https://registry.npmjs.org/${encodeURIComponent(name)}`,
+      {
+        headers: {
+          accept: "application/json",
+          "user-agent": "langfuse-pnpm-upgrade-package-skill",
+        },
+        signal: abortController.signal,
+      },
+    );
+
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch ${name} from npm registry: ${response.status}`,
+      );
+    }
+
+    const metadata = await response.json();
+    registryCache.set(name, metadata);
+    return metadata;
+  } catch (error) {
+    if (error?.name === "AbortError") {
+      throw new Error(
+        `Timed out fetching ${name} from npm registry after ${REGISTRY_FETCH_TIMEOUT_MS}ms`,
+        { cause: error },
+      );
+    }
+
+    throw error;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+function getInstallability(metadata, name, version) {
+  const publishedAt = metadata.time?.[version] ?? null;
+  const publishedAtMs = publishedAt ? Date.parse(publishedAt) : null;
+  const matchingExcludeEntries = getMatchingExcludeEntries(name, version);
+  const isYoungerThanMinimumReleaseAge =
+    publishedAtMs != null ? publishedAtMs > thresholdMs : null;
+  const isInstallableWithoutNewExclude =
+    isYoungerThanMinimumReleaseAge == null
+      ? null
+      : !isYoungerThanMinimumReleaseAge || matchingExcludeEntries.length > 0;
+
+  return {
+    name,
+    version,
+    publishedAt,
+    isYoungerThanMinimumReleaseAge,
+    isInstallableWithoutNewExclude,
+    matchingExcludeEntries,
+    suggestedExclude:
+      isInstallableWithoutNewExclude === false ? `${name}@${version}` : null,
+  };
+}
+
+function selectLatestInstallableVersion(metadata, name) {
+  const times = metadata.time ?? {};
+
+  return (
+    Object.keys(metadata.versions ?? {})
+      .filter((version) => times[version] && !isPrerelease(version))
+      .sort((left, right) => Date.parse(times[right]) - Date.parse(times[left]))
+      .map((version) => getInstallability(metadata, name, version))
+      .find((candidate) => candidate.isInstallableWithoutNewExclude) ?? null
+  );
+}
+
+function collectManifestEntries(manifest, fields) {
+  const merged = new Map();
+
+  for (const field of fields) {
+    for (const [name, spec] of Object.entries(manifest[field] ?? {})) {
+      const key = `${name}:${spec}`;
+      const entry = merged.get(key);
+
+      if (entry) {
+        entry.fields.push(field);
+        continue;
+      }
+
+      merged.set(key, { name, spec, fields: [field] });
+    }
+  }
+
+  return [...merged.values()].sort((left, right) =>
+    left.name.localeCompare(right.name),
+  );
+}
+
+async function analyzeManifestEntries(entries, { includeWorkspace = false } = {}) {
+  const exact = [];
+  const range = [];
+
+  for (const entry of entries) {
+    const workspaceReferences = includeWorkspace
+      ? getWorkspaceReferences(entry.name)
+      : null;
+
+    if (!isExactVersion(entry.spec)) {
+      range.push({
+        ...entry,
+        ...(includeWorkspace ? { workspaceReferences } : {}),
+      });
+      continue;
+    }
+
+    const metadata = await fetchRegistryPackage(entry.name);
+    const installability = getInstallability(metadata, entry.name, entry.spec);
+
+    exact.push({
+      ...entry,
+      ...installability,
+      ...(includeWorkspace
+        ? {
+            workspaceReferences,
+            isInstalledInWorkspace: workspaceReferences.length > 0,
+          }
+        : {}),
+      suggestedExclude:
+        includeWorkspace && workspaceReferences.length === 0
+          ? null
+          : installability.suggestedExclude,
+    });
+  }
+
+  return { exact, range };
+}
+
+function printWorkspaceReferences(title, references) {
+  printSectionHeader(title);
+  if (references.length === 0) {
+    console.log("- none");
+    return;
+  }
+
+  for (const reference of references) {
+    console.log(`- ${formatWorkspaceReference(reference)}`);
+  }
+}
+
+function printRootPnpmControls(rootPnpm) {
+  printSectionHeader("Root pnpm controls:");
+  if (
+    rootPnpm.overrideMatches.length === 0 &&
+    rootPnpm.patchedDependencyMatches.length === 0
+  ) {
+    console.log("- none");
+    return;
+  }
+
+  for (const match of rootPnpm.overrideMatches) {
+    console.log(`- override ${match.selector}: ${match.value}`);
+  }
+  for (const match of rootPnpm.patchedDependencyMatches) {
+    console.log(`- patched dependency ${match.selector}: ${match.value}`);
+  }
+}
+
+function printVersionEntries(title, entries, { includeWorkspace = false } = {}) {
+  printSectionHeader(title);
+  if (entries.length === 0) {
+    console.log("- none");
+    return;
+  }
+
+  for (const entry of entries) {
+    const status =
+      entry.isInstallableWithoutNewExclude == null
+        ? "unknown"
+        : entry.isInstallableWithoutNewExclude
+          ? "installable now"
+          : "needs exclude";
+
+    console.log(
+      `- ${entry.name}@${entry.version} (${status}; via ${entry.fields.join(", ")})`,
+    );
+    if (entry.publishedAt) {
+      console.log(`  published at: ${entry.publishedAt}`);
+    }
+    if (includeWorkspace) {
+      console.log(
+        `  installed in workspace: ${entry.isInstalledInWorkspace ? "yes" : "no"}`,
+      );
+      for (const reference of entry.workspaceReferences) {
+        console.log(`  workspace reference: ${formatWorkspaceReference(reference)}`);
+      }
+    }
+    if (entry.matchingExcludeEntries.length > 0) {
+      console.log(
+        `  matching exclude entries: ${entry.matchingExcludeEntries.join(", ")}`,
+      );
+    }
+    if (entry.suggestedExclude) {
+      console.log(`  suggested exclude: ${entry.suggestedExclude}`);
+    }
+  }
+}
+
+function printRangeEntries(title, entries) {
+  printSectionHeader(title);
+  if (entries.length === 0) {
+    console.log("- none");
+    return;
+  }
+
+  for (const entry of entries) {
+    console.log(
+      `- ${entry.name}: ${entry.spec} (manual review; via ${entry.fields.join(", ")})`,
+    );
+    for (const reference of entry.workspaceReferences ?? []) {
+      console.log(`  workspace reference: ${formatWorkspaceReference(reference)}`);
+    }
+  }
+}
+
+const packageMetadata = await fetchRegistryPackage(packageName);
+const latestVersion = packageMetadata["dist-tags"]?.latest ?? null;
+const targetVersion = requestedTargetVersion ?? latestVersion;
+
+if (!targetVersion) {
+  console.error(`Could not resolve a target version for ${packageName}.`);
+  process.exit(1);
+}
+
+if (!packageMetadata.versions?.[targetVersion]) {
+  console.error(`Version ${targetVersion} was not found for ${packageName}.`);
+  process.exit(1);
+}
+
+const packageWorkspaceReferences = getWorkspaceReferences(packageName);
+const rootPnpm = getRootPnpmControls(repoRoot, packageName);
+const latestInstallableWithoutNewExclude = selectLatestInstallableVersion(
+  packageMetadata,
+  packageName,
+);
+const targetInstallability = getInstallability(
+  packageMetadata,
+  packageName,
+  targetVersion,
+);
+const targetManifest = packageMetadata.versions[targetVersion];
+
+const dependencyCompanions = await analyzeManifestEntries(
+  collectManifestEntries(targetManifest, [
+    "dependencies",
+    "optionalDependencies",
+  ]),
+);
+const peerDependencies = await analyzeManifestEntries(
+  collectManifestEntries(targetManifest, ["peerDependencies"]),
+  { includeWorkspace: true },
+);
+
+const result = {
+  packageName,
+  targetVersion,
+  targetWasExplicitlyProvided: requestedTargetVersion != null,
+  packageWorkspaceReferences,
+  rootPnpm,
+  minimumReleaseAgeMinutes,
+  thresholdIso: new Date(thresholdMs).toISOString(),
+  latestRegistryVersion: latestVersion,
+  latestRegistryPublishedAt:
+    latestVersion != null ? packageMetadata.time?.[latestVersion] ?? null : null,
+  latestInstallableWithoutNewExclude,
+  targetPublishedAt: targetInstallability.publishedAt,
+  targetIsYoungerThanMinimumReleaseAge:
+    targetInstallability.isYoungerThanMinimumReleaseAge,
+  targetIsInstallableWithoutNewExclude:
+    targetInstallability.isInstallableWithoutNewExclude,
+  matchingPackageExcludeEntries: targetInstallability.matchingExcludeEntries,
+  suggestedPackageExclude: targetInstallability.suggestedExclude,
+  exactDependencyCompanions: dependencyCompanions.exact,
+  rangeDependencyCompanions: dependencyCompanions.range,
+  exactPeerDependencies: peerDependencies.exact,
+  rangePeerDependencies: peerDependencies.range,
+};
+
+if (asJson) {
+  console.log(JSON.stringify(result, null, 2));
+  process.exit(0);
+}
+
+console.log(`Package: ${packageName}`);
+console.log(
+  `Target version: ${targetVersion}${
+    requestedTargetVersion
+      ? ""
+      : " (resolved latest; still ask before bumping if version was omitted)"
+  }`,
+);
+
+printWorkspaceReferences(
+  "Target package workspace references:",
+  packageWorkspaceReferences,
+);
+printRootPnpmControls(rootPnpm);
+
+printSectionHeader("Release-age window:");
+console.log(`minimumReleaseAge: ${minimumReleaseAgeMinutes} minutes`);
+console.log(`Threshold: ${result.thresholdIso}`);
+console.log(`Latest registry version: ${result.latestRegistryVersion ?? "unknown"}`);
+if (result.latestRegistryPublishedAt) {
+  console.log(`Latest registry published at: ${result.latestRegistryPublishedAt}`);
+}
+if (latestInstallableWithoutNewExclude) {
+  console.log(
+    `Latest installable without new exclude: ${latestInstallableWithoutNewExclude.version} (${latestInstallableWithoutNewExclude.publishedAt})`,
+  );
+} else {
+  console.log("Latest installable without new exclude: none found");
+}
+console.log(`Target published at: ${result.targetPublishedAt ?? "unknown"}`);
+console.log(
+  `Target installable without new exclude: ${
+    result.targetIsInstallableWithoutNewExclude == null
+      ? "unknown"
+      : result.targetIsInstallableWithoutNewExclude
+        ? "yes"
+        : "no"
+  }`,
+);
+if (result.matchingPackageExcludeEntries.length > 0) {
+  console.log("Matching package exclude entries:");
+  for (const entry of result.matchingPackageExcludeEntries) {
+    console.log(`- ${entry}`);
+  }
+} else {
+  console.log("Matching package exclude entries: none");
+}
+if (result.suggestedPackageExclude) {
+  console.log(`Suggested package exclude: ${result.suggestedPackageExclude}`);
+}
+
+printVersionEntries(
+  "Exact dependency companions (dependencies + optionalDependencies):",
+  result.exactDependencyCompanions,
+);
+printRangeEntries(
+  "Range dependency companions (dependencies + optionalDependencies):",
+  result.rangeDependencyCompanions,
+);
+printVersionEntries("Exact peer dependencies:", result.exactPeerDependencies, {
+  includeWorkspace: true,
+});
+printRangeEntries("Range peer dependencies:", result.rangePeerDependencies);
@@ -0,0 +1,208 @@
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { join, relative } from "node:path";
+
+const packageFields = [
+  "dependencies",
+  "devDependencies",
+  "peerDependencies",
+  "optionalDependencies",
+];
+
+export function formatWorkspaceReference(reference) {
+  const label = reference.workspaceName
+    ? `${reference.path} (${reference.workspaceName})`
+    : reference.path;
+  const specs = reference.matches
+    .map((match) => `${match.field}: ${match.spec}`)
+    .join(", ");
+
+  return `${label} -> ${specs}`;
+}
+
+export function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+function stripInlineComment(line) {
+  let quote = null;
+  let escaped = false;
+
+  for (let index = 0; index < line.length; index += 1) {
+    const char = line[index];
+
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+
+    if (quote) {
+      if (char === "\\") {
+        escaped = true;
+        continue;
+      }
+
+      if (char === quote) {
+        quote = null;
+      }
+      continue;
+    }
+
+    if (char === "'" || char === '"') {
+      quote = char;
+      continue;
+    }
+
+    if (char === "#") {
+      return line.slice(0, index).trimEnd();
+    }
+  }
+
+  return line;
+}
+
+export function readWorkspaceConfig(path) {
+  const raw = readFileSync(path, "utf8");
+  const lines = raw.split(/\r?\n/);
+  let minimumReleaseAge = 0;
+  const minimumReleaseAgeExclude = [];
+  let inExcludeBlock = false;
+
+  for (const line of lines) {
+    const uncommented = stripInlineComment(line);
+    const trimmed = uncommented.trim();
+
+    if (!trimmed || trimmed.startsWith("#")) continue;
+
+    const ageMatch = trimmed.match(/^minimumReleaseAge:\s*(\d+)\s*$/);
+    if (ageMatch) {
+      minimumReleaseAge = Number(ageMatch[1]);
+      continue;
+    }
+
+    if (/^minimumReleaseAgeExclude:\s*$/.test(trimmed)) {
+      inExcludeBlock = true;
+      continue;
+    }
+
+    if (inExcludeBlock) {
+      const excludeMatch = uncommented.match(/^\s*-\s+(.+?)\s*$/);
+      if (excludeMatch) {
+        minimumReleaseAgeExclude.push(
+          excludeMatch[1].replace(/^['"]|['"]$/g, ""),
+        );
+        continue;
+      }
+
+      if (/^\S/.test(uncommented)) {
+        inExcludeBlock = false;
+      }
+    }
+  }
+
+  return { minimumReleaseAge, minimumReleaseAgeExclude };
+}
+
+export function collectPackageJsonPaths(repoRoot) {
+  const paths = [
+    "package.json",
+    "web/package.json",
+    "worker/package.json",
+    "ee/package.json",
+  ];
+  const packagesRoot = join(repoRoot, "packages");
+
+  if (!existsSync(packagesRoot)) return paths;
+
+  const stack = [packagesRoot];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    for (const entry of readdirSync(current, { withFileTypes: true })) {
+      if (
+        entry.name === "node_modules" ||
+        entry.name === "dist" ||
+        entry.name === ".git"
+      ) {
+        continue;
+      }
+
+      const nextPath = join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(nextPath);
+        continue;
+      }
+
+      if (entry.isFile() && entry.name === "package.json") {
+        paths.push(relative(repoRoot, nextPath));
+      }
+    }
+  }
+
+  return [...new Set(paths)];
+}
+
+export function matchesPackageSelector(selector, wantedPackage) {
+  if (selector === wantedPackage) return true;
+  if (selector.startsWith(`${wantedPackage}@`)) return true;
+  if (selector.endsWith(`>${wantedPackage}`)) return true;
+  if (selector.includes(`>${wantedPackage}@`)) return true;
+  if (selector.endsWith("/*")) {
+    const prefix = selector.slice(0, -1);
+    return wantedPackage.startsWith(prefix);
+  }
+  return false;
+}
+
+export function entryCoversVersion(entry, wantedPackage, wantedVersion) {
+  if (entry === wantedPackage) return true;
+  if (entry.endsWith("/*")) {
+    const prefix = entry.slice(0, -1);
+    return wantedPackage.startsWith(prefix);
+  }
+  if (!entry.startsWith(`${wantedPackage}@`)) return false;
+
+  return entry
+    .slice(wantedPackage.length + 1)
+    .split("||")
+    .map((part) => part.trim())
+    .includes(wantedVersion);
+}
+
+export function findLocalPackageReferences(repoRoot, wantedPackage) {
+  const results = [];
+
+  for (const packageJsonPath of collectPackageJsonPaths(repoRoot)) {
+    const json = readJson(join(repoRoot, packageJsonPath));
+    const matches = [];
+
+    for (const field of packageFields) {
+      if (json[field]?.[wantedPackage]) {
+        matches.push({ field, spec: json[field][wantedPackage] });
+      }
+    }
+
+    if (matches.length > 0) {
+      results.push({
+        path: packageJsonPath,
+        workspaceName: json.name ?? null,
+        matches,
+      });
+    }
+  }
+
+  return results;
+}
+
+export function getRootPnpmControls(repoRoot, packageName) {
+  const rootPackageJson = readJson(join(repoRoot, "package.json"));
+
+  return {
+    overrideMatches: Object.entries(rootPackageJson.pnpm?.overrides ?? {})
+      .filter(([selector]) => matchesPackageSelector(selector, packageName))
+      .map(([selector, value]) => ({ selector, value })),
+    patchedDependencyMatches: Object.entries(
+      rootPackageJson.pnpm?.patchedDependencies ?? {},
+    )
+      .filter(([selector]) => matchesPackageSelector(selector, packageName))
+      .map(([selector, value]) => ({ selector, value })),
+  };
+}
@@ -0,0 +1,196 @@
+#####################################################################
+# .env (template) — OCI Object Storage / S3-compatible configuration
+#
+# IMPORTANT SECURITY NOTES (Oracle best practice)
+# - Prefer OCI-native auth (Instance Principal / Workload Identity / Resource Principal)
+#   over static keys.
+# - If you must use static keys, store them in a secure secret manager
+#   (e.g., Kubernetes Secret / OCI Vault) and inject at runtime.
+# - Rotate/revoke any credentials that were previously shared or committed.
+#####################################################################
+
+
+#####################################################################
+# 1) Storage/Auth category (CHOOSE ONE)
+#
+# The app can read/write/download to/from an OCI object store for:
+# - Batch exports (exports/)
+# - Media uploads (media/)
+# - Event uploads (events/)
+#
+# Pick exactly ONE auth mechanism for OCI-native object storage by setting:
+#   LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+#   LANGFUSE_OCI_AUTH_TYPE=<one of the values below>
+#
+# Supported values:
+#   workload_identity | instance_principal | resource_principal | oci_profile | session_token
+#####################################################################
+
+
+#####################################################################
+# Category A — OCI Object Storage with INSTANCE PRINCIPAL (recommended on OCI Compute)
+# Use when:
+# - Running on OCI Compute with IAM set up (dynamic group + policies)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=instance_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category B — OCI Object Storage with WORKLOAD IDENTITY (common on OKE)
+# Use when:
+# - Running on OKE with OCI Workload Identity configured
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=workload_identity
+#
+# Optional (only if your environment requires additional CA trust):
+# NODE_EXTRA_CA_CERTS=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category C — OCI Object Storage with RESOURCE PRINCIPAL (common for OCI services)
+# Use when:
+# - Running inside an OCI service/runtime that injects Resource Principal env vars
+#   (e.g., certain managed services / automation contexts)
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=resource_principal
+#
+# NOTE: Do NOT set *_ACCESS_KEY_ID / *_SECRET_ACCESS_KEY in this category.
+#####################################################################
+
+
+#####################################################################
+# Category D — OCI Object Storage with OCI CONFIG PROFILE (developer local)
+# Use when:
+# - You have an OCI config file locally or mounted in the runtime
+# - You want to use a named profile
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=oci_profile
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#
+# NOTE: Avoid adding config files into images; mount/inject securely.
+#####################################################################
+
+
+#####################################################################
+# Category E — OCI Object Storage with SESSION TOKEN (short-lived user auth)
+# Use when:
+# - You use OCI CLI session authentication (short-lived token flow)
+# - USE oci session authenticate
+# - Appropriate for interactive/dev use; less common for long-running services
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=true
+# LANGFUSE_OCI_AUTH_TYPE=session_token
+# OCI_CONFIG_FILE=/path/to/oci/config
+# OCI_CONFIG_PROFILE=DEFAULT
+#####################################################################
+
+
+#####################################################################
+# Other possible setup — Non-OCI provider (AWS S3 / GCP / Azure / MinIO / etc.)
+# Use when:
+# - Your object storage is NOT OCI Object Storage
+#
+# Set:
+# LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE=false
+#
+# Then configure endpoints/regions/credentials for your provider.
+#####################################################################
+
+
+
+#####################################################################
+# 2) Feature: S3 Batch Export
+#
+# Required (when enabled):
+# - *_BUCKET, *_REGION, *_ENDPOINT, *_PREFIX
+# Optional:
+# - *_EXTERNAL_ENDPOINT
+# - *_FORCE_PATH_STYLE=true (needed for many S3-compatible providers like MinIO)
+#
+# Credentials:
+# - Set *_ACCESS_KEY_ID/_SECRET_ACCESS_KEY ONLY for static-key auth
+#   (non-OCI S3-compatible providers)
+#####################################################################
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse-bucket
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# OCI example region/endpoint:
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-chicago-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+
+# MinIO / S3-compat setting (safe to keep true for many S3-compatible endpoints)
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 3) Feature: S3 Media Upload
+#####################################################################
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 4) Feature: S3 Event Upload (optional)
+#####################################################################
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse-bucket
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-chicago-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=https://objectstorage.us-chicago-1.oraclecloud.com
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+
+# Static-key auth (non-OCI). Leave blank/commented for OCI-native auth types above.
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=__REPLACE_ME__
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=__REPLACE_ME__
+
+
+
+#####################################################################
+# 5) OCI native auth configuration (used by oci_profile / session_token)
+#####################################################################
+# Only required when LANGFUSE_OCI_AUTH_TYPE is: oci_profile OR session_token
+OCI_CONFIG_FILE=__REPLACE_ME__/config
+OCI_CONFIG_PROFILE=DEFAULT
+
+
+
+#####################################################################
+# 6) Troubleshooting notes (comments only)
+#
+# - If you see TLS errors to the endpoint in Kubernetes/OKE, set NODE_EXTRA_CA_CERTS to the
+#   correct CA bundle path for your environment.
+# - If using MinIO or certain S3-compatible providers and you get bucket addressing errors,
+#   set *_FORCE_PATH_STYLE=true.
+# - If downloads work inside the cluster but not externally, configure
+#   LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT to a publicly reachable endpoint/DNS.
+#####################################################################
@@ -153,6 +153,7 @@ LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
 # Langfuse AI Bedrock credentials
 AWS_ACCESS_KEY_ID="A123456789"
 AWS_SECRET_ACCESS_KEY="SAK123456789"
+LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY="1234567890abcdef"
 LANGFUSE_AWS_BEDROCK_REGION="eu-west-1"
 LANGFUSE_AWS_BEDROCK_MODEL="eu.anthropic.claude-3-haiku-20240307-v1:0"

@@ -12,7 +12,7 @@ updates:
    schedule:
      interval: "daily"
    cooldown:
-      default-days: 5
+      default-days: 7
    versioning-strategy: "increase"
    commit-message:
      prefix: chore
@@ -54,6 +54,8 @@ updates:
    directory: "/"
    schedule:
      interval: "weekly"
+    cooldown:
+      default-days: 8
    commit-message:
      prefix: ci
      include: scope
@@ -10,10 +10,21 @@ on:
        type: string
        description: Name of the service to be deployed, e.g. web-ingestion, web, or worker.
        required: true
+    # Environment secrets don't auto-resolve in reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      SENTRY_AUTH_TOKEN:
+        required: false
 jobs:
  ecs-deploy:
    runs-on: blacksmith-4vcpu-ubuntu-2404
    environment: ${{ inputs.environment }}
+    permissions:
+      contents: read
    steps:
      - name: Get app name
        uses: winterjung/split@a211a1c46e35fcdc4097d59dd6282d4a9859651b # v2
@@ -22,41 +33,57 @@ jobs:
          msg: ${{ inputs.service }}
          separator: "-"
      - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Authenticate with AWS
        # GitHub/AWS recommend to use OIDC here: https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc
        # Probably more painful to configure, but would remove all long-lived credentials.
-        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
+        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.AWS_REGION }}
      - name: Login to AWS ECR
        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2
+        uses: aws-actions/amazon-ecr-login@f2e9fc6c2b355c1890b65e6f6f0e2ac3e6e22f78 # v2
      - name: Build, tag, and push Docker image
        env:
          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          REPOSITORY: ${{ steps.split.outputs._0 }}
          IMAGE_TAG: ${{ github.sha }}
+          STEPS_SPLIT_OUTPUTS__0: ${{ steps.split.outputs._0 }}
+          VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: ${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }}
+          VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }}
+          VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT: ${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }}
+          VARS_NEXT_PUBLIC_DEMO_ORG_ID: ${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }}
+          VARS_NEXT_PUBLIC_DEMO_PROJECT_ID: ${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }}
+          VARS_NEXT_PUBLIC_SENTRY_DSN: ${{ vars.NEXT_PUBLIC_SENTRY_DSN }}
+          VARS_NEXT_PUBLIC_POSTHOG_KEY: ${{ vars.NEXT_PUBLIC_POSTHOG_KEY }}
+          VARS_NEXT_PUBLIC_POSTHOG_HOST: ${{ vars.NEXT_PUBLIC_POSTHOG_HOST }}
+          VARS_NEXT_PUBLIC_PLAIN_APP_ID: ${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }}
+          VARS_SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          VARS_SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}
+          VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE: ${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }}
+          SECRETS_SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
        run: |
          docker build \
            -t $REGISTRY/$REPOSITORY:$IMAGE_TAG \
-            -f ./${{ steps.split.outputs._0 }}/Dockerfile \
-            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${{ vars.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION }} \
-            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_LANGFUSE_TRACING_SAMPLE_RATE }} \
-            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }} \
-            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${{ vars.NEXT_PUBLIC_DEMO_ORG_ID }} \
-            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${{ vars.NEXT_PUBLIC_DEMO_PROJECT_ID }} \
-            --build-arg NEXT_PUBLIC_SENTRY_DSN=${{ vars.NEXT_PUBLIC_SENTRY_DSN }} \
-            --build-arg NEXT_PUBLIC_BUILD_ID=${{ github.sha }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ vars.NEXT_PUBLIC_POSTHOG_KEY }} \
-            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${{ vars.NEXT_PUBLIC_POSTHOG_HOST }} \
-            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${{ vars.NEXT_PUBLIC_PLAIN_APP_ID }} \
-            --build-arg SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
-            --build-arg SENTRY_ORG=${{ vars.SENTRY_ORG }} \
-            --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
-            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${{ vars.NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE }} \
+            -f ./${STEPS_SPLIT_OUTPUTS__0}/Dockerfile \
+            --build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION} \
+            --build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE} \
+            --build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT} \
+            --build-arg NEXT_PUBLIC_DEMO_ORG_ID=${VARS_NEXT_PUBLIC_DEMO_ORG_ID} \
+            --build-arg NEXT_PUBLIC_DEMO_PROJECT_ID=${VARS_NEXT_PUBLIC_DEMO_PROJECT_ID} \
+            --build-arg NEXT_PUBLIC_SENTRY_DSN=${VARS_NEXT_PUBLIC_SENTRY_DSN} \
+            --build-arg NEXT_PUBLIC_BUILD_ID=${IMAGE_TAG} \
+            --build-arg NEXT_PUBLIC_POSTHOG_KEY=${VARS_NEXT_PUBLIC_POSTHOG_KEY} \
+            --build-arg NEXT_PUBLIC_POSTHOG_HOST=${VARS_NEXT_PUBLIC_POSTHOG_HOST} \
+            --build-arg NEXT_PUBLIC_PLAIN_APP_ID=${VARS_NEXT_PUBLIC_PLAIN_APP_ID} \
+            --build-arg SENTRY_AUTH_TOKEN=${SECRETS_SENTRY_AUTH_TOKEN} \
+            --build-arg SENTRY_ORG=${VARS_SENTRY_ORG} \
+            --build-arg SENTRY_PROJECT=${VARS_SENTRY_PROJECT} \
+            --build-arg NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_PUBLIC_LANGFUSE_TRACING_SAMPLE_RATE} \
            .
          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
      - name: Render AWS ECS Task Definition
@@ -9,6 +9,8 @@ on:
  issue_comment:
    types: [created]

+permissions: {}
+
 jobs:
  retrigger_cla:
    # Only run on PR comments (not issue comments) with the /check-cla command
@@ -1,14 +1,15 @@
 name: Claude Review on Maintainer PRs

 on:
-  pull_request_target:
+  pull_request:
    types:
      - opened
      - ready_for_review

 jobs:
  comment:
-    if: github.event.pull_request.draft == false
+    # Only run on PRs that are not drafts and are from the same repository (i.e., not from forks)
+    if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    permissions:
      issues: write
@@ -16,7 +17,7 @@ jobs:
    steps:
      - name: Check author permission and existing review request
        id: check
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const owner = context.repo.owner;
@@ -57,7 +58,7 @@ jobs:

      - name: Add Claude review comment
        if: steps.check.outputs.should_comment == 'true'
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            await github.rest.issues.createComment({
@@ -55,11 +55,13 @@ jobs:
        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
    steps:
      - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
-        uses: github/codeql-action/init@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        with:
          languages: ${{ matrix.language }}
          build-mode: ${{ matrix.build-mode }}
@@ -87,6 +89,6 @@ jobs:
          exit 1

      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3
+        uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
        with:
          category: "/language:${{matrix.language}}"
@@ -22,6 +22,8 @@ jobs:

    steps:
      - name: Checkout
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Codespell
-        uses: codespell-project/actions-codespell@406322ec52dd7b488e48c1c4b82e2a8b3a1bf630 # v2
+        uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
@@ -6,6 +6,8 @@ on:
      - main
  workflow_dispatch:

+permissions: {}
+
 jobs:
  rebase-dependabot:
    runs-on: ubuntu-latest
@@ -27,6 +27,8 @@ on:
          - prod-jp
        required: true

+permissions: {}
+
 concurrency:
  # Support concurrent `push` and `workflow_dispatch`` actions
  group: deploy-${{ github.event_name }}-${{ github.ref }}
@@ -41,7 +43,7 @@ jobs:
    steps:
      - name: Get affected services
        id: affected-services
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            if (context.eventName === "workflow_dispatch") {
@@ -56,7 +58,7 @@ jobs:
            return "[]"
          result-encoding: string
      - name: Print services to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          services: ${{ steps.affected-services.outputs.result }}
        with:
@@ -71,7 +73,7 @@ jobs:
    steps:
      - name: Get affected environments
        id: affected-environments
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            if (context.eventName === "workflow_dispatch") {
@@ -88,7 +90,7 @@ jobs:
            return "[]"
          result-encoding: string
      - name: Print environments to build
-        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          environments: ${{ steps.affected-environments.outputs.result }}
        with:
@@ -99,7 +101,14 @@ jobs:
  ecs-deploy:
    uses: ./.github/workflows/_deploy_ecs_service.yml
    needs: [affected-services, affected-environments]
-    secrets: inherit
+    permissions:
+      contents: read
+    # Environment secrets must be passed explicitly to reusable workflows.
+    # See: https://github.com/actions/runner/issues/3206
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
    strategy:
      matrix:
        service: ${{ fromJson(needs.affected-services.outputs.services) }}
@@ -9,15 +9,21 @@ on:
    branches:
      - "main"

+permissions:
+  contents: read
+  checks: write # Needed to create a check run for the license compliance check results
+
 jobs:
  license_check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      - name: Setup node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 18

@@ -32,7 +38,7 @@ jobs:

      - name: Check license-checker CSV file without headers
        id: license_check_report
-        uses: pilosus/action-pip-license-checker@cc7a461bfa27b44ad187b8578c881ef5138c13fd # v2
+        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
        with:
          external: "npm-license-checker.csv"
          external-format: "csv"
@@ -44,6 +50,8 @@ jobs:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Echo error
        if: failure()
-        run: echo "::error::${{ steps.license_check_report.outputs.report }}"
+        run: echo "::error::${STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT}"
+        env:
+          STEPS_LICENSE_CHECK_REPORT_OUTPUTS_REPORT: ${{ steps.license_check_report.outputs.report }}
      - name: Delete license-checker CSV file
        run: rm npm-license-checker.csv
@@ -12,6 +12,9 @@ on:
    branches:
      - "**"

+permissions:
+  contents: read
+
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -24,19 +27,39 @@ jobs:
      llm_connections_changed: ${{ steps.filter.outputs.llm_connections }}
    timeout-minutes: 15
    permissions:
+      contents: read
      pull-requests: read
    steps:
+      # Replaces fkirc/skip-duplicate-actions — skips runs whose git tree
+      # was already tested in a prior successful run of this workflow.
      - id: skip_check
-        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5
-        with:
-          do_not_skip: '["workflow_dispatch"]'
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          CURRENT_TREE=$(gh api "repos/${{ github.repository }}/git/commits/${{ github.sha }}" --jq '.tree.sha')
+
+          MATCH=$(gh api "repos/${{ github.repository }}/actions/workflows/pipeline.yml/runs?status=success&per_page=20" \
+            --jq "[.workflow_runs[] | select(.id != ${{ github.run_id }} and .head_commit.tree_id == \"$CURRENT_TREE\")] | first | .head_sha // empty")
+
+          if [[ -n "$MATCH" ]]; then
+            echo "::notice::Tree $CURRENT_TREE already tested in a prior successful run (commit $MATCH) — skipping"
+            echo "should_skip=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "should_skip=false" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # Recommended for paths-filter action
          # may save additional git fetch roundtrip if
          # merge-base is found within latest N commits
          fetch-depth: 20
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
+          persist-credentials: false
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        id: filter
        with:
          filters: |
@@ -50,17 +73,19 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] lint job dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: 24
          cache: "pnpm"
          cache-dependency-path: "pnpm-lock.yaml"
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] lint cache only; publish jobs rebuild artifacts and do not restore this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-lint-${{ github.sha }}
@@ -82,13 +107,14 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] prettier check dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: 24
          cache: "pnpm"
@@ -129,10 +155,12 @@ jobs:
      DOCKER_COMPOSE_FILE: docker-compose.build.yml
    steps:
      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
@@ -178,7 +206,7 @@ jobs:
          done
      - name: Upload docker diagnostics
        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-docker-build-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
          path: /tmp/docker-diagnostics
@@ -197,23 +225,25 @@ jobs:
        deploy-mode: ["", "-azure", "-redis-cluster"]
        shard: [1, 2, 3]
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Install golang-migrate for Clickhouse migrations
        run: |
          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz | tar xvz
          sudo mv migrate /usr/bin/migrate
          which migrate
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] test job dependency cache only; no released artifacts are built or published from this cached state
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -232,14 +262,14 @@ jobs:
          echo "ADMIN_API_KEY=admin-api-key" >> .env
          echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test" >> .env
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test job cache only; publish jobs rebuild artifacts and do not restore this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-
      - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] test-only Next.js cache; not consumed by artifact publishing or release jobs
        with:
          path: |
            ~/.npm
@@ -310,18 +340,20 @@ jobs:
        postgres-version: [12, 15]
        deploy-mode: ["", "-azure", "-redis-cluster"]
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] worker test dependency cache only; no release artifacts are produced from this cache
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -381,18 +413,20 @@ jobs:
    if: startsWith(github.ref, 'refs/tags/') || (needs.pre-job.outputs.should_skip != 'true' && (needs.pre-job.outputs.llm_connections_changed == 'true' || github.event_name == 'workflow_dispatch'))
    name: test-worker-llm-connections (node24, pg15)
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Use Node.js 24
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] llm-connection test dependency cache only; privileged secrets are used here, but this cache is not consumed by artifact publishing
        with:
          node-version: 24
          cache: "pnpm"
@@ -442,6 +476,7 @@ jobs:
          LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID }}
          LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY }}
          LANGFUSE_LLM_CONNECTION_BEDROCK_REGION: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY }}
          LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY }}
          LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY }}

@@ -451,23 +486,25 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] e2e dependency cache only; release images are rebuilt later without restoring this cache
        with:
          node-version: 24
          cache: "pnpm"
          cache-dependency-path: "pnpm-lock.yaml"
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
      - name: Setup Turbo cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e cache only; no published artifact path restores this cache
        with:
          path: .turbo
          key: ${{ runner.os }}-turbo-e2e-${{ github.sha }}
@@ -475,7 +512,7 @@ jobs:
            ${{ runner.os }}-turbo-e2e-
            ${{ runner.os }}-turbo-
      - name: Cache Next.js builds
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 # zizmor: ignore[cache-poisoning] e2e-only Next.js cache; not part of any artifact build or publishing flow
        with:
          path: |
            ~/.npm
@@ -528,17 +565,19 @@ jobs:
      - pre-job
    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
      - name: Login to Docker Hub
        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 # zizmor: ignore[cache-poisoning] server e2e dependency cache only; release/publish steps rebuild separately
        with:
          node-version: 24
          cache: "pnpm"
@@ -589,7 +628,8 @@ jobs:
  all-ci-passed:
    # This allows us to have a branch protection rule for tests and deploys with matrix
    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [
+    needs:
+      [
        lint,
        prettier-check,
        tests-web,
@@ -620,62 +660,222 @@ jobs:
        run: exit 1
        working-directory: .
      - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        if: failure() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ CI failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ CI Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch/Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }
      - name: Output job results
        run: |
-          echo "Job results: ${{ steps.set-success-output.outputs.success }}"
+          echo "Job results: ${STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS}"
+        env:
+          STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}

-  push-docker-image:
+  build-docker-image-release:
    needs: all-ci-passed
    # if something inside all-ci-passed was skipped, but everything that ran passed, we still want to deploy
    if: always() && needs.all-ci-passed.outputs.success == 'true' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    environment: "protected branches"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component: web
+            image_name: langfuse
+            dockerfile: ./web/Dockerfile
+            platform: linux/amd64
+            platform_tag: amd64
+            runner: blacksmith-4vcpu-ubuntu-2404
+          - component: web
+            image_name: langfuse
+            dockerfile: ./web/Dockerfile
+            platform: linux/arm64
+            platform_tag: arm64
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
+          - component: worker
+            image_name: langfuse-worker
+            dockerfile: ./worker/Dockerfile
+            platform: linux/amd64
+            platform_tag: amd64
+            runner: blacksmith-4vcpu-ubuntu-2404
+          - component: worker
+            image_name: langfuse-worker
+            dockerfile: ./worker/Dockerfile
+            platform: linux/arm64
+            platform_tag: arm64
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set NEXT_PUBLIC_BUILD_ID
+        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      - name: Log in to the GitHub Container registry
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Log in to Docker Hub
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Setup Blacksmith Builder
+        uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1
+      - name: Extract metadata (labels) for Docker
+        id: meta
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
+        with:
+          images: |
+            ghcr.io/langfuse/${{ matrix.image_name }}
+            langfuse/${{ matrix.image_name }}
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
+      - name: Build and push image by digest to GitHub Container Registry (${{ matrix.component }}, ${{ matrix.platform_tag }})
+        id: build-ghcr
+        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          outputs: type=image,name=ghcr.io/langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
+          sbom: false
+      - name: Build and push image by digest to Docker Hub (${{ matrix.component }}, ${{ matrix.platform_tag }})
+        id: build-dockerhub
+        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          outputs: type=image,name=langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
+          sbom: false
+      - name: Record pushed digests
+        env:
+          DIGEST_GHCR: ${{ steps.build-ghcr.outputs.digest }}
+          DIGEST_DOCKERHUB: ${{ steps.build-dockerhub.outputs.digest }}
+          PLATFORM_TAG: ${{ matrix.platform_tag }}
+        run: |
+          if [ -z "$DIGEST_GHCR" ] || [ -z "$DIGEST_DOCKERHUB" ]; then
+            echo "Missing registry digest output"
+            exit 1
+          fi
+
+          mkdir -p "$RUNNER_TEMP/digests/ghcr" "$RUNNER_TEMP/digests/dockerhub"
+          printf '%s\n' "$DIGEST_GHCR" > "$RUNNER_TEMP/digests/ghcr/${PLATFORM_TAG}.txt"
+          printf '%s\n' "$DIGEST_DOCKERHUB" > "$RUNNER_TEMP/digests/dockerhub/${PLATFORM_TAG}.txt"
+      - name: Upload release digests
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-digests-${{ matrix.component }}-${{ matrix.platform_tag }}
+          path: |
+            ${{ runner.temp }}/digests/ghcr/${{ matrix.platform_tag }}.txt
+            ${{ runner.temp }}/digests/dockerhub/${{ matrix.platform_tag }}.txt
+          if-no-files-found: error
+
+  publish-docker-image-release:
+    needs:
+      - build-docker-image-release
+    if: always() && needs.build-docker-image-release.result == 'success' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    environment: "protected branches"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component: web
+            image_name: langfuse
+          - component: worker
+            image_name: langfuse-worker
    runs-on: blacksmith-4vcpu-ubuntu-2404
    permissions:
      packages: write
      contents: read

    steps:
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3
-        with:
-          version: 10.33.0
-      - name: Setup node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
-        with:
-          node-version: 24
-          cache-dependency-path: "pnpm-lock.yaml"
-      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-      - name: Set NEXT_PUBLIC_BUILD_ID
-        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
      - name: Log in to the GitHub Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Log in to Docker Hub
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
      - name: Setup Blacksmith Builder
        uses: useblacksmith/setup-docker-builder@5241b2e9423e8b1fa37ed6050ecb62d0fb9a4e38 # v1
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta-web
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
+      - name: Download release digests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
-          images: |
-            ghcr.io/langfuse/langfuse # GitHub
-            langfuse/langfuse # Docker Hub
+          pattern: release-digests-${{ matrix.component }}-*
+          merge-multiple: true
+          path: ${{ runner.temp }}/digests
+      - name: Extract metadata (tags) for GitHub Container Registry
+        id: meta-ghcr
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
+        with:
+          images: ghcr.io/langfuse/${{ matrix.image_name }}
          flavor: |
            latest=false
          tags: |
@@ -686,24 +886,11 @@ jobs:
            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
-      - name: Build and push Docker image (web)
-        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2
+      - name: Extract metadata (tags) for Docker Hub
+        id: meta-dockerhub
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
        with:
-          context: .
-          file: ./web/Dockerfile
-          push: true
-          tags: ${{ steps.meta-web.outputs.tags }}
-          labels: ${{ steps.meta-web.outputs.labels }}
-          platforms: |
-            linux/amd64
-            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta-worker
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
-        with:
-          images: |
-            ghcr.io/langfuse/langfuse-worker # GitHub
-            langfuse/langfuse-worker # Docker Hub
+          images: langfuse/${{ matrix.image_name }}
          flavor: |
            latest=false
          tags: |
@@ -714,22 +901,128 @@ jobs:
            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
-      - name: Build and push Docker image (worker)
-        uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: true
-          tags: ${{ steps.meta-worker.outputs.tags }}
-          labels: ${{ steps.meta-worker.outputs.labels }}
-          platforms: |
-            linux/amd64
-            ${{ startsWith(github.ref, 'refs/tags/') && 'linux/arm64' || '' }}
-      - name: Notify Slack
-        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v2
-        if: always()
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
+      - name: Publish multi-platform manifest to GitHub Container Registry
        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
+        run: |
+          ghcr_tags=()
+          ghcr_sources=()
+
+          while IFS= read -r tag; do
+            [ -n "$tag" ] || continue
+            ghcr_tags+=("-t" "$tag")
+          done <<EOF
+          ${STEPS_META_GHCR_OUTPUTS_TAGS}
+          EOF
+
+          shopt -s nullglob
+          for digest_file in "$RUNNER_TEMP"/digests/ghcr/*.txt; do
+            digest="$(cat "$digest_file")"
+            ghcr_sources+=("ghcr.io/langfuse/${IMAGE_NAME}@$digest")
+          done
+
+          if [ "${#ghcr_sources[@]}" -lt 2 ]; then
+            echo "Expected amd64 and arm64 GHCR digests for $COMPONENT"
+            exit 1
+          fi
+
+          docker buildx imagetools create "${ghcr_tags[@]}" "${ghcr_sources[@]}"
+      - name: Publish multi-platform manifest to Docker Hub
+        env:
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}
+          IMAGE_NAME: ${{ matrix.image_name }}
+          COMPONENT: ${{ matrix.component }}
+        run: |
+          dockerhub_tags=()
+          dockerhub_sources=()
+
+          while IFS= read -r tag; do
+            [ -n "$tag" ] || continue
+            dockerhub_tags+=("-t" "$tag")
+          done <<EOF
+          ${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}
+          EOF
+
+          shopt -s nullglob
+          for digest_file in "$RUNNER_TEMP"/digests/dockerhub/*.txt; do
+            digest="$(cat "$digest_file")"
+            dockerhub_sources+=("langfuse/${IMAGE_NAME}@$digest")
+          done
+
+          if [ "${#dockerhub_sources[@]}" -lt 2 ]; then
+            echo "Expected amd64 and arm64 Docker Hub digests for $COMPONENT"
+            exit 1
+          fi
+
+          docker buildx imagetools create "${dockerhub_tags[@]}" "${dockerhub_sources[@]}"
+      - name: Inspect published manifests
+        run: |
+          ghcr_first_tag="$(printf '%s\n' "${STEPS_META_GHCR_OUTPUTS_TAGS}" | sed -n '1p')"
+          dockerhub_first_tag="$(printf '%s\n' "${STEPS_META_DOCKERHUB_OUTPUTS_TAGS}" | sed -n '1p')"
+
+          docker buildx imagetools inspect "$ghcr_first_tag"
+          docker buildx imagetools inspect "$dockerhub_first_tag"
+        env:
+          STEPS_META_GHCR_OUTPUTS_TAGS: ${{ steps.meta-ghcr.outputs.tags }}
+          STEPS_META_DOCKERHUB_OUTPUTS_TAGS: ${{ steps.meta-dockerhub.outputs.tags }}
+
+  notify-docker-image-release:
+    needs:
+      - build-docker-image-release
+      - publish-docker-image-release
+    if: always() && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - name: Fail when a release image job failed
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: exit 1
+      - name: Notify Slack
+        if: failure()
+        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": "❌ Docker release failed on ${{ github.ref_name }}",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "❌ Docker Release Failed",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Tag:*\n`${{ github.ref_name }}`"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Triggered by:*\n${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View Workflow Logs",
+                        "emoji": true
+                      },
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                      "style": "danger"
+                    }
+                  ]
+                }
+              ]
+            }
@@ -12,6 +12,8 @@ concurrency:
  group: promote-main-to-production
  cancel-in-progress: false

+permissions: {}
+
 jobs:
  promote:
    runs-on: ubuntu-latest
@@ -19,16 +21,19 @@ jobs:
    steps:
      - name: Validate confirmation
        run: |
-          if [ "${{ github.event.inputs.confirm }}" != "promote" ]; then
+          if [ "${INPUT_CONFIRM}" != "promote" ]; then
            echo "Input 'confirm' must be 'promote'."
            exit 1
          fi
+        env:
+          INPUT_CONFIRM: ${{ github.event.inputs.confirm }}

-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: main
          fetch-depth: 0
          token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false

      - name: Print commit refs
        run: |
@@ -41,4 +46,6 @@ jobs:
          fi

      - name: Force push main to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
@@ -5,15 +5,20 @@ on:
    tags:
      - "v3.[0-9]+.[0-9]+" # Semantic version tags

+permissions: {}
+
 jobs:
  release:
    runs-on: ubuntu-latest
    environment: "protected branches"
    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: main # Always checkout main even for tagged releases
          fetch-depth: 0
          token: ${{ secrets.GH_ACCESS_TOKEN }}
+          persist-credentials: false
      - name: Push to production
-        run: git push origin +main:production
+        run: git push "https://x-access-token:${GH_ACCESS_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" +main:production
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
@@ -12,20 +12,26 @@ concurrency:
  group: sdk-api-spec-${{ github.ref }}
  cancel-in-progress: true

+permissions:
+  contents: read
+
 jobs:
  generate-sdk-api-specs:
    runs-on: ubuntu-latest
+    environment: "protected branches"
    steps:
      - name: Checkout repository
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false

      - name: Install pnpm
-        uses: pnpm/action-setup@eae0cfeb286e66ffb5155f1a79b90583a127a68b # v2
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10.33.0

      - name: Setup Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: "24"
          cache: "pnpm"
@@ -39,7 +45,7 @@ jobs:
        run: npx fern-api generate --api server --force

      - name: Install uv
-        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
        with:
          version: "0.11.2"

@@ -3,46 +3,55 @@ on:
  push:
    branches: ["production", "main"]

+permissions:
+  contents: read
+  security-events: write
+
 jobs:
  snyk:
    runs-on: ubuntu-latest
+    environment: snyk
    steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
      - name: Run Snyk to check Docker image for vulnerabilities
        # Snyk can be used to break the build when it detects vulnerabilities.
        # In this case we want to upload the issues to GitHub Code Scanning
        continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
        env:
          # In order to use the Snyk Action you will need to have a Snyk API token.
          # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
          # or you can sign up for free at https://snyk.io/login
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse \
+            --file=web/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
        if: always()
        run: |
          if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
            echo "SARIF file fixed"
          else
            echo "No SARIF file found"
          fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
        if: always()
        with:
          sarif_file: snyk.sarif
+          category: snyk-container-web
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif
@@ -3,46 +3,55 @@ on:
  push:
    branches: ["production", "main"]

+permissions:
+  contents: read
+  security-events: write
+
 jobs:
  snyk:
    runs-on: ubuntu-latest
+    environment: snyk
    steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Snyk CLI
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        with:
+          snyk-version: v1.1304.0
      - name: Run Snyk to check Docker image for vulnerabilities
        # Snyk can be used to break the build when it detects vulnerabilities.
        # In this case we want to upload the issues to GitHub Code Scanning
        continue-on-error: true
-        uses: snyk/actions/docker@9adf32b1121593767fc3c057af55b55db032dc04 # master
        env:
          # In order to use the Snyk Action you will need to have a Snyk API token.
          # See https://docs.snyk.io/integrations/ci-cd-integrations/github-actions-integration#getting-your-snyk-token
          # or you can sign up for free at https://snyk.io/login
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: langfuse/langfuse-worker
-          args: --sarif-file-output=snyk.sarif
-      - name: Fix SARIF file
+        run: |
+          snyk container test langfuse/langfuse-worker \
+            --file=worker/Dockerfile \
+            --sarif-file-output=snyk.sarif
+      - name: Normalize SARIF file
        if: always()
        run: |
          if [ -f snyk.sarif ]; then
-            # Fix undefined security severity values in SARIF file
-            sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
+            # Snyk emits invalid security-severity values. upload-sarif requires a numeric string.
+            sed -i \
+              -e 's/"security-severity": "undefined"/"security-severity": "0"/g' \
+              -e 's/"security-severity": "null"/"security-severity": "0"/g' \
+              -e 's/"security-severity": null/"security-severity": "0"/g' \
+              snyk.sarif
            echo "SARIF file fixed"
          else
            echo "No SARIF file found"
          fi
-      - name: Echo SARIF file for debugging
-        if: always()
-        run: |
-          if [ -f snyk.sarif ]; then
-            echo "=== SARIF File Contents ==="
-            cat snyk.sarif
-            echo "=== End of SARIF File ==="
-          else
-            echo "No SARIF file found to display"
-          fi
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4
        if: always()
        with:
          sarif_file: snyk.sarif
+          category: snyk-container-worker
+      - name: Echo SARIF file for debugging
+        if: failure() && hashFiles('snyk.sarif') != ''
+        run: cat snyk.sarif
@@ -10,7 +10,7 @@ jobs:
      issues: write
      pull-requests: write
    steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
        with:
          days-before-issue-stale: 30
          days-before-issue-close: 14
@@ -0,0 +1,35 @@
+---
+name: Check GitHub Actions
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+  merge_group:
+  pull_request:
+    branches:
+      - "main"
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: Check GitHub Actions security
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
+        with:
+          # Means that the action will only report issues, but not fail the workflow.
+          # Blocking merges are handled by rulesets:
+          # https://docs.github.com/en/code-security/concepts/code-scanning/about-code-scanning-alerts#pull-request-check-failures-for-code-scanning-alerts
+          advanced-security: true
@@ -0,0 +1,21 @@
+rules:
+  secrets-outside-env:
+    config:
+      allow:
+        # Shared read-only CI credentials used to avoid Docker Hub rate limits in test jobs.
+        - DOCKERHUB_USERNAME_READ
+        - DOCKERHUB_TOKEN_READ
+        # Shared integration-test credentials intentionally kept together for the multi-provider LLM connection test job.
+        - LANGFUSE_LLM_CONNECTION_OPENAI_KEY
+        - LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_KEY
+        - LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL
+        - LANGFUSE_LLM_CONNECTION_AZURE_MODEL
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_API_KEY
+        - LANGFUSE_LLM_CONNECTION_BEDROCK_REGION
+        - LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY
+        - LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY
+        # Fern token is generation-only; GitHub write actions are handled by GH_ACCESS_TOKEN in a protected environment.
+        - FERN_TOKEN
@@ -47,6 +47,7 @@ yarn-error.log*
 !.env.dev-redis-cluster.example
 !.env.prod.example
 !.env.test.example
+!.env.dev-oci.example

 # vercel
 .vercel
@@ -31,6 +31,8 @@ services:
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
      LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
+      LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
+      LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "3.167.2",
+  "version": "3.168.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -108,7 +108,8 @@
      "@types/node-fetch": "^2.6.13",
      "@types/react-dom": "19.2.3",
      "glob": "^10.5.0",
-      "qs": "6.14.1"
+      "qs": "6.14.1",
+      "path-to-regexp@0.1.12": "0.1.13"
    },
    "patchedDependencies": {
      "next-auth@4.24.13": "patches/next-auth@4.24.13.patch"
@@ -25,6 +25,7 @@ Use root [AGENTS.md](../../AGENTS.md) for monorepo-level rules.
 - Main exports: `src/index.ts`
 - DB clients and types: `src/db.ts`
 - Server exports: `src/server/index.ts`
+- Server cache utilities: `src/server/cache/*`
 - Domain model types: `src/domain/*`
 - Repository layer: `src/server/repositories/*`
 - Queue payload schemas: `src/server/queues.ts`
@@ -372,7 +372,8 @@ CREATE TABLE IF NOT EXISTS events_core
    INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1,
    INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,
    INDEX idx_updated_at updated_at TYPE minmax GRANULARITY 1,
-    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2
+    INDEX idx_provided_model_name provided_model_name TYPE bloom_filter(0.01) GRANULARITY 2,
+    INDEX idx_experiment_id experiment_id TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree(event_ts, is_deleted)
 PARTITION BY toYYYYMM(start_time)
@@ -112,8 +112,11 @@
    "langfuse-langchain": "3.38.20",
    "lodash": "^4.18.1",
    "lossless-json": "^4.1.1",
+    "lru-cache": "^11.2.7",
    "next-auth": "^4.24.13",
    "nodemailer": "^7.0.11",
+    "oci-objectstorage": "^2.125.0",
+    "oci-common": "^2.125.0",
    "safe-regex2": "^5.0.0",
    "undici": "^7.24.6",
    "uuid": "^9.0.1",
@@ -55,7 +55,6 @@ async function main() {
      name: "Demo User",
      email: "demo@langfuse.com",
      password: await hash("password", 12),
-      featureFlags: ["experimentsV4Enabled"],
    },
    create: {
      id: seedUserId1,
@@ -63,7 +62,6 @@ async function main() {
      email: "demo@langfuse.com",
      password: await hash("password", 12),
      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
-      featureFlags: ["experimentsV4Enabled"],
    },
  });
  const user2 = await prisma.user.upsert({
@@ -1 +1 @@
-export const VERSION = "v3.167.2";
+export const VERSION = "v3.168.0";
@@ -116,6 +116,7 @@ export const EventsObservationSchema = ObservationSchema.extend({
  userId: z.string().nullable(),
  sessionId: z.string().nullable(),
  traceName: z.string().nullable(),
+  tags: z.array(z.string()).optional(),
  bookmarked: z.boolean().optional(),
  public: z.boolean().optional(),
 });
@@ -57,6 +57,18 @@ const EnvSchema = z.object({
    .optional(),
  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_TTL_MS: z.coerce
+    .number()
+    .positive()
+    .default(10_000),
+  LANGFUSE_LOCAL_CACHE_MODEL_MATCH_MAX: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(20_000),
  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(3600), // 1h
  CLICKHOUSE_URL: z.string().url(),
@@ -179,6 +191,21 @@ const EnvSchema = z.object({
    .default("true"),
  LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
  LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
+  LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_OCI_AUTH_TYPE: z
+    .enum([
+      "workload_identity",
+      "instance_principal",
+      "resource_principal",
+      "oci_profile",
+      "session_token",
+    ])
+    .optional(),
+  LANGFUSE_OCI_CONFIG_FILE: z.string().optional(),
+  LANGFUSE_OCI_CONFIG_PROFILE: z.string().optional(),
+  NODE_EXTRA_CA_CERTS: z.string().optional(),
  STRIPE_SECRET_KEY: z.string().optional(),

  LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG: z
@@ -53,14 +53,23 @@ function parseMultiEncodedJson(value: unknown): unknown {
 }

 function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
-  // selectedColumn should already be preprocessed by preprocessObjectWithJsonFields
-  // so we can directly use it with JSONPath
+  // JSONPath can only query objects/arrays — return primitives as-is
+  if (typeof selectedColumn !== "object" || selectedColumn === null) {
+    return selectedColumn;
+  }
+
  const result = JSONPath({
    path: jsonSelector,
    json: selectedColumn as any, // JSONPath accepts unknown but types are strict
  });

-  return Array.isArray(result) && result.length > 0 ? result[0] : undefined;
+  if (!Array.isArray(result) || result.length === 0) {
+    return undefined;
+  }
+
+  // For single-match queries (e.g. $.name), return the unwrapped value.
+  // For multi-match queries (e.g. $[1:], $[*].name), return the full array.
+  return result.length === 1 ? result[0] : result;
 }

 export function extractValueFromObject(
@@ -69,12 +78,7 @@ export function extractValueFromObject(
  jsonSelector?: string,
  parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown,
 ): { value: string; error: Error | null } {
-  let selectedColumn = obj[selectedColumnId];
-
-  // Simple preprocessing: attempt to parse to valid JSON object
-  if (typeof selectedColumn === "string") {
-    selectedColumn = parseMultiEncodedJson(selectedColumn);
-  }
+  const selectedColumn = obj[selectedColumnId];

  const jsonParser = parseJson || parseJsonDefault;

@@ -82,14 +86,21 @@ export function extractValueFromObject(
  let error: Error | null = null;

  if (jsonSelector && selectedColumn) {
+    // Only parse multi-encoded JSON when a selector is present — avoids
+    // mutating formatting (e.g. whitespace) for the no-selector passthrough.
+    const parsed =
+      typeof selectedColumn === "string"
+        ? parseMultiEncodedJson(selectedColumn)
+        : selectedColumn;
+
    try {
-      jsonSelectedColumn = jsonParser(selectedColumn, jsonSelector);
+      jsonSelectedColumn = jsonParser(parsed, jsonSelector);
    } catch (err) {
      error =
        err instanceof Error
          ? err
          : new Error("There was an unknown error parsing the JSON");
-      jsonSelectedColumn = selectedColumn; // Fallback to original value
+      jsonSelectedColumn = selectedColumn; // Fallback to raw original value
    }
  } else {
    jsonSelectedColumn = selectedColumn;
@@ -11,12 +11,25 @@ export const VERTEXAI_USE_DEFAULT_CREDENTIALS =
 export const BedrockConfigSchema = z.object({ region: z.string() });
 export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;

-export const BedrockCredentialSchema = z
+export const BedrockAccessKeysSchema = z
  .object({
-    accessKeyId: z.string(),
-    secretAccessKey: z.string(),
+    accessKeyId: z.string().min(1),
+    secretAccessKey: z.string().min(1),
  })
-  .optional();
+  .strict();
+export type BedrockAccessKeys = z.infer<typeof BedrockAccessKeysSchema>;
+
+export const BedrockApiKeySchema = z
+  .object({
+    apiKey: z.string().min(1),
+  })
+  .strict();
+export type BedrockApiKey = z.infer<typeof BedrockApiKeySchema>;
+
+export const BedrockCredentialSchema = z.union([
+  BedrockAccessKeysSchema,
+  BedrockApiKeySchema,
+]);
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;

 export const VertexAIConfigSchema = z
@@ -0,0 +1 @@
+export * from "./localCache";
@@ -0,0 +1,172 @@
+import { LRUCache } from "lru-cache";
+import { logger } from "../logger";
+import { recordGauge, recordIncrement } from "../instrumentation";
+
+export type LocalCacheLoadResult<V> = {
+  value: V | undefined;
+  ttlMs?: number;
+  source?: string;
+};
+
+export type LocalCacheConfig = {
+  namespace: string;
+  enabled: boolean;
+  ttlMs: number;
+  max: number;
+};
+
+export class LocalCache<V extends {}> {
+  private readonly config: LocalCacheConfig;
+  private readonly cache: LRUCache<string, V>;
+
+  constructor(config: LocalCacheConfig) {
+    this.config = config;
+    const dispose: LRUCache.Disposer<string, V> = (_value, _key, reason) => {
+      if (reason === "evict") {
+        this.record("evict");
+        this.recordSizeMetrics();
+      }
+    };
+
+    const baseOptions = {
+      ttlAutopurge: false as const,
+      allowStale: false as const,
+      updateAgeOnGet: false as const,
+      updateAgeOnHas: false as const,
+      dispose,
+    };
+
+    this.cache = new LRUCache<string, V>({
+      ...baseOptions,
+      ttl: config.ttlMs,
+      max: config.max,
+    });
+    this.logInfo("Initialized local cache", {
+      enabled: config.enabled,
+      ttlMs: config.ttlMs,
+      max: config.max,
+    });
+  }
+
+  get(key: string): V | undefined {
+    if (!this.config.enabled) {
+      return undefined;
+    }
+
+    const value = this.cache.get(key);
+    this.record(value === undefined ? "miss" : "hit");
+    this.logDebug(
+      value === undefined ? "Local cache miss" : "Local cache hit",
+      {
+        size: this.cache.size,
+        keyLength: key.length,
+      },
+    );
+
+    return value;
+  }
+
+  set(key: string, value: V): void {
+    if (!this.config.enabled) {
+      return;
+    }
+
+    const ttlMs = this.config.ttlMs;
+
+    try {
+      this.cache.set(key, value, { ttl: ttlMs });
+      this.record("set");
+      this.recordSizeMetrics();
+      this.logDebug("Stored local cache entry", {
+        ttlMs,
+        size: this.cache.size,
+        keyLength: key.length,
+      });
+    } catch (error) {
+      logger.error(
+        `Failed to set local cache entry for namespace ${this.config.namespace}`,
+        error,
+      );
+    }
+  }
+
+  clear(): void {
+    this.cache.clear();
+    this.record("clear");
+    this.recordSizeMetrics();
+    this.logDebug("Cleared local cache");
+  }
+
+  async getOrLoad(
+    key: string,
+    loader: () => Promise<LocalCacheLoadResult<V>>,
+  ): Promise<LocalCacheLoadResult<V>> {
+    const cached = this.get(key);
+    if (cached !== undefined) {
+      return { value: cached, source: "local" };
+    }
+
+    if (!this.config.enabled) {
+      this.logDebug("Bypassing disabled local cache", {
+        keyLength: key.length,
+      });
+      return loader();
+    }
+
+    const result = await loader();
+    this.logDebug("Completed local cache load", {
+      source: result.source ?? "unknown",
+      cacheable: result.value !== undefined,
+      ttlMs: result.ttlMs ?? null,
+      keyLength: key.length,
+    });
+
+    if (result.value !== undefined) {
+      this.set(key, result.value);
+    }
+
+    return result;
+  }
+
+  private record(metric: string): void {
+    recordIncrement(`langfuse.local_cache.${metric}`, 1, {
+      namespace: this.config.namespace,
+    });
+  }
+
+  private recordSizeMetrics(): void {
+    recordGauge("langfuse.local_cache.size_entries", this.cache.size, {
+      namespace: this.config.namespace,
+    });
+  }
+
+  private logDebug(message: string, metadata?: Record<string, unknown>): void {
+    if (!logger.isLevelEnabled("debug")) {
+      return;
+    }
+
+    const formattedMetadata =
+      metadata === undefined ? "" : ` ${safeSerialize(metadata)}`;
+
+    logger.debug(
+      `[LocalCache:${this.config.namespace}] ${message}${formattedMetadata}`,
+    );
+  }
+
+  private logInfo(message: string, metadata?: Record<string, unknown>): void {
+    const formattedMetadata =
+      metadata === undefined ? "" : ` ${safeSerialize(metadata)}`;
+
+    logger.info(
+      `[LocalCache:${this.config.namespace}] ${message}${formattedMetadata}`,
+    );
+  }
+}
+
+const safeSerialize = (value: unknown): string => {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return "[unserializable]";
+  }
+};
@@ -1,4 +1,5 @@
 export * from "./services/StorageService";
+export * from "./cache";
 export * from "./services/BufferedStreamUploader";
 export * from "./services/S3ChunkedUploadStrategy";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
@@ -28,6 +29,7 @@ export * from "./llm/fetchLLMCompletion";
 export * from "./llm/errors";
 export * from "./llm/utils";
 export * from "./llm/types";
+export * from "./llm/internalTraceEvents";
 export * from "./llm/compileChatMessages";
 export * from "./llm/testModelCall";
 export * from "./llm/baseUrlValidation";
@@ -8,6 +8,7 @@ import {
  safeMultiDel,
  scanKeys,
 } from "../";
+import { LocalCache } from "../cache";
 import { env } from "../../env";
 import { Decimal } from "decimal.js";
 import { prisma } from "../../db";
@@ -24,6 +25,22 @@ export type ModelWithPrices = {
 };

 const MODEL_MATCH_CACHE_LOCKED_KEY = "LOCK:model-match-clear";
+const DEFAULT_LOCAL_CACHE_MODEL_MATCH_TTL_MS = 10_000;
+const DEFAULT_LOCAL_CACHE_MODEL_MATCH_MAX = 20_000;
+// This L1 cache is intentionally TTL-only. Cross-container consistency continues
+// to come from Redis invalidation plus the short local TTL.
+const modelMatchLocalCache = new LocalCache<ModelWithPrices>({
+  namespace: "model_match",
+  enabled: env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED === "true",
+  ttlMs: getPositiveNumberOrDefault(
+    env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_TTL_MS,
+    DEFAULT_LOCAL_CACHE_MODEL_MATCH_TTL_MS,
+  ),
+  max: getPositiveNumberOrDefault(
+    env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_MAX,
+    DEFAULT_LOCAL_CACHE_MODEL_MATCH_MAX,
+  ),
+});

 export async function findModel(p: ModelMatchProps): Promise<ModelWithPrices> {
  return instrumentAsync(
@@ -33,66 +50,132 @@ export async function findModel(p: ModelMatchProps): Promise<ModelWithPrices> {
    },
    async (span) => {
      if (logger.isLevelEnabled("debug")) {
-        logger.debug(`Finding model for ${JSON.stringify(p)}`);
-      }
-      const cachedResult = await getModelWithPricesFromRedis(p);
-      if (cachedResult) {
-        span.setAttribute("model_match_source", "redis");
-
-        if (cachedResult.model === null) {
-          return { model: null, pricingTiers: [] };
-        } else {
-          logger.debug(
-            `Found model name ${cachedResult.model?.modelName} (id: ${cachedResult.model?.id}) for project ${p.projectId} and model ${p.model}`,
-          );
-          span.setAttribute("matched_model_id", cachedResult.model.id);
-        }
-
-        return cachedResult;
-      }
-
-      // try to find model in Postgres
-      const postgresModel = await findModelInPostgres(p);
-
-      if (postgresModel && env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
-        const pricingTiers = await findPricingTiersForModel(postgresModel.id);
-        await addModelWithPricingTiersToRedis(p, postgresModel, pricingTiers);
-
-        span.setAttribute("matched_model_id", postgresModel.id);
-        span.setAttribute("model_match_source", "postgres");
-        span.setAttribute("model_cache_set", "true");
-
        logger.debug(
-          `Found model name ${postgresModel?.modelName} (id: ${postgresModel?.id}) for project ${p.projectId} and model ${p.model}`,
+          formatModelMatchDebugMessage("Resolving model match", {
+            projectId: p.projectId,
+            model: p.model,
+            localCacheEnabled:
+              env.LANGFUSE_LOCAL_CACHE_MODEL_MATCH_ENABLED === "true",
+            redisCacheEnabled:
+              env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true",
+          }),
        );
-        return { model: postgresModel, pricingTiers };
-      } else if (postgresModel) {
-        const pricingTiers = await findPricingTiersForModel(postgresModel.id);
-        span.setAttribute("matched_model_id", postgresModel.id);
-        span.setAttribute("model_match_source", "postgres");
-        span.setAttribute("model_cache_set", "false");
+      }
+      const localCacheKey = getRedisModelKey(p);
+      const { source, value } = await modelMatchLocalCache.getOrLoad(
+        localCacheKey,
+        async () => {
+          const cachedResult = await getModelWithPricesFromRedis(p);
+          if (cachedResult) {
+            return {
+              value: cachedResult,
+              source: "redis",
+            };
+          }

-        logger.debug(
-          `Found model name ${postgresModel?.modelName} (id: ${postgresModel?.id}) for project ${p.projectId} and model ${p.model}`,
-        );
-        return { model: postgresModel, pricingTiers };
-      } else {
-        span.setAttribute("model_match_source", "none");
+          const postgresModel = await findModelInPostgres(p);
+          if (postgresModel) {
+            const pricingTiers = await findPricingTiersForModel(
+              postgresModel.id,
+            );

-        if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
-          await addModelNotFoundTokenToRedis(p);
+            if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
+              await addModelWithPricingTiersToRedis(
+                p,
+                postgresModel,
+                pricingTiers,
+              );
+            }
+
+            return {
+              value: { model: postgresModel, pricingTiers },
+              source: "postgres",
+            };
+          }
+
+          if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true") {
+            await addModelNotFoundTokenToRedis(p);
+          }
+
+          return {
+            value: { model: null, pricingTiers: [] },
+            source: "none",
+          };
+        },
+      );
+
+      if (!value || value.model === null) {
+        span.setAttribute("model_match_source", source ?? "none");
+        if (
+          source === "none" &&
+          env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true"
+        ) {
          span.setAttribute("model_cache_set", "true");
        }

-        logger.debug(
-          `Model not found for project ${p.projectId} and model ${p.model}`,
-        );
+        if (logger.isLevelEnabled("debug")) {
+          logger.debug(
+            formatModelMatchDebugMessage(
+              "Model match resolved without a model",
+              {
+                projectId: p.projectId,
+                model: p.model,
+                source: source ?? "none",
+                pricingTierCount: 0,
+              },
+            ),
+          );
+        }
+
        return { model: null, pricingTiers: [] };
      }
+
+      span.setAttribute("model_match_source", source ?? "unknown");
+      span.setAttribute("matched_model_id", value.model.id);
+      if (source === "postgres") {
+        span.setAttribute(
+          "model_cache_set",
+          String(env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "true"),
+        );
+      }
+
+      if (logger.isLevelEnabled("debug")) {
+        logger.debug(
+          formatModelMatchDebugMessage("Model match resolved", {
+            projectId: p.projectId,
+            model: p.model,
+            source: source ?? "unknown",
+            matchedModelId: value.model.id,
+            matchedModelName: value.model.modelName,
+            pricingTierCount: value.pricingTiers.length,
+          }),
+        );
+      }
+      return value;
    },
  );
 }

+const formatModelMatchDebugMessage = (
+  message: string,
+  metadata: Record<string, unknown>,
+): string => {
+  try {
+    return `${message} ${JSON.stringify(metadata)}`;
+  } catch {
+    return `${message} [unserializable]`;
+  }
+};
+
+function getPositiveNumberOrDefault(value: unknown, fallback: number): number {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export const clearModelMatchLocalCache = (): void => {
+  modelMatchLocalCache.clear();
+};
+
 const getModelWithPricesFromRedis = async (
  p: ModelMatchProps,
 ): Promise<ModelWithPrices | null> => {
@@ -9,6 +9,31 @@ import { logger } from "../logger";

 // type CallbackFn<T> = () => T;

+/**
+ * IORedis request hook that records the full Redis command as a span attribute.
+ * Redacts credentials from AUTH/HELLO and values from API key cache operations.
+ */
+export function ioredisRequestHook(
+  span: opentelemetry.Span,
+  { cmdName, cmdArgs }: { cmdName: string; cmdArgs: unknown[] },
+): void {
+  if (!Array.isArray(cmdArgs) || cmdArgs.length === 0) return;
+  const cmd = cmdName.toUpperCase();
+  // AUTH and HELLO carry raw credentials — redact all args
+  if (cmd === "AUTH" || cmd === "HELLO") {
+    span.setAttribute("redis.full_command", `${cmdName} [REDACTED]`);
+    return;
+  }
+  const args = [...cmdArgs].map(String);
+  // Redact API key cache values: SET [prefix:]api-key:{hash} <json>
+  if (args[0]?.includes("api-key:")) {
+    for (let i = 1; i < args.length; i++) {
+      args[i] = "[REDACTED]";
+    }
+  }
+  span.setAttribute("redis.full_command", `${cmdName} ${args.join(" ")}`);
+}
+
 export type TCarrier = {
  traceparent?: string;
  tracestate?: string;
@@ -19,6 +19,7 @@ import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
 import { env } from "../../env";
 import GCPServiceAccountKeySchema, {
+  BedrockAccessKeysSchema,
  BedrockConfigSchema,
  BedrockCredentialSchema,
  VertexAIConfigSchema,
@@ -53,6 +54,14 @@ import { LLMCompletionError } from "./errors";

 export type CompletionWithReasoning = { text: string; reasoning?: string };

+const NON_RETRYABLE_LLM_ERROR_PATTERNS = [
+  "Request timed out",
+  "is not valid JSON",
+  "Unterminated string in JSON at position",
+  "TypeError",
+  "reached the end of its life",
+] as const;
+
 const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);

 // Maps adapters to the content block types that represent "thinking".
@@ -90,6 +99,52 @@ const googleProviderOptionsSchema = z
  })
  .optional();

+// For using Bedrock API key in Bearer token format
+const createBedrockBearerAuth = (token: string) => ({
+  clientOptions: {
+    token: { token },
+    authSchemePreference: ["httpBearerAuth"],
+  },
+});
+
+export function resolveBedrockAuth(params: {
+  secretKey: string;
+  allowDefaultCredentials: boolean;
+}): {
+  credentials?: z.infer<typeof BedrockAccessKeysSchema>;
+  clientOptions?: {
+    token: { token: string };
+    authSchemePreference: string[];
+  };
+} {
+  const { secretKey, allowDefaultCredentials } = params;
+
+  if (
+    secretKey === BEDROCK_USE_DEFAULT_CREDENTIALS &&
+    allowDefaultCredentials
+  ) {
+    return {};
+  }
+
+  try {
+    const parsedCredential = BedrockCredentialSchema.parse(
+      JSON.parse(secretKey),
+    );
+
+    if ("apiKey" in parsedCredential) {
+      return createBedrockBearerAuth(parsedCredential.apiKey);
+    }
+
+    return {
+      credentials: parsedCredential,
+    };
+  } catch {
+    throw new Error(
+      "Invalid Bedrock credentials. Expected AWS access key JSON or a Bedrock API key.",
+    );
+  }
+}
+
 type ProcessTracedEvents = () => Promise<void>;

 type LLMCompletionParams = {
@@ -363,16 +418,16 @@ export async function fetchLLMCompletion(
    // Handle both explicit credentials and default provider chain
    // Only allow default provider chain in self-hosted or internal AI features
    const isSelfHosted = !isLangfuseCloud;
-    const credentials =
-      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS &&
-      (isSelfHosted || shouldUseLangfuseAPIKey)
-        ? undefined // undefined = use AWS SDK default credential provider chain
-        : BedrockCredentialSchema.parse(JSON.parse(apiKey));
+    const { credentials, clientOptions } = resolveBedrockAuth({
+      secretKey: apiKey,
+      allowDefaultCredentials: isSelfHosted || shouldUseLangfuseAPIKey,
+    });

    chatModel = new ChatBedrockConverse({
      model: modelParams.model,
      region,
      credentials,
+      clientOptions,
      temperature: modelParams.temperature,
      maxTokens: modelParams.max_tokens,
      topP: modelParams.top_p,
@@ -577,20 +632,17 @@ export async function fetchLLMCompletion(
    return completion;
  } catch (e) {
    const responseStatusCode =
-      (e as any)?.response?.status ?? (e as any)?.status ?? 500;
+      (e as any)?.response?.status ??
+      (e as any)?.status ??
+      // Bedrock errors have status code in $metadata.httpStatusCode
+      (e as any)?.$metadata?.httpStatusCode ??
+      500;
    const rawMessage = e instanceof Error ? e.message : String(e);
    const message = extractCleanErrorMessage(rawMessage);

    // Check for non-retryable error patterns in message
-    const nonRetryablePatterns = [
-      "Request timed out",
-      "is not valid JSON",
-      "Unterminated string in JSON at position",
-      "TypeError",
-    ];
-
-    const hasNonRetryablePattern = nonRetryablePatterns.some((pattern) =>
-      message.includes(pattern),
+    const hasNonRetryablePattern = NON_RETRYABLE_LLM_ERROR_PATTERNS.some(
+      (pattern) => message.includes(pattern),
    );

    // Determine retryability:
@@ -1,88 +1,22 @@
 import CallbackHandler from "langfuse-langchain";
-import { GenerationDetails, TraceSinkParams } from "./types";
+import { ProcessedTraceEvent, TraceSinkParams } from "./types";
+import { buildInternalTraceEventInputs } from "./internalTraceEvents";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
 import { traceException } from "../instrumentation";

-type TracedEvent = {
-  type: string;
-  body: Record<string, unknown>;
-};
+export function prepareInternalTraceEvents(params: {
+  events: Array<{
+    type: string;
+    timestamp: string;
+    body: Record<string, unknown>;
+  }>;
+  environment: string;
+  prompt?: TraceSinkParams["prompt"];
+}): ProcessedTraceEvent[] {
+  const { events, environment, prompt } = params;

-/**
- * Extracts and merges generation details from a list of processed events.
- * Handles multiple generation-create and generation-update events with the same id.
- *
- * Events are merged following the "last non-null value wins" pattern:
- * - generation-create events contain: id, name, input, metadata
- * - generation-update events contain: output, usage, usageDetails
- *
- * @returns GenerationDetails or null if no generation events found
- */
-export function extractGenerationDetails(
-  processedEvents: Array<{ type: string; body: Record<string, unknown> }>,
-): GenerationDetails | null {
-  // 1. Filter to only generation events
-  const generationEvents = processedEvents.filter(
-    (event) =>
-      event.type === "generation-create" || event.type === "generation-update",
-  );
-
-  if (generationEvents.length === 0) {
-    return null;
-  }
-
-  // 2. Get the generation id from first event
-  const generationId = generationEvents[0].body.id as string;
-  if (!generationId) {
-    return null;
-  }
-
-  // 3. Filter to events for this generation id only
-  const eventsForGeneration = generationEvents.filter(
-    (event) => event.body.id === generationId,
-  );
-
-  // 4. Merge event bodies (last non-null/non-undefined value wins)
-  // Similar to IngestionService pattern but simplified for our use case
-  const mergedBody = eventsForGeneration.reduce(
-    (acc: Record<string, unknown>, event) => {
-      for (const [key, value] of Object.entries(event.body)) {
-        if (value !== undefined && value !== null) {
-          // Special handling for metadata: deep merge
-          if (
-            key === "metadata" &&
-            typeof value === "object" &&
-            !Array.isArray(value)
-          ) {
-            acc[key] = {
-              ...((acc[key] as Record<string, unknown>) || {}),
-              ...(value as Record<string, unknown>),
-            };
-          } else {
-            acc[key] = value;
-          }
-        }
-      }
-      return acc;
-    },
-    { id: generationId },
-  );
-
-  return {
-    observationId: generationId,
-    name: (mergedBody.name as string) || "generation",
-    input: mergedBody.input,
-    output: mergedBody.output,
-    metadata: (mergedBody.metadata as Record<string, unknown>) || {},
-  };
-}
-
-export function prepareTracedEventsForIngestion(
-  events: TracedEvent[],
-  { environment, prompt }: Pick<TraceSinkParams, "environment" | "prompt">,
-): TracedEvent[] {
-  const blockedSpanIds = new Set<string>();
+  const blockedSpanIds = new Set();
  const blockedSpanNames = [
    "RunnableLambda",
    "StructuredOutputParser",
@@ -91,29 +25,27 @@ export function prepareTracedEventsForIngestion(
  ];

  for (const event of events) {
-    const eventName = event.body.name;
+    const eventName = "name" in event.body ? event.body.name : "";

    if (typeof eventName !== "string" || eventName.length === 0) {
      continue;
    }

-    if (
-      blockedSpanNames.includes(eventName) &&
-      typeof event.body.id === "string"
-    ) {
+    if (blockedSpanNames.includes(eventName as string) && "id" in event.body) {
      blockedSpanIds.add(event.body.id);
    }
  }

  return events
    .filter((event) => {
-      if (typeof event.body.id === "string") {
+      if ("id" in event.body) {
        return !blockedSpanIds.has(event.body.id);
      }

      return true;
    })
    .map((event) => {
+      // Inject environment into all events
      return {
        ...event,
        body: {
@@ -142,7 +74,8 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
  handler: CallbackHandler;
  processTracedEvents: () => Promise<void>;
 } {
-  const { prompt, targetProjectId, environment, userId } = traceSinkParams;
+  const { prompt, targetProjectId, environment, userId, eventsWriter } =
+    traceSinkParams;
  const handler = new CallbackHandler({
    _projectId: targetProjectId,
    _isLocalEventExportEnabled: true,
@@ -155,41 +88,52 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
      const events = await handler.langfuse._exportLocalEvents(
        traceSinkParams.targetProjectId,
      );
+      const processedEvents = prepareInternalTraceEvents({
+        events,
+        environment,
+        prompt,
+      });

-      const processedEvents = prepareTracedEventsForIngestion(
-        events as TracedEvent[],
-        {
-          environment,
-          prompt,
-        },
-      );
+      // Legacy write to traces/observations tables
+      try {
+        await processEventBatch(
+          JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
+          {
+            validKey: true as const,
+            scope: {
+              projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
+              accessLevel: "project",
+            } as any,
+          },
+          {
+            isLangfuseInternal: true,
+            forwardToEventsTable: eventsWriter ? false : undefined, // Do not dual write when we already direct event write
+          },
+        );
+      } catch (processingError) {
+        traceException(processingError);
+        logger.error("Failed to process traced events via legacy ingestion", {
+          error: processingError,
+        });
+      }

-      await processEventBatch(
-        JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
-        {
-          validKey: true as const,
-          scope: {
-            projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
-            accessLevel: "project",
-          } as any,
-        },
-        {
-          isLangfuseInternal: true,
-        },
-      );
-
-      // Extract generation details and invoke callback (if provided)
-      if (traceSinkParams.onGenerationComplete) {
+      // Direct write to events table
+      if (eventsWriter) {
        try {
-          const generationDetails = extractGenerationDetails(processedEvents);
-          if (generationDetails) {
-            traceSinkParams.onGenerationComplete(generationDetails);
+          const { rootSpanId, eventInputs } = buildInternalTraceEventInputs({
+            processedEvents,
+            traceId: traceSinkParams.traceId,
+            projectId: targetProjectId,
+            experimentContext: eventsWriter.experimentContext,
+          });
+
+          if (eventInputs.length > 0) {
+            await eventsWriter.write({ rootSpanId, eventInputs });
          }
-        } catch (extractionError) {
-          // Don't fail the LLM call due to generation detail extraction errors
-          traceException(extractionError);
-          logger.error("Failed to extract generation details from events", {
-            error: extractionError,
+        } catch (writeError) {
+          traceException(writeError);
+          logger.error("Failed to direct-write internal traced events", {
+            error: writeError,
          });
        }
      }
@@ -0,0 +1,417 @@
+import {
+  asBoolean,
+  asNumberRecord,
+  asRecord,
+  asString,
+  asStringArray,
+} from "../../utils/objects";
+import { stringifyValue } from "../../utils/stringChecks";
+import {
+  convertCallsToArrays,
+  convertDefinitionsToMap,
+  extractToolsFromObservation,
+} from "../ingestion/extractToolsBackend";
+import { flattenJsonToPathArrays } from "../otel/utils";
+import type { ProcessedTraceEvent } from "./types";
+
+export const INTERNAL_TRACE_EVENT_SOURCE = "ingestion-api-dual-write";
+export const INTERNAL_TRACE_EXPERIMENT_EVENT_SOURCE =
+  "ingestion-api-dual-write-experiments";
+
+export type InternalTraceExperimentContext = {
+  id: string;
+  name: string;
+  metadata?: Record<string, unknown>;
+  description?: string | null;
+  datasetId: string;
+  itemId: string;
+  itemVersion: string;
+  itemExpectedOutput?: unknown;
+  itemMetadata?: Record<string, unknown> | null;
+};
+
+/**
+ * Flexible input type for writing events to the events table.
+ * This is intentionally loose to allow for iteration as the events
+ * table schema evolves. Only required fields are enforced.
+ */
+export type InternalTraceEventInput = {
+  projectId: string;
+  traceId: string;
+  spanId: string;
+  startTimeISO: string;
+  orgId?: string;
+  parentSpanId?: string;
+  name?: string;
+  type?: string;
+  environment?: string;
+  version?: string;
+  release?: string;
+  endTimeISO: string;
+  completionStartTime?: string;
+  traceName?: string;
+  tags?: string[];
+  bookmarked?: boolean;
+  public?: boolean;
+  userId?: string;
+  sessionId?: string;
+  level?: string;
+  statusMessage?: string;
+  promptId?: string;
+  promptName?: string;
+  promptVersion?: string;
+  modelId?: string;
+  modelName?: string;
+  modelParameters?: string | Record<string, unknown>;
+  providedUsageDetails?: Record<string, number>;
+  usageDetails?: Record<string, number>;
+  providedCostDetails?: Record<string, number>;
+  costDetails?: Record<string, number>;
+  toolDefinitions?: Record<string, string>;
+  toolCalls?: string[];
+  toolCallNames?: string[];
+  input?: string;
+  output?: string;
+  metadata: Record<string, unknown>;
+  source: string;
+  serviceName?: string;
+  serviceVersion?: string;
+  scopeName?: string;
+  scopeVersion?: string;
+  telemetrySdkLanguage?: string;
+  telemetrySdkName?: string;
+  telemetrySdkVersion?: string;
+  blobStorageFilePath?: string;
+  eventRaw?: string;
+  eventBytes?: number;
+  experimentId?: string;
+  experimentName?: string;
+  experimentMetadataNames?: string[];
+  experimentMetadataValues?: Array<string | null | undefined>;
+  experimentDescription?: string;
+  experimentDatasetId?: string;
+  experimentItemId?: string;
+  experimentItemVersion?: string;
+  experimentItemRootSpanId?: string;
+  experimentItemExpectedOutput?: string;
+  experimentItemMetadataNames?: string[];
+  experimentItemMetadataValues?: Array<string | null | undefined>;
+  [key: string]: any;
+};
+
+type InternalTraceSnapshot = {
+  spanId: string;
+  traceId: string;
+  parentSpanId?: string;
+  name?: string;
+  type: "SPAN" | "GENERATION";
+  environment?: string;
+  version?: string;
+  release?: string;
+  startTimeISO?: string;
+  endTimeISO?: string;
+  completionStartTime?: string;
+  level?: string;
+  statusMessage?: string;
+  promptName?: string;
+  promptVersion?: string;
+  modelName?: string;
+  modelParameters?: Record<string, unknown>;
+  providedUsageDetails?: Record<string, number>;
+  providedCostDetails?: Record<string, number>;
+  input?: unknown;
+  output?: unknown;
+  metadata: Record<string, unknown>;
+  tags?: string[];
+  public?: boolean;
+  bookmarked?: boolean;
+  userId?: string;
+  sessionId?: string;
+};
+
+export type MaterializedInternalTrace = {
+  rootSpanId: string;
+  snapshots: InternalTraceSnapshot[];
+};
+
+function isCreateEvent(type: string): boolean {
+  return type.endsWith("-create");
+}
+
+function getSnapshotType(eventType: string): "SPAN" | "GENERATION" {
+  return eventType.startsWith("generation-") ? "GENERATION" : "SPAN";
+}
+
+function getEventTime(
+  event: ProcessedTraceEvent,
+  body: Record<string, unknown>,
+): number {
+  const candidates = [body.startTime, body.timestamp, event.timestamp];
+
+  for (const candidate of candidates) {
+    if (typeof candidate === "string") {
+      const parsed = new Date(candidate).getTime();
+      if (!Number.isNaN(parsed)) {
+        return parsed;
+      }
+    }
+  }
+
+  return 0;
+}
+
+function getTimestampMs(timestamp?: string): number {
+  if (!timestamp) {
+    return 0;
+  }
+
+  const parsed = new Date(timestamp).getTime();
+  return Number.isNaN(parsed) ? 0 : parsed;
+}
+
+function sortEvents(events: ProcessedTraceEvent[]): ProcessedTraceEvent[] {
+  return [...events].sort((left, right) => {
+    const timeDelta =
+      getEventTime(left, left.body) - getEventTime(right, right.body);
+
+    if (timeDelta !== 0) {
+      return timeDelta;
+    }
+
+    if (isCreateEvent(left.type) === isCreateEvent(right.type)) {
+      return 0;
+    }
+
+    return isCreateEvent(left.type) ? -1 : 1;
+  });
+}
+
+function flattenMetadata(value: unknown): {
+  names: string[];
+  values: Array<string | null | undefined>;
+} {
+  const metadata = asRecord(value);
+  return metadata
+    ? flattenJsonToPathArrays(metadata)
+    : { names: [], values: [] };
+}
+
+function mergeSnapshotEvent(
+  snapshot: InternalTraceSnapshot,
+  event: ProcessedTraceEvent,
+): InternalTraceSnapshot {
+  const { body } = event;
+  const startTime = asString(body.startTime);
+  const timestamp = asString(body.timestamp) ?? asString(event.timestamp);
+  const metadata = asRecord(body.metadata);
+
+  return {
+    ...snapshot,
+    traceId: asString(body.traceId) ?? snapshot.traceId,
+    parentSpanId: asString(body.parentObservationId) ?? snapshot.parentSpanId,
+    type:
+      snapshot.type === "GENERATION"
+        ? snapshot.type
+        : getSnapshotType(event.type),
+    name: asString(body.name) ?? snapshot.name,
+    environment: asString(body.environment) ?? snapshot.environment,
+    version: asString(body.version) ?? snapshot.version,
+    release: asString(body.release) ?? snapshot.release,
+    startTimeISO:
+      startTime ?? snapshot.startTimeISO ?? timestamp ?? snapshot.startTimeISO,
+    endTimeISO: asString(body.endTime) ?? snapshot.endTimeISO,
+    completionStartTime:
+      asString(body.completionStartTime) ?? snapshot.completionStartTime,
+    level: asString(body.level) ?? snapshot.level,
+    statusMessage: asString(body.statusMessage) ?? snapshot.statusMessage,
+    promptName: asString(body.promptName) ?? snapshot.promptName,
+    promptVersion:
+      typeof body.promptVersion === "number"
+        ? body.promptVersion.toString()
+        : (asString(body.promptVersion) ?? snapshot.promptVersion),
+    modelName: asString(body.model) ?? snapshot.modelName,
+    modelParameters: asRecord(body.modelParameters) ?? snapshot.modelParameters,
+    providedUsageDetails:
+      asNumberRecord(body.usageDetails) ??
+      asNumberRecord(body.usage) ??
+      snapshot.providedUsageDetails,
+    providedCostDetails:
+      asNumberRecord(body.costDetails) ?? snapshot.providedCostDetails,
+    input:
+      body.input !== undefined && body.input !== null
+        ? body.input
+        : snapshot.input,
+    output:
+      body.output !== undefined && body.output !== null
+        ? body.output
+        : snapshot.output,
+    metadata: metadata
+      ? { ...snapshot.metadata, ...metadata }
+      : snapshot.metadata,
+    tags: asStringArray(body.tags) ?? snapshot.tags,
+    public: asBoolean(body.public) ?? snapshot.public,
+    bookmarked: asBoolean(body.bookmarked) ?? snapshot.bookmarked,
+    userId: asString(body.userId) ?? snapshot.userId,
+    sessionId: asString(body.sessionId) ?? snapshot.sessionId,
+  };
+}
+
+export function materializeInternalTrace(params: {
+  processedEvents: ProcessedTraceEvent[];
+  traceId: string;
+}): MaterializedInternalTrace {
+  const { processedEvents, traceId } = params;
+  const snapshots = new Map<string, InternalTraceSnapshot>();
+  const traceCreateEvent = processedEvents.find(
+    (e) => e.type === "trace-create",
+  );
+  const rootSpanId = asString(traceCreateEvent?.body.id) ?? traceId;
+
+  for (const event of sortEvents(processedEvents)) {
+    const spanId = asString(event.body.id);
+
+    if (!spanId) {
+      continue;
+    }
+
+    const existingSnapshot =
+      snapshots.get(spanId) ??
+      ({
+        spanId,
+        traceId: asString(event.body.traceId) ?? traceId,
+        type: getSnapshotType(event.type),
+        metadata: {},
+      } satisfies InternalTraceSnapshot);
+
+    snapshots.set(spanId, mergeSnapshotEvent(existingSnapshot, event));
+  }
+
+  const orderedSnapshots = [...snapshots.values()].sort((left, right) => {
+    if (left.spanId === rootSpanId) {
+      return -1;
+    }
+
+    if (right.spanId === rootSpanId) {
+      return 1;
+    }
+
+    return (
+      getTimestampMs(left.startTimeISO) - getTimestampMs(right.startTimeISO)
+    );
+  });
+
+  return { rootSpanId, snapshots: orderedSnapshots };
+}
+
+export function buildInternalTraceEventInputs(params: {
+  processedEvents: ProcessedTraceEvent[];
+  traceId: string;
+  projectId: string;
+  experimentContext?: InternalTraceExperimentContext;
+}): {
+  rootSpanId: string;
+  eventInputs: InternalTraceEventInput[];
+} {
+  const { processedEvents, traceId, projectId, experimentContext } = params;
+  // Direct write uses original IDs (observation.id === trace.id for root).
+  // The experiment backfill job skips traces already in events_core via LEFT ANTI JOIN,
+  // so there's no deduplication concern between direct write and backfill.
+  const { rootSpanId, snapshots } = materializeInternalTrace({
+    processedEvents,
+    traceId,
+  });
+  const rootSnapshot = snapshots.find((s) => s.spanId === rootSpanId);
+
+  if (!rootSnapshot) {
+    return { rootSpanId, eventInputs: [] };
+  }
+
+  const experimentMetadata = flattenMetadata(experimentContext?.metadata);
+  const experimentItemMetadata = flattenMetadata(
+    experimentContext?.itemMetadata,
+  );
+  const source = experimentContext
+    ? INTERNAL_TRACE_EXPERIMENT_EVENT_SOURCE
+    : INTERNAL_TRACE_EVENT_SOURCE;
+
+  const eventInputs = snapshots.map((snapshot) => {
+    const { toolDefinitions, toolArguments } = extractToolsFromObservation(
+      snapshot.input,
+      snapshot.output,
+    );
+    const toolCalls = convertCallsToArrays(toolArguments);
+    const isRoot = snapshot.spanId === rootSpanId;
+
+    return {
+      projectId,
+      traceId,
+      spanId: snapshot.spanId,
+      parentSpanId: isRoot ? undefined : (snapshot.parentSpanId ?? rootSpanId),
+      name:
+        snapshot.name ??
+        (snapshot.type === "GENERATION"
+          ? "generation"
+          : (rootSnapshot.name ?? "span")),
+      type: snapshot.type,
+      environment: snapshot.environment ?? rootSnapshot.environment,
+      version: snapshot.version ?? rootSnapshot.version,
+      release: rootSnapshot.release,
+      startTimeISO:
+        snapshot.startTimeISO ??
+        rootSnapshot.startTimeISO ??
+        new Date().toISOString(),
+      endTimeISO:
+        snapshot.endTimeISO ??
+        snapshot.startTimeISO ??
+        rootSnapshot.endTimeISO ??
+        rootSnapshot.startTimeISO ??
+        new Date().toISOString(),
+      completionStartTime: snapshot.completionStartTime,
+      traceName: rootSnapshot.name,
+      tags: rootSnapshot.tags ?? [],
+      bookmarked: rootSnapshot.bookmarked,
+      public: rootSnapshot.public,
+      userId: rootSnapshot.userId,
+      sessionId: rootSnapshot.sessionId,
+      level: snapshot.level ?? "DEFAULT",
+      statusMessage: snapshot.statusMessage,
+      promptName: snapshot.promptName,
+      promptVersion: snapshot.promptVersion,
+      modelName: snapshot.modelName,
+      modelParameters: snapshot.modelParameters,
+      providedUsageDetails: snapshot.providedUsageDetails,
+      providedCostDetails: snapshot.providedCostDetails,
+      toolDefinitions: convertDefinitionsToMap(toolDefinitions),
+      toolCalls: toolCalls.tool_calls,
+      toolCallNames: toolCalls.tool_call_names,
+      input:
+        snapshot.input !== undefined
+          ? stringifyValue(snapshot.input)
+          : undefined,
+      output:
+        snapshot.output !== undefined
+          ? stringifyValue(snapshot.output)
+          : undefined,
+      metadata: snapshot.metadata,
+      source,
+      experimentId: experimentContext?.id,
+      experimentName: experimentContext?.name,
+      experimentMetadataNames: experimentMetadata.names,
+      experimentMetadataValues: experimentMetadata.values,
+      experimentDescription: experimentContext?.description ?? undefined,
+      experimentDatasetId: experimentContext?.datasetId,
+      experimentItemId: experimentContext?.itemId,
+      experimentItemVersion: experimentContext?.itemVersion,
+      experimentItemRootSpanId: experimentContext ? rootSpanId : undefined,
+      experimentItemExpectedOutput:
+        experimentContext?.itemExpectedOutput !== undefined &&
+        experimentContext?.itemExpectedOutput !== null
+          ? stringifyValue(experimentContext.itemExpectedOutput)
+          : undefined,
+      experimentItemMetadataNames: experimentItemMetadata.names,
+      experimentItemMetadataValues: experimentItemMetadata.values,
+    } satisfies InternalTraceEventInput;
+  });
+
+  return { rootSpanId, eventInputs };
+}
@@ -5,6 +5,10 @@ import {
  VertexAIConfigSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { JSONObjectSchema } from "../../utils/zod";
+import type {
+  InternalTraceEventInput,
+  InternalTraceExperimentContext,
+} from "./internalTraceEvents";

 // disable lint as this is exported and used in web/worker

@@ -429,6 +433,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 export const anthropicModels = [
  "claude-sonnet-4-5-20250929",
  "claude-haiku-4-5-20251001",
+  "claude-opus-4-7",
  "claude-sonnet-4-6",
  "claude-opus-4-6",
  "claude-opus-4-5-20251101",
@@ -533,16 +538,22 @@ export enum LangfuseInternalTraceEnvironment {
  LLMJudge = "langfuse-llm-as-a-judge",
 }

+export type ProcessedTraceEvent = {
+  type: string;
+  timestamp: string;
+  body: Record<string, unknown>;
+};
+
 /**
- * Details of a generation extracted from traced events.
- * Used to pass generation information from internal tracing to callbacks.
+ * Configuration for direct writing of trace events to the events table.
+ * Used by internal tracing (prompt experiments, evaluations).
 */
-export type GenerationDetails = {
-  observationId: string;
-  name: string;
-  input: unknown;
-  output: unknown;
-  metadata: Record<string, unknown>;
+export type InternalEventsWriter = {
+  experimentContext?: InternalTraceExperimentContext;
+  write: (params: {
+    rootSpanId: string;
+    eventInputs: InternalTraceEventInput[];
+  }) => Promise<void>;
 };

 export type TraceSinkParams = {
@@ -561,8 +572,10 @@ export type TraceSinkParams = {
    version: number;
  };
  /**
-   * Optional callback invoked after the generation events have been processed.
-   * Called with merged generation details (from create + update events).
+   * When provided, traced events are written directly to the events table,
+   * bypassing the legacy traces/observations ingestion pipeline for the events write.
+   * Used for internal tracing (prompt experiments, LLM-as-a-judge evaluations). Traced
+   * events are still written to the legacy traces/observations tables.
   */
-  onGenerationComplete?: (details: GenerationDetails) => void;
+  eventsWriter?: InternalEventsWriter;
 };
@@ -1315,18 +1315,25 @@ export class OtelIngestionProcessor {
    const keys = Object.keys(input).map((key) => key.replace(`${prefix}.`, ""));
    const useArray = keys.some((key) => key.match(/^\d+\./));

+    // Blocklist to prevent prototype pollution via crafted OTel attribute keys
+    const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
    // Helper function to set a value at a nested path
    const setNestedValue = (obj: any, path: string[], value: unknown): void => {
      let current = obj;
      for (let i = 0; i < path.length - 1; i++) {
        const key = path[i];
+        if (DANGEROUS_KEYS.has(key)) return;
        if (!(key in current)) {
          // Check if next key is a number to decide if we need an array or object
          current[key] = /^\d+$/.test(path[i + 1]) ? [] : {};
        }
        current = current[key];
      }
-      current[path[path.length - 1]] = value;
+      const finalKey = path[path.length - 1];
+      if (!DANGEROUS_KEYS.has(finalKey)) {
+        current[finalKey] = value;
+      }
    };

    if (useArray) {
@@ -1335,7 +1342,7 @@ export class OtelIngestionProcessor {
        const pathParts = key.split(".");
        const index = parseInt(pathParts[0], 10);
        if (!result[index]) {
-          result[index] = {};
+          result[index] = Object.create(null);
        }
        if (pathParts.length === 2) {
          // Simple case: 0.content -> result[0].content
@@ -1351,7 +1358,7 @@ export class OtelIngestionProcessor {
      }
      return result;
    } else {
-      const result: Record<string, unknown> = {};
+      const result: Record<string, unknown> = Object.create(null);
      for (const key of keys) {
        const pathParts = key.split(".");
        if (pathParts.length === 1) {
@@ -2476,51 +2483,120 @@ export class OtelIngestionProcessor {
    }

    if (instrumentationScopeName === "pydantic-ai") {
-      const inputTokens = attributes["gen_ai.usage.input_tokens"];
-      const outputTokens = attributes["gen_ai.usage.output_tokens"];
-      const cacheReadTokens =
-        attributes["gen_ai.usage.cache_read_tokens"] ??
-        attributes["gen_ai.usage.details.cache_read_input_tokens"];
-      const cacheWriteTokens =
-        attributes["gen_ai.usage.cache_write_tokens"] ??
-        attributes["gen_ai.usage.details.cache_creation_input_tokens"];
-
-      return {
-        input: inputTokens,
-        output: outputTokens,
-        input_cache_read: cacheReadTokens,
-        input_cache_creation: cacheWriteTokens,
-      };
+      const usageDetails = this.extractGenericGenAiUsageDetails(attributes);
+      if (Object.keys(usageDetails).length > 0) return usageDetails;
    }

+    return this.extractGenericGenAiUsageDetails(attributes);
+  }
+
+  private extractGenericGenAiUsageDetails(
+    attributes: Record<string, unknown>,
+  ): Record<string, number> {
    const usageDetails = Object.keys(attributes).filter(
      (key) =>
        (key.startsWith("gen_ai.usage.") && key !== "gen_ai.usage.cost") ||
-        key.startsWith("llm.token_count"),
+        key.startsWith("llm.token_count."),
    );

-    const usageDetailKeyMapping: Record<string, string> = {
-      prompt_tokens: "input",
-      completion_tokens: "output",
-      total_tokens: "total",
-      input_tokens: "input",
-      output_tokens: "output",
-      prompt: "input",
-      completion: "output",
-    };
+    if (usageDetails.length === 0) return {};

-    return usageDetails.reduce((acc: any, key) => {
-      const usageDetailKey = key
-        .replace("gen_ai.usage.", "")
-        .replace("llm.token_count.", "");
-      const mappedUsageDetailKey =
-        usageDetailKeyMapping[usageDetailKey] ?? usageDetailKey;
-      const value = Number(attributes[key]);
-      if (!Number.isNaN(value)) {
-        acc[mappedUsageDetailKey] = value;
-      }
-      return acc;
-    }, {});
+    const rawUsageDetails = usageDetails.reduce(
+      (acc: Record<string, number>, key) => {
+        const usageDetailKey = key
+          .replace("gen_ai.usage.", "")
+          .replace("llm.token_count.", "");
+        const value = Number(attributes[key]);
+
+        if (!Number.isNaN(value)) {
+          acc[usageDetailKey] = value;
+        }
+
+        return acc;
+      },
+      {},
+    );
+
+    const inputTokens =
+      rawUsageDetails["prompt_tokens"] ??
+      rawUsageDetails["input_tokens"] ??
+      rawUsageDetails["prompt"];
+    const outputTokens =
+      rawUsageDetails["completion_tokens"] ??
+      rawUsageDetails["output_tokens"] ??
+      rawUsageDetails["completion"];
+    const totalTokens =
+      rawUsageDetails["total_tokens"] ?? rawUsageDetails["total"];
+    const cacheReadTokens =
+      rawUsageDetails["cache_read.input_tokens"] ??
+      rawUsageDetails["cache_read_tokens"] ??
+      rawUsageDetails["details.cache_read_tokens"] ??
+      rawUsageDetails["details.cache_read_input_tokens"];
+    const cacheCreationTokens =
+      rawUsageDetails["cache_creation.input_tokens"] ??
+      rawUsageDetails["cache_write_tokens"] ??
+      rawUsageDetails["details.cache_write_tokens"] ??
+      rawUsageDetails["details.cache_creation_input_tokens"];
+
+    const normalizedUsageDetails = Object.entries(rawUsageDetails).reduce(
+      (acc: Record<string, number>, [key, value]) => {
+        if (
+          [
+            "prompt_tokens",
+            "input_tokens",
+            "prompt",
+            "completion_tokens",
+            "output_tokens",
+            "completion",
+            "total_tokens",
+            "total",
+            "cache_read.input_tokens",
+            "cache_read_tokens",
+            "details.cache_read_tokens",
+            "details.cache_read_input_tokens",
+            "cache_creation.input_tokens",
+            "cache_write_tokens",
+            "details.cache_write_tokens",
+            "details.cache_creation_input_tokens",
+          ].includes(key)
+        ) {
+          return acc;
+        }
+
+        const normalizedKey = key.startsWith("details.")
+          ? key.replace("details.", "")
+          : key;
+
+        acc[normalizedKey] = value;
+        return acc;
+      },
+      {},
+    );
+
+    if (inputTokens !== undefined) {
+      normalizedUsageDetails.input = Math.max(
+        inputTokens - (cacheReadTokens ?? 0) - (cacheCreationTokens ?? 0),
+        0,
+      );
+    }
+
+    if (outputTokens !== undefined) {
+      normalizedUsageDetails.output = outputTokens;
+    }
+
+    if (totalTokens !== undefined) {
+      normalizedUsageDetails.total = totalTokens;
+    }
+
+    if (cacheReadTokens !== undefined) {
+      normalizedUsageDetails.input_cached_tokens = cacheReadTokens;
+    }
+
+    if (cacheCreationTokens !== undefined) {
+      normalizedUsageDetails.input_cache_creation = cacheCreationTokens;
+    }
+
+    return normalizedUsageDetails;
  }

  private extractCostDetails(
@@ -181,6 +181,7 @@ const FIELD_SETS = {
    "userId",
    "sessionId",
    "traceName",
+    "tags",
    "toolDefinitions",
    "toolCalls",
    "toolCallNames",
@@ -217,6 +218,7 @@ const FIELD_SETS = {
    "userId",
    "sessionId",
    "traceName",
+    "tags",
  ],
  calculated: ["latency", "timeToFirstToken"],
  io: ["input", "output"],
@@ -239,6 +241,12 @@ const FIELD_SETS = {
    "level",
    "statusMessage",
    "version",
+    "userId",
+    "sessionId",
+    "traceName",
+    "tags",
+    "bookmarked",
+    "public",
    "toolDefinitions",
    "toolCalls",
    "toolCallNames",
@@ -1710,6 +1718,11 @@ const EXPERIMENTS_AGGREGATION_FIELDS = {
    "groupUniqArrayIf(tuple(e.prompt_name, e.prompt_version), e.prompt_name != '') AS prompts",
  experimentMetadata:
    "any(mapFromArrays(e.experiment_metadata_names, e.experiment_metadata_values)) AS experiment_metadata",
+
+  // Metrics fields
+  totalCost: "SUM(e.total_cost) AS total_cost",
+  latencyAvg:
+    "avgIf(date_diff('millisecond', e.start_time, e.end_time), e.span_id = e.experiment_item_root_span_id AND e.end_time IS NOT NULL) AS latency_avg",
 } as const;

 /**
@@ -1728,6 +1741,7 @@ const EXPERIMENTS_AGGREGATION_FIELD_SETS = {
    "prompts",
    "experimentMetadata",
  ] as const,
+  metrics: ["experimentId", "totalCost", "latencyAvg"] as const,
 } as const;

 export type ExperimentsAggregationFieldSetName =
@@ -1736,9 +1750,9 @@ export type ExperimentsAggregationFieldSetName =
 /**
 * ExperimentsAggregationQueryBuilder - Aggregates events by (experiment_id, project_id).
 *
- * For metrics requiring trace-level aggregation first (cost, latency), use CTEQueryBuilder
- * to wrap a trace CTE and re-aggregate at experiment level with selectRaw() + groupBy().
- * selectRaw() is intentionally used for explicit two-level aggregation semantics.
+ * Use the "metrics" field set for cost and latency aggregations:
+ * - Cost: SUM of all event costs for the experiment
+ * - Latency: AVG of root span duration (where span_id = experiment_item_root_span_id)
 */
 export class ExperimentsAggregationQueryBuilder extends BaseEventsQueryBuilder<
  typeof EXPERIMENTS_AGGREGATION_FIELDS
@@ -115,6 +115,7 @@ export const eventsObservationRecordReadSchema =
    user_id: z.string().nullish(),
    session_id: z.string().nullish(),
    trace_name: z.string().nullish(),
+    tags: z.array(z.string()).optional(),
    bookmarked: z.boolean().optional(),
    public: z.boolean().optional(),
  });
@@ -51,16 +51,16 @@ import {
  queryClickhouse,
  queryClickhouseStream,
 } from "./clickhouse";
-import { ObservationRecordReadType, TraceRecordReadType } from "./definitions";
+import {
+  EventsObservationRecordReadType,
+  TraceRecordReadType,
+} from "./definitions";
 import type { AnalyticsObservationEvent } from "../analytics-integrations/types";
 import {
  ObservationsTableQueryResult,
  ObservationTableQuery,
 } from "./observations";
-import {
-  convertEventsObservation,
-  convertObservation,
-} from "./observations_converters";
+import { convertEventsObservation } from "./observations_converters";
 import {
  EventsQueryBuilder,
  CTEQueryBuilder,
@@ -193,18 +193,22 @@ async function enrichObservationsWithModelData(
 async function enrichObservationsWithTraceFields(
  observationRecords: Array<EventsObservation & ObservationPriceFields>,
 ): Promise<FullEventsObservations> {
-  return observationRecords.map((o) => {
+  return observationRecords.map((observation) => {
+    // Remove raw tags field as this is re-mapped to traceTags
+    const { tags: _tags, ...observationWithoutRawTags } = observation;
    return {
-      ...o,
-      traceTags: [], // TODO pull from PG
+      ...observationWithoutRawTags,
+      traceTags: observation.tags ?? [],
      traceTimestamp: null,
-      toolDefinitions: o.toolDefinitions ?? null,
-      toolCalls: o.toolCalls ?? null,
+      toolDefinitions: observation.toolDefinitions ?? null,
+      toolCalls: observation.toolCalls ?? null,
      // Compute counts from actual data for events table
-      toolDefinitionsCount: o.toolDefinitions
-        ? Object.keys(o.toolDefinitions).length
+      toolDefinitionsCount: observation.toolDefinitions
+        ? Object.keys(observation.toolDefinitions).length
+        : null,
+      toolCallsCount: observation.toolCalls
+        ? observation.toolCalls.length
        : null,
-      toolCallsCount: o.toolCalls ? o.toolCalls.length : null,
    };
  });
 }
@@ -609,9 +613,19 @@ export const getObservationByIdFromEventsTable = async ({
    renderingProps,
    preferredClickhouseService: preferredClickhouseService ?? "EventsReadOnly",
  });
-  const mapped = records.map((record) =>
-    convertObservation(record, renderingProps),
-  );
+  const mapped = records.map((record) => {
+    // Remove raw tags field as this is re-mapped to traceTags
+    const { tags, ...converted } = convertEventsObservation(
+      record,
+      renderingProps,
+      true,
+    );
+
+    return {
+      ...converted,
+      traceTags: tags ?? [],
+    };
+  });

  mapped.forEach((observation) => {
    recordDistribution(
@@ -682,7 +696,7 @@ async function getObservationByIdFromEventsTableInternal({

  const { query, params } = queryBuilder.buildWithParams();

-  return await queryClickhouse<ObservationRecordReadType>({
+  return await queryClickhouse<EventsObservationRecordReadType>({
    query,
    params,
    tags: {
@@ -1072,7 +1086,7 @@ async function getObservationsCountFromEventsTableForPublicApiInternal(
 */
 export const getObservationsFromEventsTableForPublicApi = async (
  opts: Omit<PublicApiObservationsQuery, "fields">,
-): Promise<Array<Observation & ObservationPriceFields>> => {
+): Promise<Array<EventsObservation & ObservationPriceFields>> => {
  const { projectId } = opts;

  // Build query with filters and common CTEs
@@ -1090,12 +1104,15 @@ export const getObservationsFromEventsTableForPublicApi = async (
      projectId,
      queryBuilder,
    );
-  return await enrichObservationsWithModelData(
+
+  const observations = await enrichObservationsWithModelData(
    observationRecords,
    opts.projectId,
    opts.parseIoAsJson ?? true, // V1 API: default to parsing JSON (backwards compatibility)
    null, // V1 API: no field groups, return complete observations
  );
+
+  return observations;
 };

 /**
@@ -19,7 +19,6 @@ import {
  eventsExperimentsAggregation,
  eventsScoresAggregation,
  eventsTracesScoresAggregation,
-  eventsTracesAggregation,
 } from "../queries/clickhouse-sql/query-fragments";
 import { extractTimeFilter, queryClickhouse } from "../repositories";
 import { parseClickhouseUTCDateTimeFormat } from "../repositories/clickhouse";
@@ -174,27 +173,17 @@ export const getExperimentMetricsFromEvents = async (props: {
    return [];
  }

-  const tracesBuilder = eventsTracesAggregation({
+  // Use eventsExperimentsAggregation with "metrics" field set for simplified aggregation
+  const queryBuilder = eventsExperimentsAggregation({
    projectId: props.projectId,
-  }).whereRaw("e.experiment_id IN ({experimentIds: Array(String)})", {
+    fieldSet: "metrics",
    experimentIds: props.experimentIds,
  });

-  // Build the final query
-  const queryBuilder = new CTEQueryBuilder()
-    .withCTEFromBuilder("traces_agg", tracesBuilder)
-    .from("traces_agg", "ta")
-    .select(
-      "ta.experiment_id AS experiment_id",
-      "SUM(ta.total_cost) AS total_cost",
-      "AVG(ta.latency_milliseconds) AS latency_avg",
-    )
-    .groupBy("ta.project_id", "ta.experiment_id");
-
  const { query, params } = queryBuilder.buildWithParams();

  const res = await measureAndReturn({
-    operationName: "getExperimentsFromEventsGeneric",
+    operationName: "getExperimentMetricsFromEvents",
    projectId: props.projectId,
    input: {
      params,
@@ -202,7 +191,7 @@ export const getExperimentMetricsFromEvents = async (props: {
        feature: "experiments",
        type: "experiments-table",
        projectId: props.projectId,
-        operation_name: `getExperimentMetricsFromEvents`,
+        operation_name: "getExperimentMetricsFromEvents",
      },
    },
    fn: async (input) => {
@@ -411,6 +411,7 @@ export function convertEventsObservation(
    userId: record.user_id ?? null,
    sessionId: record.session_id ?? null,
    traceName: record.trace_name ?? null,
+    tags: record.tags ?? [],
    bookmarked: record.bookmarked,
    public: record.public,
  };
@@ -22,6 +22,10 @@ import { backOff } from "exponential-backoff";
 import { ServiceUnavailableError } from "../../errors";
 import { BufferedStreamUploader } from "./BufferedStreamUploader";
 import { S3ChunkedUploadStrategy } from "./S3ChunkedUploadStrategy";
+import * as objectstorage from "oci-objectstorage";
+import * as common from "oci-common";
+import { UploadManager as OciUploadManager } from "oci-objectstorage";
+import { URL } from "node:url";

 export interface S3SseConfig {
  serverSideEncryption?: string;
@@ -127,6 +131,7 @@ export class StorageServiceFactory {
   * @param params.region - Region in which the bucket resides
   * @param params.forcePathStyle - Add bucket name into the path instead of the domain name. Mainly used for MinIO.
   * @param params.useAzureBlob - Use Azure Blob Storage instead of S3
+   * @param params.useOCIObjectStorage - Use OCI Object Storage instead of S3
   * @param params.useGoogleCloudStorage - Use Google Cloud Storage instead of S3
   * @param params.googleCloudCredentials - Google Cloud Storage credentials JSON string or path to credentials file
   * @param params.awsSse - Server-side encryption method (e.g., "aws:kms")
@@ -142,6 +147,7 @@ export class StorageServiceFactory {
    forcePathStyle: boolean;
    useAzureBlob?: boolean;
    useGoogleCloudStorage?: boolean;
+    useOCIObjectStorage?: boolean;
    googleCloudCredentials?: string;
    awsSse: string | undefined;
    awsSseKmsKeyId: string | undefined;
@@ -167,6 +173,13 @@ export class StorageServiceFactory {
      };
      return new GoogleCloudStorageService(googleParams);
    }
+    if (
+      params.useOCIObjectStorage !== undefined
+        ? params.useOCIObjectStorage
+        : env.LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE === "true"
+    ) {
+      return new OCIObjectStorageService(params);
+    }
    return new S3StorageService(params);
  }
 }
@@ -1009,3 +1022,492 @@ class GoogleCloudStorageService implements StorageService {
    }
  }
 }
+
+class OCIObjectStorageService implements StorageService {
+  private client?: objectstorage.ObjectStorageClient;
+  private clientInit: Promise<void>;
+  private bucketName: string;
+  private externalEndpoint?: string;
+  private namespaceName: string = "";
+
+  constructor(params: {
+    bucketName: string;
+    endpoint: string | undefined;
+    externalEndpoint?: string | undefined;
+    region: string | undefined;
+  }) {
+    this.bucketName = params.bucketName;
+    this.externalEndpoint = params.externalEndpoint;
+    this.clientInit = this.initClient(params);
+  }
+
+  private async initClient(params: { endpoint?: string; region?: string }) {
+    let provider: common.AuthenticationDetailsProvider;
+    switch (env.LANGFUSE_OCI_AUTH_TYPE) {
+      case "workload_identity": {
+        provider =
+          new common.OkeWorkloadIdentityAuthenticationDetailsProvider.OkeWorkloadIdentityAuthenticationDetailsProviderBuilder().build();
+
+        break;
+      }
+
+      case "instance_principal": {
+        provider =
+          await new common.InstancePrincipalsAuthenticationDetailsProviderBuilder().build();
+
+        break;
+      }
+
+      case "resource_principal": {
+        provider =
+          common.ResourcePrincipalAuthenticationDetailsProvider.builder();
+
+        break;
+      }
+
+      case "oci_profile": {
+        provider = new common.ConfigFileAuthenticationDetailsProvider(
+          env.LANGFUSE_OCI_CONFIG_FILE,
+          env.LANGFUSE_OCI_CONFIG_PROFILE,
+        );
+
+        break;
+      }
+
+      case "session_token": {
+        provider = new common.SessionAuthDetailProvider(
+          env.LANGFUSE_OCI_CONFIG_FILE,
+          env.LANGFUSE_OCI_CONFIG_PROFILE,
+        );
+
+        break;
+      }
+
+      default:
+        throw new Error(
+          "OCI auth not configured: set LANGFUSE_OCI_AUTH_TYPE to " +
+            "'workload_identity' | 'instance_principal' | 'resource_principal' | 'oci_profile' | 'session_token'",
+        );
+    }
+
+    this.client = new objectstorage.ObjectStorageClient({
+      authenticationDetailsProvider: provider,
+    });
+
+    const regionId = params.region?.trim();
+    if (regionId) this.client.region = common.Region.fromRegionId(regionId);
+    const endpoint = params.endpoint?.trim();
+    if (endpoint) this.client.endpoint = endpoint;
+  }
+
+  private async ensureClient() {
+    await this.clientInit;
+    if (!this.client)
+      throw new Error("OCI ObjectStorage client failed to initialize");
+    return this.client;
+  }
+
+  private async ensureNamespace(): Promise<string> {
+    if (this.namespaceName) return this.namespaceName;
+    const client = await this.ensureClient();
+    const nsResp = await client.getNamespace({});
+    this.namespaceName = nsResp.value ?? "";
+    return this.namespaceName;
+  }
+
+  private async getClientAndNamespace(): Promise<{
+    client: objectstorage.ObjectStorageClient;
+    namespaceName: string;
+  }> {
+    const client = await this.ensureClient();
+    const namespaceName = await this.ensureNamespace(); // uses the same client init + cached namespace
+    return { client, namespaceName };
+  }
+
+  private async streamToString(
+    readable: any, // could be many shapes, so use `any`
+  ): Promise<string> {
+    if (!readable) return "";
+
+    // Helper: convert many chunk shapes to Buffer
+    const toBuffer = (chunk: any): Buffer => {
+      if (Buffer.isBuffer(chunk)) return chunk;
+      if (typeof chunk === "string") return Buffer.from(chunk, "utf8");
+      if (chunk instanceof ArrayBuffer) return Buffer.from(chunk);
+      // TypedArray / DataView
+      if (ArrayBuffer.isView(chunk)) {
+        return Buffer.from(
+          (chunk as Uint8Array).buffer,
+          (chunk as any).byteOffset ?? 0,
+          (chunk as any).byteLength ?? undefined,
+        );
+      }
+      // Fallback: try Buffer.from (may throw)
+      return Buffer.from(chunk);
+    };
+
+    // 1) Node.js Readable (EventEmitter style)
+    if (
+      typeof readable.on === "function" &&
+      typeof readable.read !== "undefined"
+    ) {
+      return await new Promise<string>((resolve, reject) => {
+        const chunks: Buffer[] = [];
+        readable.on("data", (chunk: any) => {
+          try {
+            chunks.push(toBuffer(chunk));
+          } catch (_err) {
+            // if conversion fails, push as Buffer of stringified chunk
+            chunks.push(Buffer.from(String(chunk)));
+          }
+        });
+        readable.on("error", (err: any) => reject(err));
+        readable.on("end", () => {
+          resolve(Buffer.concat(chunks).toString("utf8"));
+        });
+      });
+    }
+
+    // 2) WHATWG ReadableStream (browser / some fetch-like APIs)
+    if (typeof readable.getReader === "function") {
+      const reader = readable.getReader();
+      const chunks: Buffer[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(toBuffer(value));
+        }
+        return Buffer.concat(chunks).toString("utf8");
+      } finally {
+        // safe to close reader if available
+        try {
+          if (reader.releaseLock) reader.releaseLock();
+        } catch (_err) {
+          // intentionally ignore releaseLock errors
+        }
+      }
+    }
+
+    // 3) Buffer / Uint8Array / ArrayBuffer direct
+    if (Buffer.isBuffer(readable)) return readable.toString("utf8");
+    if (readable instanceof Uint8Array)
+      return Buffer.from(readable).toString("utf8");
+    if (readable instanceof ArrayBuffer)
+      return Buffer.from(readable).toString("utf8");
+
+    // 4) Blob (browser)
+    if (typeof Blob !== "undefined" && readable instanceof Blob) {
+      const ab = await readable.arrayBuffer();
+      return Buffer.from(ab).toString("utf8");
+    }
+
+    // 5) Async iterable (some stream implementations)
+    if (typeof readable[Symbol.asyncIterator] === "function") {
+      const chunks: Buffer[] = [];
+      for await (const chunk of readable) {
+        chunks.push(toBuffer(chunk));
+      }
+      return Buffer.concat(chunks).toString("utf8");
+    }
+
+    // 6) Synchronous iterable
+    if (typeof readable[Symbol.iterator] === "function") {
+      const chunks: Buffer[] = [];
+      for (const chunk of readable) {
+        chunks.push(toBuffer(chunk));
+      }
+      return Buffer.concat(chunks).toString("utf8");
+    }
+
+    // 7) Fallback: try string conversion
+    try {
+      return String(readable);
+    } catch (_err) {
+      // If all else fails, throw a helpful error
+      throw new TypeError("Unsupported body type passed to streamToString");
+    }
+  }
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    partSize,
+    queueSize,
+  }: UploadFile): Promise<void> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const uploadManager = new OciUploadManager(client, {
+        partSize: partSize ?? 20 * 1024 * 1024,
+        maxConcurrentUploads: queueSize ?? 5,
+      });
+
+      // UploadManager in the OCI SDK expects content shaped as one of:
+      // { blob }, { filePath }, or { stream }.
+      // To work reliably in Node, always provide { stream }.
+      const stream =
+        typeof data === "string"
+          ? Readable.from([data])
+          : data instanceof Readable
+            ? data
+            : Buffer.isBuffer(data as any)
+              ? Readable.from([data as any])
+              : Readable.from([String(data)]);
+
+      const contentLength =
+        typeof data === "string"
+          ? Buffer.byteLength(data)
+          : Buffer.isBuffer(data as any)
+            ? (data as any).byteLength
+            : undefined;
+
+      await uploadManager.upload({
+        requestDetails: {
+          namespaceName,
+          bucketName: this.bucketName,
+          objectName: fileName,
+          contentType: fileType,
+          ...(contentLength ? { contentLength } : {}),
+        },
+        content: { stream },
+      });
+
+      return;
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(err, "upload file to OCI Object Storage ");
+    }
+  }
+
+  public async uploadFileBuffered({
+    fileName,
+    fileType,
+    data,
+    partSizeBytes,
+  }: UploadFileBuffered): Promise<void> {
+    await this.uploadFile({
+      fileName,
+      fileType,
+      data,
+      partSize: partSizeBytes,
+    });
+  }
+
+  public async uploadWithSignedUrl({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+    partSize,
+    queueSize,
+  }: UploadWithSignedUrl): Promise<{ signedUrl: string }> {
+    try {
+      await this.uploadFile({ fileName, data, fileType, partSize, queueSize });
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(
+        err,
+        "upload file to OCI Object Storage  or generate signed URL",
+      );
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const jsonString = JSON.stringify(body);
+      const req: objectstorage.requests.PutObjectRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        objectName: path,
+        contentLength: Buffer.byteLength(jsonString),
+        putObjectBody: Readable.from([jsonString]),
+        contentType: "application/json",
+      };
+      await client.putObject(req);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to OCI Object Storage ${path}`, err);
+      handleStorageError(err, "upload JSON to OCI Object Storage ");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const req: objectstorage.requests.GetObjectRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        objectName: path,
+      };
+      const response = await client.getObject(req);
+      const bodyStream = (response as any).value as
+        | NodeJS.ReadableStream
+        | undefined;
+      return await this.streamToString(bodyStream);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from OCI Object Storage  ${path}`,
+        err,
+      );
+      handleStorageError(err, "download file from OCI Object Storage ");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const req: objectstorage.requests.ListObjectsRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        prefix,
+      };
+      const resp = await client.listObjects(req);
+      const objects = ((resp as any).listObjects?.objects ?? []) as Array<{
+        name?: string;
+        timeCreated?: Date | string;
+      }>;
+      return (
+        objects.flatMap((obj) =>
+          obj.name
+            ? [
+                {
+                  file: obj.name,
+                  createdAt: obj.timeCreated
+                    ? new Date(obj.timeCreated as any)
+                    : new Date(),
+                },
+              ]
+            : [],
+        ) ?? []
+      );
+    } catch (err) {
+      logger.error(
+        `Failed to list files from OCI Object Storage  ${prefix}`,
+        err,
+      );
+      handleStorageError(err, "list files from OCI Object Storage ");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const expiresOn = new Date(Date.now() + ttlSeconds * 1000);
+      const req: objectstorage.requests.CreatePreauthenticatedRequestRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        createPreauthenticatedRequestDetails: {
+          name: `read-${fileName}-${Date.now()}`,
+          accessType: "ObjectRead" as any,
+          objectName: fileName,
+          timeExpires: expiresOn as any,
+        } as any,
+      };
+      const resp = await client.createPreauthenticatedRequest(req);
+      const accessUri = (resp.preauthenticatedRequest as any)
+        .accessUri as string;
+      const base = this.externalEndpoint ?? client.endpoint;
+
+      if (!base) {
+        throw new Error(
+          "Cannot build PAR URL: no externalEndpoint configured and client.endpoint is empty",
+        );
+      }
+      const baseUrl = new URL(base);
+      const parUrl = new URL(accessUri, baseUrl);
+      if (asAttachment) {
+        parUrl.searchParams.set("download", "1");
+      }
+      const url = parUrl.toString();
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL (PAR) for OCI Object Storage  ${fileName}`,
+        err,
+      );
+      handleStorageError(err, "generate signed URL for OCI Object Storage ");
+    }
+  }
+
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      for (const p of paths) {
+        const req: objectstorage.requests.DeleteObjectRequest = {
+          namespaceName,
+          bucketName: this.bucketName,
+          objectName: p,
+        } as any;
+        await client.deleteObject(req as any);
+      }
+    } catch (err) {
+      logger.error(`Failed to delete files from OCI Object Storage `, {
+        error: err,
+        files: paths,
+      });
+      handleStorageError(err, "delete files from OCI Object Storage ");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds } = params;
+    try {
+      const { client, namespaceName } = await this.getClientAndNamespace();
+      const expiresOn = new Date(Date.now() + ttlSeconds * 1000);
+      const req: objectstorage.requests.CreatePreauthenticatedRequestRequest = {
+        namespaceName,
+        bucketName: this.bucketName,
+        createPreauthenticatedRequestDetails: {
+          name: `write-${path}-${Date.now()}`,
+          accessType: "ObjectWrite" as any,
+          objectName: path,
+          timeExpires: expiresOn as any,
+        } as any,
+      };
+      const resp = await client.createPreauthenticatedRequest(req);
+      const accessUri = (resp.preauthenticatedRequest as any)
+        .accessUri as string;
+      const base = this.externalEndpoint ?? client.endpoint;
+
+      if (!base) {
+        throw new Error(
+          "Cannot build PAR URL: no externalEndpoint configured and client.endpoint is empty",
+        );
+      }
+      const baseUrl = new URL(base);
+      let url = new URL(accessUri, baseUrl).toString();
+      return url;
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL (PAR) for OCI Object Storage  ${path}`,
+        err,
+      );
+      handleStorageError(
+        err,
+        "generate presigned upload URL for OCI Object Storage ",
+      );
+    }
+  }
+}
@@ -227,6 +227,22 @@ export async function notifyBlockedEvaluatorConfigs({
    return;
  }

+  const project = await prisma.project.findUnique({
+    where: {
+      id: projectId,
+    },
+    select: {
+      name: true,
+    },
+  });
+
+  if (!project) {
+    logger.warn(
+      `[EVALUATOR BLOCK] Project ${projectId} not found. Skipping notifications.`,
+    );
+    return;
+  }
+
  const blockedConfigs = await prisma.jobConfiguration.findMany({
    where: {
      projectId,
@@ -254,6 +270,7 @@ export async function notifyBlockedEvaluatorConfigs({
    adminEmails.map((receiverEmail) =>
      sendEvaluatorBlockedEmail({
        env: emailEnv,
+        projectName: project.name,
        evaluatorName: config.evalTemplate?.name ?? config.scoreName,
        blockReason,
        blockMessage,
@@ -16,6 +16,7 @@ import {
 import { EvaluatorBlockReason } from "@prisma/client";

 type EvaluatorBlockedEmailTemplateProps = {
+  projectName: string;
  evaluatorName: string;
  blockReason: EvaluatorBlockReason;
  blockMessage: string;
@@ -104,6 +105,7 @@ const getResolutionSteps = (blockReason: EvaluatorBlockReason) => {
 };

 export const EvaluatorBlockedEmailTemplate = ({
+  projectName,
  evaluatorName,
  blockReason,
  blockMessage,
@@ -114,8 +116,8 @@ export const EvaluatorBlockedEmailTemplate = ({
    <Html>
      <Head />
      <Preview>
-        LLM evaluator &quot;{evaluatorName}&quot; paused:{" "}
-        {getReasonSummary(blockReason)}
+        LLM evaluator &quot;{evaluatorName}&quot; in project &quot;
+        {projectName}&quot; paused: {getReasonSummary(blockReason)}
      </Preview>
      <Tailwind>
        <Body className="bg-background my-auto mx-auto font-sans">
@@ -135,8 +137,9 @@ export const EvaluatorBlockedEmailTemplate = ({
                ⚠️ Evaluator Paused
              </Heading>
              <Text className="text-gray-700 text-sm leading-6">
-                The LLM evaluator &quot;{evaluatorName}&quot; was automatically
-                paused because {getReasonSummary(blockReason).toLowerCase()}.
+                The LLM evaluator &quot;{evaluatorName}&quot; in project &quot;
+                {projectName}&quot; was automatically paused because{" "}
+                {getReasonSummary(blockReason).toLowerCase()}.
              </Text>
            </Section>

@@ -174,7 +177,8 @@ export const EvaluatorBlockedEmailTemplate = ({
            <Section>
              <Text className="text-[#666666] text-[12px] leading-[24px]">
                This notification was sent to {receiverEmail} regarding the
-                paused evaluator &quot;{evaluatorName}&quot;.
+                paused evaluator &quot;{evaluatorName}&quot; in project &quot;
+                {projectName}&quot;.
              </Text>
            </Section>
          </Container>
@@ -17,6 +17,7 @@ export type SendEvaluatorBlockedEmailParams = {
      string | undefined
    >
  >;
+  projectName: string;
  evaluatorName: string;
  blockReason: EvaluatorBlockReason;
  blockMessage: string;
@@ -26,6 +27,7 @@ export type SendEvaluatorBlockedEmailParams = {

 export const sendEvaluatorBlockedEmail = async ({
  env,
+  projectName,
  evaluatorName,
  blockReason,
  blockMessage,
@@ -42,9 +44,11 @@ export const sendEvaluatorBlockedEmail = async ({
  try {
    const mailer = createTransport(parseConnectionUrl(env.SMTP_CONNECTION_URL));
    const safeEvaluatorName = sanitizeEmailSubject(evaluatorName);
+    const safeProjectName = sanitizeEmailSubject(projectName);
    const subject = `⚠️ LLM evaluator "${safeEvaluatorName}" paused - action required`;
    const html = await render(
      EvaluatorBlockedEmailTemplate({
+        projectName: safeProjectName,
        evaluatorName: safeEvaluatorName,
        blockReason,
        blockMessage,
@@ -3,7 +3,6 @@ type OmitKeys<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
 /**
 * Removes specified keys from an object and returns a new object without those keys.
 */
-
 export function removeObjectKeys<T, K extends keyof T>(
  obj: T,
  keys: K[],
@@ -14,3 +13,62 @@ export function removeObjectKeys<T, K extends keyof T>(
  }
  return result;
 }
+
+/**
+ * Safely coerces a value to a Record if it's a plain object.
+ * Returns undefined for null, undefined, arrays, and non-objects.
+ */
+export function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+/**
+ * Safely coerces a value to a string if it's a non-empty string.
+ * Returns undefined for empty strings and non-strings.
+ */
+export function asString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+/**
+ * Safely coerces a value to a boolean if it's a boolean.
+ * Returns undefined for non-booleans.
+ */
+export function asBoolean(value: unknown): boolean | undefined {
+  return typeof value === "boolean" ? value : undefined;
+}
+
+/**
+ * Safely coerces a value to a string array if it's an array of strings.
+ * Returns undefined for non-arrays or arrays with non-string elements.
+ */
+export function asStringArray(value: unknown): string[] | undefined {
+  return Array.isArray(value) &&
+    value.every((entry) => typeof entry === "string")
+    ? value
+    : undefined;
+}
+
+/**
+ * Safely coerces a value to a Record<string, number> if it's an object with numeric values.
+ * Filters out non-finite numbers. Returns undefined if result is empty or input is not an object.
+ */
+export function asNumberRecord(
+  value: unknown,
+): Record<string, number> | undefined {
+  const record = asRecord(value);
+
+  if (!record) {
+    return undefined;
+  }
+
+  const normalized = Object.fromEntries(
+    Object.entries(record).filter(
+      ([, entry]) => typeof entry === "number" && Number.isFinite(entry),
+    ),
+  ) as Record<string, number>;
+
+  return Object.keys(normalized).length > 0 ? normalized : undefined;
+}
@@ -29,6 +29,9 @@ MINIO_SHA256_AMD64="${MINIO_SHA256_AMD64:-7c5bd8512c6e966455b1d198209358b2d191c7
 MINIO_SHA256_ARM64="${MINIO_SHA256_ARM64:-5c83cd2cf151717ba0243f73e1c7802ff36e272b67144bdd7f1f7d684fd6f03d}"
 MC_SHA256_AMD64="${MC_SHA256_AMD64:-01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891}"
 MC_SHA256_ARM64="${MC_SHA256_ARM64:-14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c}"
+MIGRATE_RELEASE_TAG="${MIGRATE_RELEASE_TAG:-v4.19.1}"
+MIGRATE_SHA256_AMD64="${MIGRATE_SHA256_AMD64:-2ac648fbd1b127b69ab5a7b33cf96212178f71e22379fc50573630c6f4c7ce18}"
+MIGRATE_SHA256_ARM64="${MIGRATE_SHA256_ARM64:-2fea2455c0f3f07cc3f4b98471c951ad1a716059574b20b6416bd1e9058751c5}"

 export DEBIAN_FRONTEND=noninteractive

@@ -114,6 +117,55 @@ ensure_clickhouse_binaries() {
  stop_service_if_running clickhouse-server
 }

+detect_migrate_arch() {
+  local machine_arch
+  machine_arch="$(uname -m)"
+
+  case "$machine_arch" in
+    x86_64|amd64)
+      echo "amd64"
+      ;;
+    aarch64|arm64)
+      echo "arm64"
+      ;;
+    *)
+      echo "Unsupported architecture for golang-migrate binary: $machine_arch" >&2
+      exit 1
+      ;;
+  esac
+}
+
+ensure_migrate_binary() {
+  if command -v migrate >/dev/null 2>&1; then
+    return 0
+  fi
+
+  ensure_apt_package ca-certificates
+  ensure_apt_package curl
+
+  local migrate_arch
+  local migrate_sha256
+  local tmp_dir
+  migrate_arch="$(detect_migrate_arch)"
+  case "$migrate_arch" in
+    amd64)
+      migrate_sha256="$MIGRATE_SHA256_AMD64"
+      ;;
+    arm64)
+      migrate_sha256="$MIGRATE_SHA256_ARM64"
+      ;;
+  esac
+  tmp_dir="$(mktemp -d)"
+  trap 'rm -rf "$tmp_dir"' RETURN
+
+  download_and_verify_sha256 \
+    "https://github.com/golang-migrate/migrate/releases/download/${MIGRATE_RELEASE_TAG}/migrate.linux-${migrate_arch}.tar.gz" \
+    "$tmp_dir/migrate.tar.gz" \
+    "$migrate_sha256"
+  tar -xzf "$tmp_dir/migrate.tar.gz" -C "$tmp_dir" migrate
+  install -m 0755 "$tmp_dir/migrate" /usr/local/bin/migrate
+}
+
 detect_minio_arch() {
  local machine_arch
  machine_arch="$(uname -m)"
@@ -359,14 +411,18 @@ ensure_clickhouse_running() {
  local clickhouse_log="$clickhouse_root/clickhouse.log"
  local clickhouse_err="$clickhouse_root/clickhouse.err.log"
  local clickhouse_pid="$clickhouse_root/clickhouse.pid"
+  local -a clickhouse_runner

  mkdir -p "$clickhouse_data"
  if [ "${EUID:-$(id -u)}" -eq 0 ] && id -u clickhouse >/dev/null 2>&1; then
    chown -R clickhouse:clickhouse "$clickhouse_root"
+    clickhouse_runner=(runuser -u clickhouse --)
+  else
+    clickhouse_runner=()
  fi

  if ! wait_for_http "http://127.0.0.1:$CLICKHOUSE_HTTP_PORT/ping" 1; then
-    clickhouse-server \
+    "${clickhouse_runner[@]}" clickhouse-server \
      --daemon \
      --config-file=/etc/clickhouse-server/config.xml \
      --pid-file="$clickhouse_pid" \
@@ -389,7 +445,9 @@ ensure_clickhouse_running() {
  clickhouse_user_identifier="$(escape_clickhouse_identifier "$CLICKHOUSE_USER")"

  clickhouse-client --host 127.0.0.1 --port "$CLICKHOUSE_NATIVE_PORT" -q "CREATE USER IF NOT EXISTS $clickhouse_user_identifier IDENTIFIED WITH plaintext_password BY '$clickhouse_password_sql'"
-  clickhouse-client --host 127.0.0.1 --port "$CLICKHOUSE_NATIVE_PORT" -q "GRANT ALL ON *.* TO $clickhouse_user_identifier WITH GRANT OPTION"
+  if ! clickhouse-client --host 127.0.0.1 --port "$CLICKHOUSE_NATIVE_PORT" -q "GRANT CURRENT GRANTS ON *.* TO $clickhouse_user_identifier" >/dev/null 2>&1; then
+    clickhouse-client --host 127.0.0.1 --port "$CLICKHOUSE_NATIVE_PORT" -q "GRANT ALL ON *.* TO $clickhouse_user_identifier WITH GRANT OPTION"
+  fi
 }

 ensure_minio_running() {
@@ -448,6 +506,7 @@ ensure_minio_running() {
 ensure_cloud_dependencies() {
  mkdir -p "$CODEX_SERVICES_ROOT"

+  ensure_migrate_binary
  ensure_postgres_running
  ensure_redis_running
  ensure_clickhouse_running
@@ -18,3 +18,9 @@ pnpm install --frozen-lockfile

 # Keep generated Prisma artifacts aligned after dependency or schema updates.
 pnpm run db:generate
+
+# Keep local databases initialized for worker/web tests during maintenance runs.
+pnpm --filter=shared run db:reset:test
+pnpm --filter=shared run db:reset -f
+SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset
+pnpm --filter=shared run db:seed:examples
@@ -40,3 +40,10 @@ pnpm --filter=shared run db:generate

 # Prisma client generation is needed for typecheck/build tasks in Codex.
 pnpm run db:generate
+
+# Initialize local databases so worker/web tests can run immediately after
+# bootstrap without "table does not exist" failures.
+pnpm --filter=shared run db:reset:test
+pnpm --filter=shared run db:reset -f
+SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset
+pnpm --filter=shared run db:seed:examples
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "3.167.2",
+  "version": "3.168.0",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -1,42 +1,37 @@
 import { getExperimentsAccess } from "@/src/features/experiments/utils/experimentsAccess";

 describe("getExperimentsAccess", () => {
-  it("returns enabled only when cloud, v4 beta, and admin/flag gate all pass", () => {
-    const enabledViaAdmin = getExperimentsAccess({
-      isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: true,
-      isFeatureEnabledOnUser: false,
-    });
-
-    const enabledViaFlag = getExperimentsAccess({
-      isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: false,
-      isFeatureEnabledOnUser: true,
-    });
-
-    expect(enabledViaAdmin.isEnabled).toBe(true);
-    expect(enabledViaFlag.isEnabled).toBe(true);
-  });
-
-  it("returns disabled when v4 beta is off even for eligible users", () => {
+  it("returns enabled when cloud and v4 beta are both enabled", () => {
    const access = getExperimentsAccess({
      isLangfuseCloud: true,
-      isV4BetaEnabled: false,
-      isAdmin: true,
-      isFeatureEnabledOnUser: true,
+      isV4BetaEnabled: true,
+    });
+
+    expect(access.isEnabled).toBe(true);
+  });
+
+  it("returns disabled when not on cloud", () => {
+    const access = getExperimentsAccess({
+      isLangfuseCloud: false,
+      isV4BetaEnabled: true,
    });

    expect(access.isEnabled).toBe(false);
  });

-  it("returns disabled when user is neither admin nor flagged", () => {
+  it("returns disabled when v4 beta is off", () => {
    const access = getExperimentsAccess({
      isLangfuseCloud: true,
-      isV4BetaEnabled: true,
-      isAdmin: false,
-      isFeatureEnabledOnUser: false,
+      isV4BetaEnabled: false,
+    });
+
+    expect(access.isEnabled).toBe(false);
+  });
+
+  it("returns disabled when both cloud and v4 beta are off", () => {
+    const access = getExperimentsAccess({
+      isLangfuseCloud: false,
+      isV4BetaEnabled: false,
    });

    expect(access.isEnabled).toBe(false);
@@ -1885,6 +1885,106 @@ describe("OTel Resource Span Mapping", () => {
      expect(metadataAttributes).not.toHaveProperty("pydantic_ai.all_messages");
    });

+    it("should normalize current Pydantic AI cache usage fields into Langfuse usage details", async () => {
+      const traceId = "abcdef1234567890abcdef1234567891";
+
+      const pydanticAiRootSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "telemetry.sdk.language",
+              value: { stringValue: "python" },
+            },
+            {
+              key: "telemetry.sdk.name",
+              value: { stringValue: "opentelemetry" },
+            },
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "pydantic-ai",
+              version: "1.66.0",
+              attributes: [],
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex"),
+                spanId: Buffer.from("80854cd6bd218bf6", "hex"),
+                name: "pydantic-cache-test",
+                kind: 1,
+                startTimeUnixNano: {
+                  low: 1000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                endTimeUnixNano: {
+                  low: 2000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                attributes: [
+                  {
+                    key: "gen_ai.usage.input_tokens",
+                    value: { intValue: { low: 120, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_tokens",
+                    value: { intValue: { low: 40, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_read.input_tokens",
+                    value: { intValue: { low: 30, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_creation.input_tokens",
+                    value: { intValue: { low: 10, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.input_audio_tokens",
+                    value: { intValue: { low: 5, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.cache_audio_read_tokens",
+                    value: { intValue: { low: 2, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.details.output_audio_tokens",
+                    value: { intValue: { low: 7, high: 0, unsigned: false } },
+                  },
+                ],
+                events: [],
+                status: { code: 1 },
+              },
+            ],
+          },
+        ],
+      };
+
+      const events = await convertOtelSpanToIngestionEvent(
+        pydanticAiRootSpan,
+        new Set(),
+      );
+
+      const observationEvent = events.find((e) => e.type === "span-create");
+
+      expect(observationEvent).toBeDefined();
+      expect(observationEvent?.body.usageDetails.input).toBe(80);
+      expect(observationEvent?.body.usageDetails.output).toBe(40);
+      expect(observationEvent?.body.usageDetails.input_cached_tokens).toBe(30);
+      expect(observationEvent?.body.usageDetails.input_cache_creation).toBe(10);
+      expect(observationEvent?.body.usageDetails.input_audio_tokens).toBe(5);
+      expect(observationEvent?.body.usageDetails.cache_audio_read_tokens).toBe(
+        2,
+      );
+      expect(observationEvent?.body.usageDetails.output_audio_tokens).toBe(7);
+    });
+
    it("should prepend gen_ai.system_instructions to pydantic_ai.all_messages input when system message is absent", async () => {
      const traceId = "9d7aa9a729def1eadc0b2063ca4ebeb4";

@@ -6202,6 +6302,92 @@ describe("OTel Resource Span Mapping", () => {
    });
  });

+  describe("GenAI usage normalization", () => {
+    it("should normalize official gen_ai cache usage into Langfuse canonical keys", async () => {
+      const traceId = "abcdef1234567890abcdef1234567892";
+
+      const genAiSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "gen_ai",
+              version: "1.0.0",
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex"),
+                spanId: Buffer.from("1234567890abcde1", "hex"),
+                name: "normalized-genai-usage",
+                kind: 1,
+                startTimeUnixNano: {
+                  low: 1000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                endTimeUnixNano: {
+                  low: 2000000,
+                  high: 406528574,
+                  unsigned: true,
+                },
+                attributes: [
+                  {
+                    key: "gen_ai.usage.input_tokens",
+                    value: { intValue: { low: 100, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_tokens",
+                    value: { intValue: { low: 40, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.total_tokens",
+                    value: { intValue: { low: 140, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_read.input_tokens",
+                    value: { intValue: { low: 20, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.cache_creation.input_tokens",
+                    value: { intValue: { low: 10, high: 0, unsigned: false } },
+                  },
+                  {
+                    key: "gen_ai.usage.output_audio_tokens",
+                    value: { intValue: { low: 5, high: 0, unsigned: false } },
+                  },
+                ],
+                status: {},
+              },
+            ],
+          },
+        ],
+      };
+
+      const events = await convertOtelSpanToIngestionEvent(
+        genAiSpan,
+        new Set(),
+      );
+      const observationEvent = events.find(
+        (e) => e.type === "generation-create" || e.type === "span-create",
+      );
+
+      expect(observationEvent).toBeDefined();
+      expect(observationEvent?.body.usageDetails.input).toBe(70);
+      expect(observationEvent?.body.usageDetails.output).toBe(40);
+      expect(observationEvent?.body.usageDetails.total).toBe(140);
+      expect(observationEvent?.body.usageDetails.input_cached_tokens).toBe(20);
+      expect(observationEvent?.body.usageDetails.input_cache_creation).toBe(10);
+      expect(observationEvent?.body.usageDetails.output_audio_tokens).toBe(5);
+    });
+  });
+
  describe("Vercel AI SDK Usage details", () => {
    it("should extract usage details from both provider metadata and 'ai.usage'", async () => {
      const traceId = "abcdef1234567890abcdef1234567890";
@@ -7279,4 +7465,73 @@ describe("OTel Resource Span Mapping", () => {
      );
    });
  });
+
+  describe("Prototype pollution protection", () => {
+    const publicKey = "pk-lf-1234567890";
+
+    it("should not pollute Object.prototype via __proto__ in gen_ai.prompt attributes", async () => {
+      const traceId = "abcdef1234567890abcdef1234567890";
+      const spanId = "abcdef1234567890";
+
+      const resourceSpan = {
+        resource: {
+          attributes: [
+            {
+              key: "service.name",
+              value: { stringValue: "test-service" },
+            },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: {
+              name: "langfuse-sdk",
+              version: "2.0.0",
+              attributes: [
+                {
+                  key: "public_key",
+                  value: { stringValue: publicKey },
+                },
+              ],
+            },
+            spans: [
+              {
+                traceId: Buffer.from(traceId, "hex").toJSON(),
+                spanId: Buffer.from(spanId, "hex").toJSON(),
+                name: "pollution-test",
+                kind: 1,
+                startTimeUnixNano: { low: 1000000000, high: 0, unsigned: true },
+                endTimeUnixNano: { low: 2000000000, high: 0, unsigned: true },
+                attributes: [
+                  {
+                    key: "gen_ai.prompt.role",
+                    value: { stringValue: "user" },
+                  },
+                  {
+                    key: "gen_ai.prompt.content",
+                    value: { stringValue: "hello" },
+                  },
+                  {
+                    key: "gen_ai.prompt.__proto__.POLLUTED",
+                    value: { stringValue: "SUCCESS" },
+                  },
+                ],
+                status: {},
+              },
+            ],
+          },
+        ],
+      };
+
+      await convertOtelSpanToIngestionEvent(
+        resourceSpan,
+        new Set([traceId]),
+        publicKey,
+      );
+
+      // Verify Object.prototype was NOT polluted
+      expect(({} as any).POLLUTED).toBeUndefined();
+      expect(Object.prototype.hasOwnProperty("POLLUTED" as any)).toBe(false);
+    });
+  });
 });
@@ -9,11 +9,13 @@ jest.mock("@langfuse/shared/src/server", () => {
 });

 import type { Session } from "next-auth";
-import { LLMAdapter } from "@langfuse/shared";
+import { BEDROCK_USE_DEFAULT_CREDENTIALS, LLMAdapter } from "@langfuse/shared";
+import { env } from "@/src/env.mjs";
 import { prisma } from "@langfuse/shared/src/db";
 import { appRouter } from "@/src/server/api/root";
 import { createInnerTRPCContext } from "@/src/server/api/trpc";
 import { decrypt, encrypt } from "@langfuse/shared/encryption";
+import { AuthMethod } from "@/src/features/llm-api-key/types";
 import {
  createOrgProjectAndApiKey,
  fetchLLMCompletion,
@@ -140,6 +142,49 @@ describe("llmApiKey.all RPC", () => {
    expect(llmApiKeys[0].displaySecretKey).toMatch(/^...[a-zA-Z0-9]{4}$/);
  });

+  it("should create a Bedrock llm api key with a Bedrock API key", async () => {
+    const secret = "bedrock-api-key-1234";
+
+    await caller.llmApiKey.create({
+      projectId,
+      secretKey: JSON.stringify({ apiKey: secret }),
+      provider: "bedrock",
+      adapter: LLMAdapter.Bedrock,
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const llmApiKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: {
+        projectId,
+        provider: "bedrock",
+      },
+    });
+
+    expect(decrypt(llmApiKey.secretKey)).toBe(
+      JSON.stringify({
+        apiKey: secret,
+      }),
+    );
+    expect(llmApiKey.displaySecretKey).toBe("...1234");
+    expect(llmApiKey.config).toEqual({ region: "us-east-1" });
+  });
+
+  it("should reject creating a Bedrock key with invalid secret key JSON", async () => {
+    await expect(
+      caller.llmApiKey.create({
+        projectId,
+        secretKey: JSON.stringify({ unknownField: "value" }),
+        provider: "bedrock",
+        adapter: LLMAdapter.Bedrock,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      }),
+    ).rejects.toThrow("Invalid Bedrock credentials");
+  });
+
  it("should block creating an llm api key with a localhost base URL", async () => {
    await expect(
      caller.llmApiKey.create({
@@ -199,6 +244,87 @@ describe("llmApiKey.all RPC", () => {
    expect(secretKey).toBeUndefined();
  });

+  it("should derive the Bedrock auth method in llmApiKey.all without returning secrets", async () => {
+    await prisma.llmApiKeys.createMany({
+      data: [
+        {
+          projectId,
+          provider: "bedrock-access",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(
+            JSON.stringify({
+              accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+              secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+            }),
+          ),
+          displaySecretKey: "...MPLE",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "bedrock-api",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(
+            JSON.stringify({
+              apiKey: "bedrock-api-key-1234",
+            }),
+          ),
+          displaySecretKey: "...1234",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "bedrock-default",
+          adapter: LLMAdapter.Bedrock,
+          secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
+          displaySecretKey: "Default AWS credentials",
+          customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+          withDefaultModels: false,
+          extraHeaderKeys: [],
+          config: { region: "us-east-1" },
+        },
+        {
+          projectId,
+          provider: "openai",
+          adapter: LLMAdapter.OpenAI,
+          secretKey: encrypt("sk-test"),
+          displaySecretKey: "...test",
+          customModels: [],
+          withDefaultModels: true,
+          extraHeaderKeys: [],
+        },
+      ],
+    });
+
+    const { data: llmApiKeys } = await caller.llmApiKey.all({
+      projectId,
+    });
+
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-access")?.authMethod,
+    ).toBe(AuthMethod.AccessKeys);
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-api")?.authMethod,
+    ).toBe(AuthMethod.ApiKey);
+    expect(
+      llmApiKeys.find((key) => key.provider === "bedrock-default")?.authMethod,
+    ).toBe(AuthMethod.DefaultCredentials);
+    expect(
+      llmApiKeys.find((key) => key.provider === "openai")?.authMethod,
+    ).toBeUndefined();
+    expect(
+      llmApiKeys.every(
+        (key) => key.secretKey === undefined && key.extraHeaders === undefined,
+      ),
+    ).toBe(true);
+  });
+
  it("should require llmApiKeys:create access for testing a new llm api key", async () => {
    const memberCaller = createCallerForProjectRole("MEMBER");

@@ -448,6 +574,287 @@ describe("llmApiKey.all RPC", () => {
    expect(updatedKeys[0].withDefaultModels).toBe(newWithDefaultModels);
  });

+  it("should update a Bedrock Access key auth to a Bedrock API key", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: {
+        projectId,
+        provider,
+      },
+    });
+
+    await caller.llmApiKey.update({
+      id: existingKey.id,
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-5678" }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "eu-west-1" },
+    });
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(
+      JSON.stringify({
+        apiKey: "bedrock-api-key-5678",
+      }),
+    );
+    expect(updatedKey.displaySecretKey).toBe("...5678");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
+  it("should update a Bedrock API key auth to Access keys", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    await caller.llmApiKey.update({
+      id: existingKey.id,
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "eu-west-1" },
+    });
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(
+      JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+    );
+    expect(updatedKey.displaySecretKey).toBe("...EKEY");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
+  it("should update a Bedrock DefaultCredentials key to explicit Access keys", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    try {
+      // Simulate self-hosted to allow default credentials
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.create({
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      });
+
+      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+        where: { projectId, provider },
+      });
+
+      expect(decrypt(existingKey.secretKey)).toBe(
+        BEDROCK_USE_DEFAULT_CREDENTIALS,
+      );
+      expect(existingKey.displaySecretKey).toBe("Default AWS credentials");
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: JSON.stringify({
+          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        }),
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+
+      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+        where: { id: existingKey.id },
+      });
+
+      expect(decrypt(updatedKey.secretKey)).toBe(
+        JSON.stringify({
+          accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+          secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        }),
+      );
+      expect(updatedKey.displaySecretKey).toBe("...EKEY");
+      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+  });
+
+  it("should update a Bedrock DefaultCredentials key to a Bedrock API key", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    try {
+      // Simulate self-hosted to allow default credentials
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.create({
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "us-east-1" },
+      });
+
+      const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+        where: { projectId, provider },
+      });
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+
+      const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+        where: { id: existingKey.id },
+      });
+
+      expect(decrypt(updatedKey.secretKey)).toBe(
+        JSON.stringify({ apiKey: "bedrock-api-key-9999" }),
+      );
+      expect(updatedKey.displaySecretKey).toBe("...9999");
+      expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+  });
+
+  it("should reject updating a Bedrock key back to DefaultCredentials on cloud", async () => {
+    const provider = "bedrock";
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    await expect(
+      caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      }),
+    ).rejects.toThrow(
+      "Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
+    );
+  });
+
+  it("should update a Bedrock Access key auth back to DefaultCredentials (self-hosted)", async () => {
+    const provider = "bedrock";
+    const originalRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+
+    await caller.llmApiKey.create({
+      projectId,
+      provider,
+      adapter: LLMAdapter.Bedrock,
+      secretKey: JSON.stringify({
+        accessKeyId: "AKIAIOSFODNN7EXAMPLE",
+        secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      }),
+      customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+      withDefaultModels: false,
+      config: { region: "us-east-1" },
+    });
+
+    const existingKey = await prisma.llmApiKeys.findFirstOrThrow({
+      where: { projectId, provider },
+    });
+
+    try {
+      // Simulate self-hosted deployment where default credentials are allowed
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+
+      await caller.llmApiKey.update({
+        id: existingKey.id,
+        projectId,
+        provider,
+        adapter: LLMAdapter.Bedrock,
+        secretKey: BEDROCK_USE_DEFAULT_CREDENTIALS,
+        customModels: ["us.anthropic.claude-3-5-sonnet-20240620-v1:0"],
+        withDefaultModels: false,
+        config: { region: "eu-west-1" },
+      });
+    } finally {
+      (env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalRegion;
+    }
+
+    const updatedKey = await prisma.llmApiKeys.findUniqueOrThrow({
+      where: { id: existingKey.id },
+    });
+
+    expect(decrypt(updatedKey.secretKey)).toBe(BEDROCK_USE_DEFAULT_CREDENTIALS);
+    expect(updatedKey.displaySecretKey).toBe("Default AWS credentials");
+    expect(updatedKey.config).toEqual({ region: "eu-west-1" });
+  });
+
  it("should update only the secret key", async () => {
    const secret = "test-secret";
    const provider = "openai";
@@ -577,6 +577,26 @@ describe("/api/public/llm-connections API Endpoints", () => {
      expect(bedrockResponse.body.adapter).toBe(LLMAdapter.Bedrock);
      expect(bedrockResponse.body.config).toEqual({ region: "us-east-1" });

+      const bedrockApiKeyResponse = await makeZodVerifiedAPICall(
+        PutLlmConnectionV1Response,
+        "PUT",
+        "/api/public/llm-connections",
+        {
+          provider: generateUniqueProvider("test-bedrock-api-key"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ apiKey: "bedrock-api-key-1234" }),
+          config: { region: "us-east-1" },
+        },
+        auth,
+        201,
+      );
+      expect(bedrockApiKeyResponse.status).toBe(201);
+      expect(bedrockApiKeyResponse.body.adapter).toBe(LLMAdapter.Bedrock);
+      expect(bedrockApiKeyResponse.body.displaySecretKey).toBe("...1234");
+      expect(bedrockApiKeyResponse.body.config).toEqual({
+        region: "us-east-1",
+      });
+
      // VertexAI works with or without config
      const vertexResponse = await makeZodVerifiedAPICall(
        PutLlmConnectionV1Response,
@@ -904,6 +924,41 @@ describe("/api/public/llm-connections API Endpoints", () => {
        expect(dbConnection?.config).toEqual({ region: "us-east-1" });
      });

+      it("should create Bedrock connection with a Bedrock API key", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-api-key-config-test"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ apiKey: "bedrock-api-key-9876" }),
+          config: {
+            region: "us-west-2",
+          },
+        };
+
+        const response = await makeZodVerifiedAPICall(
+          PutLlmConnectionV1Response,
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+          201,
+        );
+
+        expect(response.status).toBe(201);
+        expect(response.body.displaySecretKey).toBe("...9876");
+        expect(response.body.config).toEqual({ region: "us-west-2" });
+
+        const dbConnection = await prisma.llmApiKeys.findUnique({
+          where: {
+            projectId_provider: {
+              projectId,
+              provider: createData.provider,
+            },
+          },
+        });
+
+        expect(dbConnection?.displaySecretKey).toBe("...9876");
+      });
+
      it("should reject Bedrock connection without config", async () => {
        const createData = {
          provider: generateUniqueProvider("bedrock-no-config"),
@@ -954,6 +1009,48 @@ describe("/api/public/llm-connections API Endpoints", () => {
        );
      });

+      it("should reject Bedrock connection with default credentials sentinel on cloud", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-default-creds"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: "__BEDROCK_DEFAULT_CREDENTIALS__",
+          config: { region: "us-east-1" },
+        };
+
+        const response = await makeAPICall(
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+        );
+
+        expect(response.status).toBe(400);
+        expect(JSON.stringify(response.body)).toContain(
+          "Default AWS credentials are only allowed for Bedrock in self-hosted deployments",
+        );
+      });
+
+      it("should reject Bedrock connection with invalid credential JSON", async () => {
+        const createData = {
+          provider: generateUniqueProvider("bedrock-invalid-creds"),
+          adapter: LLMAdapter.Bedrock,
+          secretKey: JSON.stringify({ unknownField: "value" }),
+          config: { region: "us-east-1" },
+        };
+
+        const response = await makeAPICall(
+          "PUT",
+          "/api/public/llm-connections",
+          createData,
+          auth,
+        );
+
+        expect(response.status).toBe(400);
+        expect(JSON.stringify(response.body)).toContain(
+          "Invalid Bedrock credentials",
+        );
+      });
+
      it("should create VertexAI connection with location config", async () => {
        const createData = {
          provider: generateUniqueProvider("vertexai-config-test"),
@@ -39,6 +39,34 @@ describe("Clickhouse Events Repository Test", () => {
  });

  maybe("getObservationsWithModelDataFromEventsTable", () => {
+    it("should return trace tags for events table observations", async () => {
+      const traceId = randomUUID();
+      const observationId = randomUUID();
+
+      await createEventsCh([
+        createEvent({
+          id: observationId,
+          span_id: observationId,
+          project_id: projectId,
+          trace_id: traceId,
+          type: "SPAN",
+          name: "tagged-event",
+          tags: ["chat", "prod"],
+        }),
+      ]);
+
+      const result = await getObservationsWithModelDataFromEventsTable({
+        projectId,
+        filter: [idFilter(observationId)],
+        limit: 1000,
+        offset: 0,
+      });
+
+      const observation = result.find((o) => o.id === observationId);
+      expect(observation).toBeDefined();
+      expect(observation?.traceTags).toEqual(["chat", "prod"]);
+    });
+
    it("should return observations with model data", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
@@ -1219,6 +1247,37 @@ describe("Clickhouse Events Repository Test", () => {
  });

  maybe("getObservationByIdFromEventsTable", () => {
+    it("should return trace-level fields for event observations", async () => {
+      const traceId = randomUUID();
+      const spanId = randomUUID();
+
+      await createEventsCh([
+        createEvent({
+          id: spanId,
+          span_id: spanId,
+          project_id: projectId,
+          trace_id: traceId,
+          type: "SPAN",
+          name: "test-trace-fields-byid",
+          trace_name: "trace-with-tags",
+          tags: ["chat", "prod"],
+          user_id: "user-123",
+          session_id: "session-123",
+        }),
+      ]);
+
+      const observation = await getObservationByIdFromEventsTable({
+        id: spanId,
+        projectId,
+      });
+
+      expect(observation).toBeDefined();
+      expect(observation?.traceName).toBe("trace-with-tags");
+      expect(observation?.traceTags).toEqual(["chat", "prod"]);
+      expect(observation?.userId).toBe("user-123");
+      expect(observation?.sessionId).toBe("session-123");
+    });
+
    it("should return observation by id with input and output", async () => {
      const traceId = randomUUID();
      const generationId = randomUUID();
@@ -333,7 +333,7 @@ describe("Clickhouse Experiment Repository Test", () => {
      const now = new Date().getTime();

      // Trace 1: Multiple events with timing data to test latency calculation
-      // Latency should be: earliest start_time to latest end_time
+      // Latency is calculated from ROOT SPAN only (span_id = experiment_item_root_span_id)
      const trace1Id = randomUUID();
      const rootSpanId = randomUUID();
      const event1 = createEvent({
@@ -351,8 +351,8 @@ describe("Clickhouse Experiment Repository Test", () => {
        experiment_item_id: randomUUID(),
        experiment_item_version: null,
        experiment_item_root_span_id: rootSpanId,
-        start_time: (now - 3500) * 1000, // Earliest start: now - 3500ms (convert to microseconds)
-        end_time: (now - 2500) * 1000, // End: now - 2500ms (convert to microseconds)
+        start_time: (now - 3500) * 1000, // Root span start (convert to microseconds)
+        end_time: (now - 2500) * 1000, // Root span end: latency = 1000ms (convert to microseconds)
      });

      const childSpan1Id = randomUUID();
@@ -394,7 +394,7 @@ describe("Clickhouse Experiment Repository Test", () => {
        experiment_item_version: null,
        experiment_item_root_span_id: event1.experiment_item_root_span_id,
        start_time: (now - 3000) * 1000,
-        end_time: (now - 1500) * 1000, // Latest end: now - 1500ms (convert to microseconds)
+        end_time: (now - 1500) * 1000, // Child spans are NOT included in latency calculation
      });

      // Trace 2: Single event with known latency (1000ms)
@@ -415,8 +415,8 @@ describe("Clickhouse Experiment Repository Test", () => {
        experiment_item_id: randomUUID(),
        experiment_item_version: null,
        experiment_item_root_span_id: rootSpan2Id,
-        start_time: (now - 2500) * 1000, // Start: now - 2500ms (convert to microseconds)
-        end_time: (now - 1500) * 1000, // End: now - 1500ms (latency = 1000ms, convert to microseconds)
+        start_time: (now - 2500) * 1000, // Root span start (convert to microseconds)
+        end_time: (now - 1500) * 1000, // Root span end: latency = 1000ms (convert to microseconds)
      });

      await createEventsCh([event1, event2, event3, event4]);
@@ -433,7 +433,8 @@ describe("Clickhouse Experiment Repository Test", () => {
      expect(metric.latencyAvg).toBeDefined();
      expect(typeof metric.latencyAvg).toBe("number");

-      expect(metric.latencyAvg).toBeCloseTo(1500, -1); // Within 10ms tolerance
+      // Latency avg = (1000ms + 1000ms) / 2 = 1000ms (only root spans count)
+      expect(metric.latencyAvg).toBeCloseTo(1000, -1); // Within 10ms tolerance
    });

    it("should handle cost calculations correctly", async () => {
@@ -17,7 +17,7 @@ describe("shouldUseWidgetSSE", () => {
  it("should enable SSE on the v4 beta v2 path", () => {
    expect(
      shouldUseWidgetSSE({
-        isV4BetaEnabled: true,
+        isV4Enabled: true,
        version: "v2",
      }),
    ).toBe(true);
@@ -26,7 +26,7 @@ describe("shouldUseWidgetSSE", () => {
  it("should disable SSE when v4 beta is off", () => {
    expect(
      shouldUseWidgetSSE({
-        isV4BetaEnabled: false,
+        isV4Enabled: false,
        version: "v2",
      }),
    ).toBe(false);
@@ -35,7 +35,7 @@ describe("shouldUseWidgetSSE", () => {
  it("should disable SSE for non-v4 query versions", () => {
    expect(
      shouldUseWidgetSSE({
-        isV4BetaEnabled: true,
+        isV4Enabled: true,
        version: "v1",
      }),
    ).toBe(false);
@@ -142,6 +142,7 @@ export function useMessageSearch() {
    openSearch: controller.openSearch,
    closeSearch: controller.closeSearch,
    setQueryInput: controller.setQueryInput,
+    blurQueryInput: controller.blurQueryInput,
    nextMatch: controller.nextMatch,
    previousMatch: controller.previousMatch,
  };
@@ -123,4 +123,60 @@ describe("message search controller", () => {
      expect.objectContaining({ from: 5, to: 6 }),
    ]);
  });
+
+  it("preserves leading and trailing whitespace in literal queries", () => {
+    const controller = createMessageSearchController(["page-1"]);
+
+    controller.registerPageMessages("page-1", [
+      {
+        id: "message-1",
+        type: ChatMessageType.System,
+        role: ChatMessageRole.System,
+        content: " foo foo ",
+      },
+    ]);
+
+    expect(commitQuery(controller, " foo")).toEqual([
+      expect.objectContaining({ from: 0, to: 4 }),
+      expect.objectContaining({ from: 4, to: 8 }),
+    ]);
+    expect(controller.getSnapshot().query).toBe(" foo");
+
+    controller.setQueryInput("foo ");
+    controller.nextMatch();
+
+    expect(controller.getSnapshot().query).toBe("foo ");
+    expect(controller.getSnapshot().matches).toEqual([
+      expect.objectContaining({ from: 1, to: 5 }),
+      expect.objectContaining({ from: 5, to: 9 }),
+    ]);
+  });
+
+  it("clears whitespace-only input on blur", () => {
+    const controller = createMessageSearchController(["page-1"]);
+
+    controller.registerPageMessages("page-1", [
+      {
+        id: "message-1",
+        type: ChatMessageType.System,
+        role: ChatMessageRole.System,
+        content: "a   b",
+      },
+    ]);
+
+    controller.setQueryInput("   ");
+    jest.runAllTimers();
+
+    expect(controller.getSnapshot().queryInput).toBe("   ");
+    expect(controller.getSnapshot().query).toBe("   ");
+    expect(controller.getSnapshot().matches).toEqual([
+      expect.objectContaining({ from: 1, to: 4 }),
+    ]);
+
+    controller.blurQueryInput();
+
+    expect(controller.getSnapshot().queryInput).toBe("");
+    expect(controller.getSnapshot().query).toBe("");
+    expect(controller.getSnapshot().matches).toEqual([]);
+  });
 });
@@ -67,6 +67,7 @@ export type MessageSearchController = {
  openSearch: () => void;
  closeSearch: () => void;
  setQueryInput: (value: string) => void;
+  blurQueryInput: () => void;
  nextMatch: () => void;
  previousMatch: () => void;
  setPageIds: (pageIds: string[]) => void;
@@ -376,7 +377,7 @@ export function createMessageSearchController(

    clearPendingQueryTimeout();

-    const queryChanged = commitSearchQuery(state.queryInput.trim());
+    const queryChanged = commitSearchQuery(state.queryInput);
    if (queryChanged) {
      emit();
    }
@@ -465,8 +466,7 @@ export function createMessageSearchController(
      state.queryInput = value;
      clearPendingQueryTimeout();

-      const nextSearchQuery = value.trim();
-      if (nextSearchQuery === "") {
+      if (value === "") {
        commitSearchQuery("");
        emit();
        return;
@@ -474,20 +474,38 @@ export function createMessageSearchController(

      emit();

-      if (nextSearchQuery === state.searchQuery) {
+      if (value === state.searchQuery) {
        return;
      }

      pendingQueryTimeout = window.setTimeout(() => {
        pendingQueryTimeout = null;

-        const queryChanged = commitSearchQuery(nextSearchQuery);
+        const queryChanged = commitSearchQuery(value);
        if (queryChanged) {
          emit();
        }
      }, SEARCH_INPUT_DEBOUNCE_MS);
    },

+    blurQueryInput() {
+      if (state.queryInput.trim() !== "") {
+        return;
+      }
+
+      clearPendingQueryTimeout();
+
+      const queryChanged = commitSearchQuery("");
+      const inputChanged = state.queryInput !== "";
+      if (inputChanged) {
+        state.queryInput = "";
+      }
+
+      if (queryChanged || inputChanged) {
+        emit();
+      }
+    },
+
    nextMatch() {
      moveActiveMatch(1);
    },
@@ -25,6 +25,7 @@ export function MessageSearchToolbar({ className }: { className?: string }) {
    openSearch,
    closeSearch,
    setQueryInput,
+    blurQueryInput,
    nextMatch,
    previousMatch,
  } = useMessageSearch();
@@ -71,6 +72,7 @@ export function MessageSearchToolbar({ className }: { className?: string }) {
        ref={inputRef}
        value={queryInput}
        onChange={(event) => setQueryInput(event.target.value)}
+        onBlur={blurQueryInput}
        placeholder="Find in messages"
        className="h-6 min-w-40 border-0 px-1 text-xs shadow-none focus-visible:ring-0 sm:min-w-56"
        onKeyDown={(event) => {
@@ -6,7 +6,12 @@ import CodeMirror, {
  ViewPlugin,
  type ViewUpdate,
 } from "@uiw/react-codemirror";
-import { RangeSetBuilder, StateEffect, StateField } from "@codemirror/state";
+import {
+  EditorState,
+  RangeSetBuilder,
+  StateEffect,
+  StateField,
+} from "@codemirror/state";
 import { SearchQuery, search, setSearchQuery } from "@codemirror/search";
 import { json, jsonParseLinter } from "@codemirror/lang-json";
 import { linter, type Diagnostic } from "@codemirror/lint";
@@ -428,6 +433,10 @@ export function CodeMirrorEditor({
      }}
      lang={mode === "json" ? "json" : undefined}
      extensions={[
+        // Block document changes (including paste) when not editable; the
+        // `editable` DOM facet alone does not always prevent paste (see CM6
+        // EditorState.readOnly vs EditorView.editable).
+        ...(!editable ? [EditorState.readOnly.of(true)] : []),
        searchHighlightingSupport,
        search(),
        // RTL/bidi support - must be early for proper line decoration
@@ -8,15 +8,10 @@ import type { NavigationFilterContext } from "./navigationFilters.types";
 import { hasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { hasOrganizationAccess } from "@/src/features/rbac/utils/checkOrganizationAccess";
 import type { User } from "next-auth";
-import type { Flag } from "@/src/features/feature-flags/types";
-import { getExperimentsAccess } from "@/src/features/experiments/utils/experimentsAccess";

 /** Organization type from user session (can be null when not in project/org context) */
 type Organization = User["organizations"][number] | null | undefined;

-// Admin-only flags that don't respect experimental features
-const adminOnlyFlags: Flag[] = ["experimentsV4Enabled"];
-
 /**
 * Individual filter functions - each handles one concern
 * Exported for testing and composition
@@ -73,32 +68,20 @@ export const filters = {
   * - Experimental features enabled
   * - User is cloud admin
   * - User has specific feature flag
-   * - For v4Beta: show to all cloud users and keep it visible for opted-in users outside cloud
   */
  featureFlags: (route: Route, ctx: NavigationFilterContext): Route | null => {
    if (route.featureFlag === undefined) return route;

-    if (route.featureFlag && adminOnlyFlags.includes(route.featureFlag)) {
-      const access = getExperimentsAccess({
-        isLangfuseCloud: ctx.isLangfuseCloud,
-        isV4BetaEnabled: ctx.session?.user?.v4BetaEnabled === true,
-        isAdmin: ctx.cloudAdmin,
-        isFeatureEnabledOnUser:
-          ctx.session?.user?.featureFlags?.[route.featureFlag] === true,
-      });
-
-      return access.isEnabled ? route : null;
+    if (route.featureFlag === "experimentsV4Enabled") {
+      return ctx.isLangfuseCloud && ctx.session?.user?.v4BetaEnabled === true
+        ? route
+        : null;
    }

    if (route.featureFlag === "v4BetaToggleVisible") {
-      const hasOptedIn = ctx.session?.user?.v4BetaEnabled === true;
+      const canToggleV4 = ctx.session?.user?.canToggleV4 === true;

-      return ctx.isLangfuseCloud ||
-        ctx.enableExperimentalFeatures ||
-        ctx.cloudAdmin ||
-        hasOptedIn
-        ? route
-        : null;
+      return canToggleV4 && ctx.isLangfuseCloud ? route : null;
    }

    const hasFlag =
@@ -37,25 +37,21 @@ const PaymentBanner = dynamic(
  },
 );

-const V4BetaEnabledBanner = dynamic(
+const V4EnabledBanner = dynamic(
  () =>
-    import("@/src/features/events/components/V4BetaEnabledBanner").then(
-      (mod) => ({
-        default: mod.V4BetaEnabledBanner,
-      }),
-    ),
+    import("@/src/features/events/components/V4EnabledBanner").then((mod) => ({
+      default: mod.V4EnabledBanner,
+    })),
  {
    ssr: false,
  },
 );

-const V4BetaPromoBanner = dynamic(
+const V4PromoBanner = dynamic(
  () =>
-    import("@/src/features/events/components/V4BetaPromoBanner").then(
-      (mod) => ({
-        default: mod.V4BetaPromoBanner,
-      }),
-    ),
+    import("@/src/features/events/components/V4PromoBanner").then((mod) => ({
+      default: mod.V4PromoBanner,
+    })),
  {
    ssr: false,
  },
@@ -140,8 +136,8 @@ export function AuthenticatedLayout({
        <SidebarProvider>
          <div className="flex h-dvh w-full flex-col">
            <PaymentBanner />
-            <V4BetaEnabledBanner />
-            <V4BetaPromoBanner />
+            <V4EnabledBanner />
+            <V4PromoBanner />
            <div className="pt-banner-offset flex min-h-0 flex-1">
              <AppSidebar
                navItems={navigation.mainNavigation}
@@ -26,7 +26,7 @@ import { type User } from "next-auth";
 import { type OrganizationScope } from "@/src/features/rbac/constants/organizationAccessRights";
 import { SupportButton } from "@/src/components/nav/support-button";
 import { BookACallButton } from "@/src/components/nav/book-a-call-button";
-import { V4BetaSidebarToggle } from "@/src/features/events/components/V4BetaSidebarToggle";
+import { V4SidebarToggle } from "@/src/features/events/components/V4SidebarToggle";
 import { SidebarMenuButton } from "@/src/components/ui/sidebar";
 import { useCommandMenu } from "@/src/features/command-k-menu/CommandMenuProvider";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -209,7 +209,7 @@ export const ROUTES: Route[] = [
    pathname: "",
    section: RouteSection.Secondary,
    featureFlag: "v4BetaToggleVisible",
-    menuNode: <V4BetaSidebarToggle />,
+    menuNode: <V4SidebarToggle />,
  },
  {
    title: "Settings",
@@ -1,6 +1,5 @@
 import React from "react";
-import { SplashScreen } from "@/src/components/ui/splash-screen";
-import { TracingSetup } from "@/src/pages/project/[projectId]/traces/setup";
+import { TracesSetupOnboardingCard } from "@/src/features/setup/components/TracesSetupOnboardingCard";

 interface TracesOnboardingProps {
  projectId: string;
@@ -8,15 +7,8 @@ interface TracesOnboardingProps {

 export function TracesOnboarding({ projectId }: TracesOnboardingProps) {
  return (
-    <SplashScreen
-      title="You don't have any traces yet"
-      description="Traces show you how your LLM calls behave in your application: what they cost, how they perform, and where things go wrong. It's the first step towards improving the behavior of your app."
-      videoSrc="https://static.langfuse.com/prod-assets/onboarding/tracing-overview-v1.mp4"
-    >
-      <div className="mt-8">
-        <h3 className="mb-8 text-2xl font-semibold">Get started</h3>
-        <TracingSetup projectId={projectId} hasTracingConfigured={false} />
-      </div>
-    </SplashScreen>
+    <div className="space-y-10">
+      <TracesSetupOnboardingCard projectId={projectId} />
+    </div>
  );
 }
@@ -33,6 +33,7 @@ export const TraceEventsRow = React.memo(
    traceCommentCounts,
    showCorrections,
    filterState,
+    hideTracePanel = false,
  }: {
    trace: RouterOutputs["sessions"]["tracesFromEvents"][number];
    projectId: string;
@@ -41,6 +42,7 @@ export const TraceEventsRow = React.memo(
    traceCommentCounts: Map<string, number> | undefined;
    showCorrections: boolean;
    filterState: FilterState;
+    hideTracePanel?: boolean;
  }) => {
    const observationsQuery =
      api.sessions.observationsForTraceFromEvents.useQuery(
@@ -59,10 +61,20 @@ export const TraceEventsRow = React.memo(

    return (
      <Card className="border-border shadow-none">
-        <div className="grid md:grid-cols-[1fr_1px_358px] lg:grid-cols-[1fr_1px_30rem]">
+        <div
+          className={
+            hideTracePanel
+              ? "grid"
+              : "grid md:grid-cols-[1fr_1px_358px] lg:grid-cols-[1fr_1px_30rem]"
+          }
+        >
          <div className="overflow-hidden py-4 pr-4 pl-4">
            {observationsQuery.isLoading ? (
              <JsonSkeleton className="h-full w-full" numRows={8} />
+            ) : observationsQuery.isError ? (
+              <div className="text-destructive p-2 text-xs">
+                Failed to load observations.
+              </div>
            ) : observationsQuery.data && observationsQuery.data.length > 0 ? (
              <div className="flex flex-col gap-4">
                {observationsQuery.data.map((observation) => (
@@ -110,78 +122,82 @@ export const TraceEventsRow = React.memo(
              </div>
            )}
          </div>
-          <div className="bg-border hidden md:block"></div>
-          <div className="flex flex-col border-t py-4 pr-4 pl-4 md:border-0">
-            <div className="mb-4 flex flex-col gap-2">
-              <Link
-                href={`/project/${projectId}/traces/${trace.id}`}
-                className="hover:bg-accent flex items-start gap-2 rounded-lg border p-2 transition-colors"
-                onClick={(e) => {
-                  if (!e.metaKey && !e.ctrlKey && !e.shiftKey) {
-                    e.preventDefault();
-                    openPeek(trace.id, trace);
-                  }
-                }}
-              >
-                <ItemBadge type="TRACE" isSmall />
-                <div className="flex flex-col">
-                  <span className="text-xs font-medium">
-                    {trace.name ?? "Trace"} ({trace.id})&nbsp;↗
-                  </span>
-                  <span className="text-muted-foreground text-xs">
-                    {trace.timestamp.toLocaleString()}
-                  </span>
+          {!hideTracePanel && (
+            <>
+              <div className="bg-border hidden md:block"></div>
+              <div className="flex flex-col border-t py-4 pr-4 pl-4 md:border-0">
+                <div className="mb-4 flex flex-col gap-2">
+                  <Link
+                    href={`/project/${projectId}/traces/${trace.id}`}
+                    className="hover:bg-accent flex items-start gap-2 rounded-lg border p-2 transition-colors"
+                    onClick={(e) => {
+                      if (!e.metaKey && !e.ctrlKey && !e.shiftKey) {
+                        e.preventDefault();
+                        openPeek(trace.id, trace);
+                      }
+                    }}
+                  >
+                    <ItemBadge type="TRACE" isSmall />
+                    <div className="flex flex-col">
+                      <span className="text-xs font-medium">
+                        {trace.name ?? "Trace"} ({trace.id})&nbsp;↗
+                      </span>
+                      <span className="text-muted-foreground text-xs">
+                        {trace.timestamp.toLocaleString()}
+                      </span>
+                    </div>
+                  </Link>
+                  <div className="flex flex-wrap gap-2">
+                    <NewDatasetItemFromTraceId
+                      projectId={projectId}
+                      traceId={trace.id}
+                      timestamp={new Date(trace.timestamp)}
+                      buttonVariant="outline"
+                    />
+                    <div className="flex items-start">
+                      <AnnotateDrawer
+                        key={"annotation-drawer" + trace.id}
+                        projectId={projectId}
+                        scoreTarget={{
+                          type: "trace",
+                          traceId: trace.id,
+                        }}
+                        scores={trace.scores}
+                        buttonVariant="outline"
+                        analyticsData={{
+                          type: "trace",
+                          source: "SessionDetail",
+                        }}
+                        scoreMetadata={{
+                          projectId: projectId,
+                          environment: trace.environment ?? undefined,
+                        }}
+                      />
+                      <CreateNewAnnotationQueueItem
+                        projectId={projectId}
+                        objectId={trace.id}
+                        objectType={AnnotationQueueObjectType.TRACE}
+                        variant="outline"
+                      />
+                    </div>
+                    <CommentDrawerButton
+                      projectId={projectId}
+                      variant="outline"
+                      objectId={trace.id}
+                      objectType="TRACE"
+                      count={getNumberFromMap(traceCommentCounts, trace.id)}
+                    />
+                  </div>
                </div>
-              </Link>
-              <div className="flex flex-wrap gap-2">
-                <NewDatasetItemFromTraceId
-                  projectId={projectId}
-                  traceId={trace.id}
-                  timestamp={new Date(trace.timestamp)}
-                  buttonVariant="outline"
-                />
-                <div className="flex items-start">
-                  <AnnotateDrawer
-                    key={"annotation-drawer" + trace.id}
-                    projectId={projectId}
-                    scoreTarget={{
-                      type: "trace",
-                      traceId: trace.id,
-                    }}
-                    scores={trace.scores}
-                    buttonVariant="outline"
-                    analyticsData={{
-                      type: "trace",
-                      source: "SessionDetail",
-                    }}
-                    scoreMetadata={{
-                      projectId: projectId,
-                      environment: trace.environment ?? undefined,
-                    }}
-                  />
-                  <CreateNewAnnotationQueueItem
-                    projectId={projectId}
-                    objectId={trace.id}
-                    objectType={AnnotationQueueObjectType.TRACE}
-                    variant="outline"
-                  />
+                <div className="flex-1">
+                  <p className="mb-1 font-medium">Scores</p>
+                  <div className="flex flex-wrap content-start items-start gap-1">
+                    <GroupedScoreBadges scores={trace.scores} />
+                  </div>
                </div>
-                <CommentDrawerButton
-                  projectId={projectId}
-                  variant="outline"
-                  objectId={trace.id}
-                  objectType="TRACE"
-                  count={getNumberFromMap(traceCommentCounts, trace.id)}
-                />
              </div>
-            </div>
-            <div className="flex-1">
-              <p className="mb-1 font-medium">Scores</p>
-              <div className="flex flex-wrap content-start items-start gap-1">
-                <GroupedScoreBadges scores={trace.scores} />
-              </div>
-            </div>
-          </div>
+            </>
+          )}
        </div>
      </Card>
    );
@@ -201,6 +217,7 @@ export const LazyTraceEventsRow = React.forwardRef<
    traceCommentCounts: Map<string, number> | undefined;
    showCorrections: boolean;
    filterState: FilterState;
+    hideTracePanel?: boolean;
    onLoad?: (index: number) => void;
  }
 >((props, measureRef) => {
@@ -6,7 +6,6 @@ import {
  type TableViewPresetState,
  type ColumnDefinition,
 } from "@langfuse/shared";
-import { type DefaultViewScope } from "@langfuse/shared/src/server";
 import { useRouter } from "next/router";
 import { useEffect, useCallback, useState, useRef } from "react";
 import { type VisibilityState } from "@tanstack/react-table";
@@ -31,6 +30,7 @@ interface TableStateUpdaters {
 interface UseTableStateProps {
  tableName: TableViewPresetTableName;
  projectId: string;
+  viewPersistenceKey?: string;
  stateUpdaters: TableStateUpdaters;
  validationContext?: {
    columns?: LangfuseColumnDef<any, any>[];
@@ -46,6 +46,7 @@ interface UseTableStateProps {
 export function useTableViewManager({
  projectId,
  tableName,
+  viewPersistenceKey,
  stateUpdaters,
  validationContext = {},
  currentFilterState,
@@ -58,9 +59,14 @@ export function useTableViewManager({
  const capture = usePostHogClientCapture();
  const pendingFiltersRef = useRef<FilterState | null>(null);
  const pendingFiltersPreviousStateRef = useRef<FilterState | null>(null);
+  // Session storage needs a mode-specific key because the same tableName can be
+  // rendered by different route variants (for example legacy vs v4 pages) with
+  // distinct saved-view IDs. Reusing tableName would restore stale IDs across
+  // modes and boot the user into an incompatible saved view.
+  const resolvedViewPersistenceKey = viewPersistenceKey ?? tableName;

  const [storedViewId, setStoredViewId] = useSessionStorage<string | null>(
-    `${tableName}-${projectId}-viewId`,
+    `${resolvedViewPersistenceKey}-${projectId}-viewId`,
    null,
  );
  const [selectedViewIdParam, setSelectedViewId] = useQueryParam(
@@ -288,6 +294,10 @@ export function useTableViewManager({
    if (isInitializedRef.current) return;
    if (selectedViewIdRef.current !== requestedViewId) return;
    if (selectedViewData.id !== requestedViewId) return;
+    if (selectedViewData.tableName !== tableName) {
+      handleSetViewId(null);
+      return;
+    }

    // Track permalink visit
    capture("saved_views:permalink_visit", {
@@ -304,6 +314,7 @@ export function useTableViewManager({
    isSelectedViewSuccess,
    selectedViewData,
    selectedViewId,
+    handleSetViewId,
    capture,
    tableName,
    applyViewState,
@@ -368,6 +379,6 @@ export function useTableViewManager({
    applyViewState,
    handleSetViewId,
    selectedViewId,
-    defaultViewScope: resolvedDefault?.scope as DefaultViewScope | null,
+    defaultViewScope: resolvedDefault?.scope ?? null,
  };
 }
@@ -12,8 +12,9 @@ import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState
 import { usePaginationState } from "@/src/hooks/usePaginationState";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
-  observationFilterConfig,
+  getObservationsFilterConfig,
  OBSERVATION_COLUMN_TO_BACKEND_KEY,
+  type ObservationsOmittableFilterColumn,
 } from "@/src/features/filters/config/observations-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { transformFiltersForBackend } from "@/src/features/filters/lib/filter-transform";
@@ -134,9 +135,10 @@ export type ObservationsTableProps = {
  promptName?: string;
  promptVersion?: number;
  modelId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: ObservationsOmittableFilterColumn[];
  // External control props for embedded preview tables
  hideControls?: boolean;
+  viewPersistenceKey?: string;
  externalFilterState?: FilterState;
  externalDateRange?: TableDateRange;
  limitRows?: number;
@@ -147,11 +149,17 @@ export default function ObservationsTable({
  promptName,
  promptVersion,
  modelId,
+  omittedFilter = [],
  hideControls = false,
+  viewPersistenceKey,
  externalFilterState,
  externalDateRange,
  limitRows,
 }: ObservationsTableProps) {
+  const observationsFilterConfig = useMemo(
+    () => getObservationsFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
  const router = useRouter();
  const { viewId } = router.query;
  const utils = api.useUtils();
@@ -435,7 +443,7 @@ export default function ObservationsTable({
  }, [environmentFilterOptions.data, filterOptions.data]);

  const queryFilter = useSidebarFilterState(
-    observationFilterConfig,
+    observationsFilterConfig,
    newFilterOptions,
    {
      loading: filterOptions.isPending || environmentFilterOptions.isPending,
@@ -468,7 +476,7 @@ export default function ObservationsTable({
  const backendFilterState = transformFiltersForBackend(
    filterState,
    OBSERVATION_COLUMN_TO_BACKEND_KEY,
-    observationFilterConfig.columnDefinitions,
+    observationsFilterConfig.columnDefinitions,
  );

  const getCountPayload = {
@@ -1215,6 +1223,7 @@ export default function ObservationsTable({
  const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
    tableName: TableViewPresetTableName.Observations,
    projectId,
+    viewPersistenceKey,
    stateUpdaters: {
      setOrderBy: setOrderByState,
      setFilters: setFiltersWrapper,
@@ -1224,7 +1233,7 @@ export default function ObservationsTable({
    },
    validationContext: {
      columns,
-      filterColumnDefinition: observationFilterConfig.columnDefinitions,
+      filterColumnDefinition: observationsFilterConfig.columnDefinitions,
    },
    currentFilterState: queryFilter.explicitFilterState,
    disabled: hideControls,
@@ -1287,7 +1296,7 @@ export default function ObservationsTable({
  }, [generations]);

  return (
-    <DataTableControlsProvider tableName={observationFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={observationsFilterConfig.tableName}>
      <div className="flex h-full w-full flex-col">
        {/* Toolbar spanning full width */}
        {!hideControls && (
@@ -12,8 +12,9 @@ import { TokenUsageBadge } from "@/src/components/token-usage-badge";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
-  sessionFilterConfig,
+  getSessionFilterConfig,
  SESSION_COLUMN_TO_BACKEND_KEY,
+  type SessionOmittableFilterColumn,
 } from "@/src/features/filters/config/sessions-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { transformFiltersForBackend } from "@/src/features/filters/lib/filter-transform";
@@ -78,7 +79,7 @@ export type SessionTableRow = {
 export type SessionTableProps = {
  projectId: string;
  userId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: SessionOmittableFilterColumn[];
  isBetaEnabled?: boolean;
 };

@@ -88,6 +89,10 @@ export default function SessionsTable({
  omittedFilter = [],
  isBetaEnabled = false,
 }: SessionTableProps) {
+  const sessionsFilterConfig = useMemo(
+    () => getSessionFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
  const { setDetailPageList } = useDetailPageLists();
  const { timeRange, setTimeRange } = useTableDateRange(projectId);

@@ -239,7 +244,7 @@ export default function SessionsTable({
  }, [environmentOptions, filterOptions.data]);

  const queryFilter = useSidebarFilterState(
-    sessionFilterConfig,
+    sessionsFilterConfig,
    newFilterOptions,
    {
      loading: filterOptions.isPending || environmentFilterOptions.isPending,
@@ -266,7 +271,7 @@ export default function SessionsTable({
  const backendFilterState = transformFiltersForBackend(
    combinedFilterState,
    SESSION_COLUMN_TO_BACKEND_KEY,
-    sessionFilterConfig.columnDefinitions,
+    sessionsFilterConfig.columnDefinitions,
  );

  const payloadCount = {
@@ -528,7 +533,7 @@ export default function SessionsTable({
    },
    {
      accessorKey: "userIds",
-      enableColumnFilter: !omittedFilter.find((f) => f === "userIds"),
+      enableColumnFilter: !omittedFilter.includes("userIds"),
      id: "userIds",
      header: "User IDs",
      size: 200,
@@ -754,13 +759,13 @@ export default function SessionsTable({
    },
    validationContext: {
      columns,
-      filterColumnDefinition: sessionFilterConfig.columnDefinitions,
+      filterColumnDefinition: sessionsFilterConfig.columnDefinitions,
    },
    currentFilterState: queryFilter.explicitFilterState,
  });

  return (
-    <DataTableControlsProvider tableName={sessionFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={sessionsFilterConfig.tableName}>
      <div className="flex h-full w-full flex-col">
        {/* Toolbar spanning full width */}
        <DataTableToolbar
@@ -73,7 +73,10 @@ import {
 import { Button } from "@/src/components/ui/button";
 import TableIdOrName from "@/src/components/table/table-id";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
-import { traceFilterConfig } from "@/src/features/filters/config/traces-config";
+import {
+  getTraceFilterConfig,
+  type TraceOmittableFilterColumn,
+} from "@/src/features/filters/config/traces-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { PeekViewTraceDetail } from "@/src/components/table/peek/peek-trace-detail";
 import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNavigation";
@@ -135,8 +138,9 @@ export type TracesTableRow = {
 export type TracesTableProps = {
  projectId: string;
  userId?: string;
-  omittedFilter?: string[];
+  omittedFilter?: TraceOmittableFilterColumn[];
  hideControls?: boolean;
+  viewPersistenceKey?: string;
  externalFilterState?: FilterState;
  externalDateRange?: TableDateRange;
  limitRows?: number;
@@ -147,10 +151,15 @@ export default function TracesTable({
  userId,
  omittedFilter = [],
  hideControls = false,
+  viewPersistenceKey,
  externalFilterState,
  externalDateRange,
  limitRows,
 }: TracesTableProps) {
+  const tracesFilterConfig = useMemo(
+    () => getTraceFilterConfig(omittedFilter),
+    [omittedFilter],
+  );
  const utils = api.useUtils();
  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
  const [rawRefreshInterval, setRawRefreshInterval] =
@@ -326,7 +335,7 @@ export default function TracesTable({
    };
  }, [environmentFilterOptions.data, traceFilterOptionsResponse.data]);

-  const queryFilter = useSidebarFilterState(traceFilterConfig, filterOptions, {
+  const queryFilter = useSidebarFilterState(tracesFilterConfig, filterOptions, {
    loading:
      traceFilterOptionsResponse.isPending ||
      environmentFilterOptions.isPending,
@@ -863,7 +872,7 @@ export default function TracesTable({
        ]),
    {
      accessorKey: "sessionId",
-      enableColumnFilter: !omittedFilter.find((f) => f === "sessionId"),
+      enableColumnFilter: !omittedFilter.includes("sessionId"),
      id: "sessionId",
      header: "Session",
      size: 150,
@@ -1240,6 +1249,7 @@ export default function TracesTable({
  const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
    tableName: TableViewPresetTableName.Traces,
    projectId,
+    viewPersistenceKey,
    stateUpdaters: {
      setOrderBy: setOrderByState,
      setFilters: setFiltersWrapper,
@@ -1249,7 +1259,7 @@ export default function TracesTable({
    },
    validationContext: {
      columns,
-      filterColumnDefinition: traceFilterConfig.columnDefinitions,
+      filterColumnDefinition: tracesFilterConfig.columnDefinitions,
    },
    currentFilterState: queryFilter.explicitFilterState,
    disabled: hideControls,
@@ -1302,7 +1312,7 @@ export default function TracesTable({
  }, [traces.isSuccess, traceRowData?.rows]);

  return (
-    <DataTableControlsProvider tableName={traceFilterConfig.tableName}>
+    <DataTableControlsProvider tableName={tracesFilterConfig.tableName}>
      <div className="flex h-full w-full flex-col">
        {/* Toolbar spanning full width */}
        {!hideControls && (
@@ -63,6 +63,7 @@ import {
  aggregateTraceMetrics,
  getDescendantIds,
 } from "@/src/components/trace2/lib/trace-aggregation";
+import TagList from "@/src/features/tag/components/TagList";

 export interface ObservationDetailViewProps {
  observation: ObservationReturnTypeWithMetadata;
@@ -82,9 +83,9 @@ export function ObservationDetailView({
  } = useSelection();

  // V4 beta mode and observations for log tab
-  const { isBetaEnabled: isV4BetaEnabled } = useV4Beta();
+  const { isBetaEnabled: isV4Enabled } = useV4Beta();
  const { observations, roots, nodeMap } = useTraceData();
-  const showLogViewTab = isV4BetaEnabled && observations.length > 0;
+  const showLogViewTab = isV4Enabled && observations.length > 0;
  const isLogViewVirtualized =
    observations.length >= TRACE_VIEW_CONFIG.logView.virtualizationThreshold;

@@ -397,6 +398,25 @@ export function ObservationDetailView({
                : "overflow-auto pb-4"
            }`}
          >
+            {isRoot &&
+              observation.traceTags &&
+              observation.traceTags.length > 0 && (
+                <>
+                  <div
+                    className={`px-2 pt-2 text-sm font-medium ${currentView !== "pretty" ? "shrink-0" : ""}`}
+                  >
+                    Tags
+                  </div>
+                  <div
+                    className={`flex flex-wrap gap-x-1 gap-y-1 px-2 pb-2 ${currentView !== "pretty" ? "shrink-0" : ""}`}
+                  >
+                    <TagList
+                      selectedTags={observation.traceTags}
+                      isLoading={false}
+                    />
+                  </div>
+                </>
+              )}
            <IOPreview
              key={observation.id}
              observationName={observation.name ?? undefined}
@@ -1,4 +1,4 @@
-import { useMemo, useRef, useState } from "react";
+import { useMemo, useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import {
  Check,
@@ -24,6 +24,7 @@ import {
  usePromptReferenceProjectId,
 } from "@/src/components/ui/PromptReferences";
 import { copyTextToClipboard } from "@/src/utils/clipboard";
+import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";

 export const IO_TABLE_CHAR_LIMIT = 10000;

@@ -221,32 +222,32 @@ export function CodeView(props: {
 }) {
  const { copiedToClipboardMessage } = props;

-  const [isCopied, setIsCopied] = useState(false);
  const [isCollapsed, setCollapsed] = useState(props.defaultCollapsed);

-  const copySuccessStateDuration = copiedToClipboardMessage ? 3_000 : 1_000;
-  const copySuccessTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(
-    null,
-  );
+  const { copy, isCopied } = useCopyToClipboard({
+    successDuration: copiedToClipboardMessage ? 3_000 : 1_000,
+  });

-  const handleCopy = (event: React.MouseEvent<HTMLButtonElement>) => {
+  const handleCopy = async (event: React.MouseEvent<HTMLButtonElement>) => {
    event.preventDefault();
-    setIsCopied(true);
+    const button = event.currentTarget;
    const content =
      props.originalContent ??
      (typeof props.content === "string"
        ? props.content
        : (props.content?.join("\n") ?? ""));
-    void copyTextToClipboard(content);
-    if (copySuccessTimeoutRef.current)
-      clearTimeout(copySuccessTimeoutRef.current);
-    copySuccessTimeoutRef.current = setTimeout(
-      () => setIsCopied(false),
-      copySuccessStateDuration,
-    );

-    // Keep focus on the copy button to prevent focus shifting
-    event.currentTarget.focus();
+    try {
+      await copy(content);
+    } catch {
+      // Clipboard writes can be rejected when the browser denies permission.
+    }
+
+    if (button) {
+      // Keep focus on the copy button to prevent focus shifting
+      // Note: the original button might no longer be in the DOM if React re-rendered the component after the state update.
+      button.focus();
+    }
  };

  const handleShowAll = () => setCollapsed(!isCollapsed);
@@ -339,10 +340,13 @@ export const JsonSkeleton = ({
  borderless?: boolean;
  className?: string;
 }) => {
+  const isSingleLine = numRows === 1;
+
  return (
    <div
      className={cn(
-        "w-[400px] rounded-md",
+        isSingleLine ? "w-full" : "w-[400px]",
+        "rounded-md",
        borderless ? "" : "border",
        className,
      )}
@@ -352,7 +356,7 @@ export const JsonSkeleton = ({
          <Skeleton
            className={cn(
              "h-4 w-full",
-              i === numRows - 1 ? "w-3/4" : undefined,
+              !isSingleLine && i === numRows - 1 ? "w-3/4" : undefined,
            )}
            key={i}
          />
@@ -87,6 +87,7 @@ export const IOTableCell = ({
    return (
      <JsonSkeleton
        borderless
+        numRows={singleLine ? 1 : undefined}
        className="h-full w-full overflow-hidden px-2 py-1"
      />
    );
@@ -4,6 +4,7 @@ import Image from "next/image";
 import { InfoIcon } from "lucide-react";
 import { ActionButton } from "@/src/components/ActionButton";
 import { Alert, AlertTitle, AlertDescription } from "@/src/components/ui/alert";
+import { StatusBadge } from "@/src/components/layouts/status-badge";

 export interface ValueProposition {
  title: string;
@@ -18,9 +19,17 @@ export interface ActionConfig {
  component?: React.ReactNode;
 }

+export interface Step {
+  title: string;
+  description?: string;
+  badge?: React.ReactNode;
+  content?: React.ReactNode;
+}
+
 export interface SplashScreenProps {
  title: string;
  description: string;
+  waitingFor?: string;
  image?: {
    src: string;
    alt: string;
@@ -28,19 +37,17 @@ export interface SplashScreenProps {
    height: number;
  };
  videoSrc?: string;
+  steps?: Step[];
  valuePropositions?: ValueProposition[];
  primaryAction?: ActionConfig;
  secondaryAction?: ActionConfig;
  gettingStarted?: string | React.ReactNode;
  children?: React.ReactNode;
-  className?: string;
+  /** Where to render the video. Defaults to "top" (after header, before content) */
+  videoPosition?: "top" | "bottom";
 }

-interface VideoPlayerProps {
-  videoSrc: string;
-}
-
-function VideoPlayer({ videoSrc }: VideoPlayerProps) {
+function VideoPlayer({ videoSrc }: { videoSrc: string }) {
  const [hasError, setHasError] = useState(false);
  const [isLoaded, setIsLoaded] = useState(false);

@@ -48,9 +55,7 @@ function VideoPlayer({ videoSrc }: VideoPlayerProps) {
    <div
      className={cn(
        "border-border my-6 w-full max-w-3xl overflow-hidden rounded-lg border",
-        {
-          hidden: !isLoaded || hasError,
-        },
+        { hidden: !isLoaded || hasError },
      )}
    >
      <video
@@ -72,17 +77,46 @@ function VideoPlayer({ videoSrc }: VideoPlayerProps) {
 export function SplashScreen({
  title,
  description,
+  waitingFor,
  image,
  videoSrc,
+  steps,
  valuePropositions = [],
  primaryAction,
  secondaryAction,
  gettingStarted,
  children,
+  videoPosition = "top",
 }: SplashScreenProps) {
+  const mediaBlock = (
+    <>
+      {videoSrc && <VideoPlayer videoSrc={videoSrc} />}
+      {!videoSrc && image && (
+        <div className="my-6 w-full max-w-3xl">
+          <Image
+            src={image.src}
+            alt={image.alt}
+            width={image.width}
+            height={image.height}
+            className="rounded-md"
+          />
+        </div>
+      )}
+    </>
+  );
+
  return (
-    <div className={cn("mx-auto flex max-w-4xl flex-col items-center p-8")}>
+    <div className="mx-auto flex max-w-4xl flex-col items-center p-8">
      <div className="mb-6 text-center">
+        {waitingFor && (
+          <StatusBadge
+            type="waiting"
+            showText={false}
+            className="mb-3 px-3 py-1 text-sm"
+          >
+            {waitingFor}
+          </StatusBadge>
+        )}
        <h2 className="mb-2 text-2xl font-bold">{title}</h2>
        <p className="text-muted-foreground">{description}</p>
      </div>
@@ -98,7 +132,6 @@ export function SplashScreen({
              {primaryAction.label}
            </ActionButton>
          ))}
-
        {secondaryAction &&
          (secondaryAction.component || (
            <ActionButton
@@ -120,17 +153,42 @@ export function SplashScreen({
        </Alert>
      )}

-      {videoSrc && <VideoPlayer videoSrc={videoSrc} />}
+      {videoPosition === "top" && mediaBlock}

-      {!videoSrc && image && (
-        <div className="my-6 w-full max-w-3xl">
-          <Image
-            src={image.src}
-            alt={image.alt}
-            width={image.width}
-            height={image.height}
-            className="rounded-md"
-          />
+      {/* Numbered steps */}
+      {steps && steps.length > 0 && (
+        <div className="w-full max-w-4xl">
+          {steps.map((step, index) => (
+            <div key={index} className="flex gap-4">
+              {/* Left: circle + connecting line */}
+              <div className="flex flex-col items-center">
+                <div className="bg-foreground text-background flex h-8 w-8 shrink-0 items-center justify-center rounded-full text-sm font-semibold">
+                  {index + 1}
+                </div>
+                {index < steps.length - 1 && (
+                  <div className="bg-border mt-1 w-px flex-1" />
+                )}
+              </div>
+              {/* Right: title + content */}
+              <div
+                className={cn(
+                  "min-w-0 flex-1 pt-1",
+                  index < steps.length - 1 ? "pb-8" : "pb-0",
+                )}
+              >
+                <div className="mb-2 flex items-center gap-3">
+                  <h3 className="text-xl font-semibold">{step.title}</h3>
+                  {step.badge}
+                </div>
+                {step.description && (
+                  <p className="text-muted-foreground text-sm leading-6">
+                    {step.description}
+                  </p>
+                )}
+                {step.content && <div className="mt-3">{step.content}</div>}
+              </div>
+            </div>
+          ))}
        </div>
      )}

@@ -147,6 +205,8 @@ export function SplashScreen({
          ))}
        </div>
      )}
+
+      {videoPosition === "bottom" && mediaBlock}
    </div>
  );
 }
@@ -1,6 +1,9 @@
 import * as React from "react";

+import { Button } from "@/src/components/ui/button";
+import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
 import { cn } from "@/src/utils/tailwind";
+import { Check, Copy } from "lucide-react";

 type TableDensity = "compact" | "comfortable";

@@ -101,6 +104,59 @@ const TableCell = React.forwardRef<
 ));
 TableCell.displayName = "TableCell";

+type TableCellWithCopyButtonProps =
+  React.TdHTMLAttributes<HTMLTableCellElement> & {
+    text: string;
+    density?: TableDensity;
+    copyButtonLabel?: string;
+  };
+
+const TableCellWithCopyButton = React.forwardRef<
+  HTMLTableCellElement,
+  TableCellWithCopyButtonProps
+>(({ text, copyButtonLabel, className, ...props }, ref) => {
+  const { copy, isCopied } = useCopyToClipboard();
+
+  return (
+    <TableCell
+      ref={ref}
+      className={cn("relative min-w-0 pr-10", className)}
+      title={text}
+      {...props}
+    >
+      {text}
+      <Button
+        variant="ghost"
+        size="icon-xs"
+        className="absolute top-1/2 right-2 -translate-y-1/2"
+        title={copyButtonLabel ?? "Copy to clipboard"}
+        aria-label={copyButtonLabel ?? "Copy to clipboard"}
+        onClick={async (event) => {
+          event.preventDefault();
+          const button = event.currentTarget;
+          try {
+            await copy(text);
+          } catch {
+            // Clipboard writes can be rejected when the browser denies permission.
+          }
+
+          if (button) {
+            // The original button might no longer be in the DOM if React re-rendered the component after the state update.
+            button.focus();
+          }
+        }}
+      >
+        {isCopied ? (
+          <Check className="h-3 w-3" />
+        ) : (
+          <Copy className="h-3 w-3" />
+        )}
+      </Button>
+    </TableCell>
+  );
+});
+TableCellWithCopyButton.displayName = "TableCellWithCopyButton";
+
 const TableCaption = React.forwardRef<
  HTMLTableCaptionElement,
  React.HTMLAttributes<HTMLTableCaptionElement>
@@ -121,5 +177,6 @@ export {
  TableHead,
  TableRow,
  TableCell,
+  TableCellWithCopyButton,
  TableCaption,
 };
@@ -1 +1 @@
-export const VERSION = "v3.167.2";
+export const VERSION = "v3.168.0";
@@ -188,6 +188,7 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
      ...provider.authConfig,
      clientSecret: decrypt(provider.authConfig.clientSecret),
+      issuer: "https://github.com/login/oauth",
      ...getClientConfig(provider.authConfig),
    });
  else if (provider.authProvider === "gitlab")
@@ -264,6 +265,8 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
      enterprise: {
        baseUrl: provider.authConfig.enterprise.baseUrl,
      },
+      issuer: new URL("/login/oauth", provider.authConfig.enterprise.baseUrl)
+        .href,
      ...getClientConfig(provider.authConfig),
    });
  else if (provider.authProvider === "jumpcloud")
@@ -106,7 +106,7 @@ export const env = createEnv({
    AUTH_GITHUB_CHECKS: zAuthChecks,
    AUTH_GITHUB_ENTERPRISE_CLIENT_ID: z.string().optional(),
    AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET: z.string().optional(),
-    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().url().optional(),
    AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING: z
      .enum(["true", "false"])
      .optional(),
@@ -5,7 +5,8 @@ import {
 import { AnnotationDrawerSection } from "../shared/AnnotationDrawerSection";
 import { AnnotationProcessingLayout } from "../shared/AnnotationProcessingLayout";
 import { SessionIO } from "@/src/components/session";
-import { useState, useEffect } from "react";
+import { LazyTraceEventsRow } from "@/src/components/session/TraceEventsRow";
+import { useState, useMemo, useCallback } from "react";
 import { Button } from "@/src/components/ui/button";
 import { ItemBadge } from "@/src/components/ItemBadge";
 import { CopyIdsPopover } from "@/src/components/trace2/components/_shared/CopyIdsPopover";
@@ -13,6 +14,9 @@ import { Badge } from "@/src/components/ui/badge";
 import { Separator } from "@/src/components/ui/separator";
 import Link from "next/link";
 import { Card } from "@/src/components/ui/card";
+import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
+import { api } from "@/src/utils/api";
+import { JsonSkeleton } from "@/src/components/ui/CodeJsonViewer";

 interface SessionAnnotationProcessorProps {
  item: AnnotationQueueItem & {
@@ -27,39 +31,67 @@ interface SessionAnnotationProcessorProps {
 // some projects have thousands of traces in a session, paginate to avoid rendering all at once
 const PAGE_SIZE = 10;

+// Stable empty array to avoid creating new references on every render (defeats React.memo)
+const EMPTY_FILTER_STATE: [] = [];
+
 export const SessionAnnotationProcessor: React.FC<
  SessionAnnotationProcessorProps
 > = ({ item, data, configs, projectId }) => {
  const [visibleTraces, setVisibleTraces] = useState(PAGE_SIZE);
-  const [currentTraceIndex, setCurrentTraceIndex] = useState(1);
+  const { isBetaEnabled } = useV4Beta();

-  // Intersection observer to which trace is currently in view
-  useEffect(() => {
-    if (!data?.traces) return;
-
-    const observer = new IntersectionObserver(
-      (entries) => {
-        entries.forEach((entry) => {
-          if (entry.isIntersecting && entry.intersectionRatio > 0.5) {
-            const index = parseInt(
-              entry.target.getAttribute("data-trace-index") || "0",
-            );
-            setCurrentTraceIndex(index + 1);
-          }
-        });
+  // Fetch traces separately when v4 beta is enabled (events table path)
+  // The byIdWithScoresFromEvents endpoint doesn't include traces array
+  const tracesFromEventsQuery = api.sessions.tracesFromEvents.useQuery(
+    { projectId, sessionId: item.objectId },
+    {
+      enabled: isBetaEnabled,
+      retry(failureCount, error) {
+        if (
+          error.data?.code === "UNAUTHORIZED" ||
+          error.data?.code === "NOT_FOUND"
+        )
+          return false;
+        return failureCount < 3;
      },
+    },
+  );
+
+  const traceCommentCounts =
+    api.comments.getTraceCommentCountsBySessionId.useQuery(
      {
-        threshold: 0.5, // Trigger when 50% of trace is visible
-        rootMargin: "-25% 0px -25% 0px", // Focus on center area
+        projectId,
+        sessionId: item.objectId,
      },
+      { enabled: isBetaEnabled },
    );

-    // Observe all trace cards
-    const traceCards = document.querySelectorAll("[data-trace-index]");
-    traceCards.forEach((card) => observer.observe(card));
+  // Unify traces from both paths:
+  // - v4 beta OFF: traces come from data.traces (byIdWithScores endpoint)
+  // - v4 beta ON: traces come from separate tracesFromEvents query
+  const traces = useMemo(() => {
+    if (isBetaEnabled) {
+      return tracesFromEventsQuery.data ?? [];
+    }
+    return data?.traces ?? [];
+  }, [isBetaEnabled, tracesFromEventsQuery.data, data?.traces]);

-    return () => observer.disconnect();
-  }, [data?.traces, visibleTraces]);
+  // For the "Total traces" badge, show countTraces from session metadata when available (v4),
+  // or fall back to loaded traces length
+  const totalTracesForBadge = useMemo(() => {
+    if (isBetaEnabled) {
+      return data?.countTraces ?? traces.length;
+    }
+    return traces.length;
+  }, [isBetaEnabled, data?.countTraces, traces.length]);
+
+  // Stable callback to avoid creating new function reference on every render (defeats React.memo)
+  const openPeek = useCallback(
+    (traceId: string) => {
+      window.open(`/project/${projectId}/traces/${traceId}`, "_blank");
+    },
+    [projectId],
+  );

  const leftPanel = (
    <div className="flex h-full flex-col overflow-hidden">
@@ -77,22 +109,15 @@ export const SessionAnnotationProcessor: React.FC<
              idItems={[{ id: item.objectId, name: "Session ID" }]}
            />
          </div>
-          {data?.traces && (
-            <div className="flex items-center">
-              <Badge variant="outline" className="text-xs">
-                Trace {currentTraceIndex} / {data.traces.length}
-              </Badge>
-            </div>
-          )}
        </div>
        <div className="mt-2 mb-4 grid w-full min-w-0 items-center justify-between px-4">
          <div className="flex max-w-full min-w-0 shrink flex-col">
            <div className="flex max-w-full min-w-0 flex-wrap items-center gap-1">
-              {data.environment && (
+              {data?.environment && (
                <Badge variant="tertiary">Env: {data.environment}</Badge>
              )}
              <Badge variant="outline">
-                Total traces: {data?.traces.length}
+                Total traces: {totalTracesForBadge}
              </Badge>
            </div>
          </div>
@@ -103,13 +128,53 @@ export const SessionAnnotationProcessor: React.FC<
      {/* Scrollable Content */}
      <div className="flex-1 overflow-y-auto">
        <div className="p-4">
-          {data?.traces
-            .slice(0, visibleTraces)
-            .map((trace: any, index: number) => (
+          {/* Loading state for v4 beta traces */}
+          {isBetaEnabled && tracesFromEventsQuery.isLoading && (
+            <div className="space-y-4">
+              {Array.from({ length: 3 }).map((_, i) => (
+                <Card
+                  key={i}
+                  className="border-border mb-2 grid gap-2 p-2 shadow-none"
+                >
+                  <JsonSkeleton
+                    className="h-full w-full overflow-hidden"
+                    numRows={4}
+                  />
+                </Card>
+              ))}
+            </div>
+          )}
+          {/* Error state for v4 beta traces */}
+          {isBetaEnabled && tracesFromEventsQuery.isError && (
+            <div className="text-destructive p-2 text-sm">
+              Failed to load traces for this session.
+            </div>
+          )}
+          {/* Trace list - v4 path uses LazyTraceEventsRow for deferred loading */}
+          {isBetaEnabled &&
+            tracesFromEventsQuery.isSuccess &&
+            traces
+              .slice(0, visibleTraces)
+              .map((trace: any, index: number) => (
+                <LazyTraceEventsRow
+                  key={trace.id}
+                  trace={trace}
+                  projectId={projectId}
+                  sessionId={item.objectId}
+                  openPeek={openPeek}
+                  traceCommentCounts={traceCommentCounts.data ?? undefined}
+                  showCorrections
+                  filterState={EMPTY_FILTER_STATE}
+                  hideTracePanel
+                  index={index}
+                />
+              ))}
+          {/* Trace list - v3 path uses SessionIO */}
+          {!isBetaEnabled &&
+            traces.slice(0, visibleTraces).map((trace: any) => (
              <Card
                className="border-border hover:border-ring group mb-2 grid gap-2 p-2 shadow-none"
                key={trace.id}
-                data-trace-index={index}
              >
                <div className="-mt-1 p-1 pt-0 opacity-50 transition-opacity group-hover:opacity-100">
                  <Link
@@ -130,16 +195,17 @@ export const SessionAnnotationProcessor: React.FC<
                />
              </Card>
            ))}
-          {data?.traces && data.traces.length > visibleTraces && (
-            <div className="flex justify-center py-4">
-              <Button
-                onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
-                variant="ghost"
-              >
-                {`Load ${Math.min(data.traces.length - visibleTraces, PAGE_SIZE)} More`}
-              </Button>
-            </div>
-          )}
+          {(!isBetaEnabled || tracesFromEventsQuery.isSuccess) &&
+            traces.length > visibleTraces && (
+              <div className="flex justify-center py-4">
+                <Button
+                  onClick={() => setVisibleTraces((prev) => prev + PAGE_SIZE)}
+                  variant="ghost"
+                >
+                  {`Load ${Math.min(traces.length - visibleTraces, PAGE_SIZE)} More`}
+                </Button>
+              </div>
+            )}
        </div>
      </div>
    </div>
@@ -42,7 +42,7 @@ export async function createUserEmailPassword(
    },
  });

-  await createProjectMembershipsOnSignup(newUser);
+  await createProjectMembershipsOnSignup(newUser, { userWasJustCreated: true });

  return newUser.id;
 }
@@ -4,12 +4,18 @@ import { logger } from "@langfuse/shared/src/server";
 import { ServerPosthog } from "@/src/features/posthog-analytics/ServerPosthog";
 import { hasEntitlementBasedOnPlan } from "@/src/features/entitlements/server/hasEntitlement";
 import { getOrganizationPlanServerSide } from "@/src/features/entitlements/server/getPlan";
+import { shouldAutoEnableV4 } from "@/src/features/events/lib/v4Rollout";

-export async function createProjectMembershipsOnSignup(user: {
-  id: string;
-  email: string | null;
-}) {
+export async function createProjectMembershipsOnSignup(
+  user: {
+    id: string;
+    email: string | null;
+  },
+  options?: { userWasJustCreated?: boolean },
+) {
  try {
+    const isCloudDeployment = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
+
    // in no case do we want to send duplicate sign up events to posthog
    const isNewUser = !(await prisma.organizationMembership.findFirst({
      where: { userId: user.id },
@@ -150,6 +156,60 @@ export async function createProjectMembershipsOnSignup(user: {
    // Invites do not work for users without emails (some future SSO users)
    if (user.email) await processMembershipInvitations(user.email, user.id);

+    if (isCloudDeployment && (options?.userWasJustCreated || isNewUser)) {
+      const userRolloutState = await prisma.user.findUnique({
+        where: { id: user.id },
+        select: {
+          createdAt: true,
+          v4BetaEnabled: true,
+          organizationMemberships: {
+            select: {
+              organization: {
+                select: {
+                  id: true,
+                  createdAt: true,
+                },
+              },
+            },
+          },
+        },
+      });
+
+      if (userRolloutState) {
+        const shouldAutoEnableV4ForUser = shouldAutoEnableV4({
+          userCreatedAt: userRolloutState.createdAt,
+          organizations: userRolloutState.organizationMemberships.map(
+            (membership) => ({
+              id: membership.organization.id,
+              createdAt: membership.organization.createdAt,
+            }),
+          ),
+          excludedOrganizationIds: env.NEXT_PUBLIC_DEMO_ORG_ID
+            ? [env.NEXT_PUBLIC_DEMO_ORG_ID]
+            : [],
+        });
+        const shouldInitializeForNewUser =
+          options?.userWasJustCreated &&
+          !userRolloutState.v4BetaEnabled &&
+          shouldAutoEnableV4ForUser;
+        const shouldInitializeForFirstOrganization =
+          !options?.userWasJustCreated &&
+          isNewUser &&
+          !userRolloutState.v4BetaEnabled &&
+          shouldAutoEnableV4ForUser;
+
+        if (
+          shouldInitializeForNewUser ||
+          shouldInitializeForFirstOrganization
+        ) {
+          await prisma.user.update({
+            where: { id: user.id },
+            data: { v4BetaEnabled: true },
+          });
+        }
+      }
+    }
+
    // for conversion metric tracking in posthog: did a new user sign up?
    if (
      isNewUser &&
@@ -274,6 +274,7 @@ export const blobStorageIntegrationRouter = createTRPCRouter({
          forcePathStyle: forcePathStyle || false,
          useAzureBlob: type === BlobStorageIntegrationType.AZURE_BLOB_STORAGE,
          useGoogleCloudStorage: false, // Not supported in blob storage integration
+          useOCIObjectStorage: false, // Not supported in blob storage integration
          googleCloudCredentials: undefined,
          awsSse: undefined,
          awsSseKmsKeyId: undefined,
@@ -1,4 +1,6 @@
+import { useSession } from "next-auth/react";
 import { api } from "@/src/utils/api";
+import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
 export interface EvalCapabilities {
  isNewCompatible: boolean;
  allowLegacy: boolean;
@@ -18,6 +20,9 @@ const mockOtelStatus = {
 * @returns Capabilities object indicating which eval features are allowed
 */
 export function useEvalCapabilities(projectId: string): EvalCapabilities {
+  const { data: session, status: sessionStatus } = useSession();
+  const isSessionLoading = sessionStatus === "loading";
+
  // Query OTEL SDK status
  const { isOtel, isPropagating } = mockOtelStatus;

@@ -25,13 +30,20 @@ export function useEvalCapabilities(projectId: string): EvalCapabilities {
  const evalCounts = api.evals.counts.useQuery({ projectId });
  const hasLegacyEvals = (evalCounts.data?.legacyConfigCount ?? 0) > 0;

+  // Only hide legacy options for new cloud users (canToggleV4 = false)
+  // Non-cloud deployments always see legacy options
+  // Use === true to default to false while session is loading, preventing flash of legacy options
+  // New users (canToggleV4 = false) default to observation-level evals regardless of v3/v4
+  const { isLangfuseCloud } = useLangfuseCloudRegion();
+  const canToggleV4 = session?.user?.canToggleV4 === true;
+
  return {
    isNewCompatible: isOtel,
-    // Allow legacy evals if user already has them OR if not using OTEL
-    allowLegacy: hasLegacyEvals || !isOtel,
+    // Allow legacy if: not cloud OR user has legacy evals OR user can toggle v4 (existing user)
+    allowLegacy: !isLangfuseCloud || hasLegacyEvals || canToggleV4,
    // Allow propagation filters only when using OTEL and spans are propagating
    allowPropagationFilters: isOtel && isPropagating,
-    isLoading: evalCounts.isLoading,
+    isLoading: evalCounts.isLoading || isSessionLoading,
    hasLegacyEvals,
  };
 }
@@ -11,7 +11,8 @@ import { usePaginationState } from "@/src/hooks/usePaginationState";
 import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
 import {
  getEventsColumnName,
-  observationEventsFilterConfig,
+  getObservationEventsFilterConfig,
+  type ObservationEventsOmittableFilterColumn,
 } from "../config/filter-config";
 import { DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG } from "@/src/features/filters/constants/internal-environments";
 import { formatIntervalSeconds } from "@/src/utils/dates";
@@ -165,7 +166,9 @@ export type EventsTableRow = {
 export type EventsTableProps = {
  projectId: string;
  userId?: string;
+  omittedFilter?: ObservationEventsOmittableFilterColumn[];
  hideControls?: boolean;
+  viewPersistenceKey?: string;
  // External control props for embedded preview tables
  externalFilterState?: FilterState;
  externalDateRange?: TableDateRange;
@@ -176,7 +179,9 @@ export type EventsTableProps = {
 export default function ObservationsEventsTable({
  projectId,
  userId,
+  omittedFilter = [],
  hideControls = false,
+  viewPersistenceKey,
  externalFilterState,
  externalDateRange,
  limitRows,
@@ -184,6 +189,10 @@ export default function ObservationsEventsTable({
 }: EventsTableProps) {
  const router = useRouter();
  const { viewId } = router.query;
+  const eventsFilterConfig = useMemo(
+    () => getObservationEventsFilterConfig(omittedFilter),
+    [omittedFilter],
+  );

  const { setDetailPageList } = useDetailPageLists();
  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
@@ -353,17 +362,13 @@ export default function ObservationsEventsTable({
    oldFilterState,
  });

-  const queryFilter = useSidebarFilterState(
-    observationEventsFilterConfig,
-    filterOptions,
-    {
-      loading: isFilterOptionsPending,
-      disableUrlPersistence: hideControls, // Disable URL persistence for embedded preview tables
-      sessionFilterContextId: projectId,
-      // Sidebar-only implicit environment defaults
-      implicitDefaultConfig: DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG,
-    },
-  );
+  const queryFilter = useSidebarFilterState(eventsFilterConfig, filterOptions, {
+    loading: isFilterOptionsPending,
+    disableUrlPersistence: hideControls, // Disable URL persistence for embedded preview tables
+    sessionFilterContextId: projectId,
+    // Sidebar-only implicit environment defaults
+    implicitDefaultConfig: DEFAULT_SIDEBAR_IMPLICIT_ENVIRONMENT_CONFIG,
+  });

  // Create ref-based wrapper to avoid stale closure when queryFilter updates
  const queryFilterRef = useRef(queryFilter);
@@ -580,6 +585,7 @@ export default function ObservationsEventsTable({
        if (ioLoading) {
          return (
            <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
              borderless
              className="h-full w-full overflow-hidden px-2 py-1"
            />
@@ -605,6 +611,7 @@ export default function ObservationsEventsTable({
        if (ioLoading) {
          return (
            <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
              borderless
              className="h-full w-full overflow-hidden px-2 py-1"
            />
@@ -634,6 +641,7 @@ export default function ObservationsEventsTable({
        if (ioLoading) {
          return (
            <JsonSkeleton
+              numRows={rowHeight === "s" ? 1 : undefined}
              borderless
              className="h-full w-full overflow-hidden px-2 py-1"
            />
@@ -1017,7 +1025,8 @@ export default function ObservationsEventsTable({
      cell: ({ row }) => {
        const traceTags: string[] | undefined = row.getValue("traceTags");
        return (
-          traceTags && (
+          traceTags &&
+          traceTags.length > 0 && (
            <div
              className={cn(
                "flex gap-x-2 gap-y-1",
@@ -1146,6 +1155,7 @@ export default function ObservationsEventsTable({
  const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({
    tableName: TableViewPresetTableName.Observations,
    projectId,
+    viewPersistenceKey,
    stateUpdaters: {
      setOrderBy: setOrderByState,
      setFilters: setFiltersWrapper,
@@ -1155,7 +1165,7 @@ export default function ObservationsEventsTable({
    },
    validationContext: {
      columns,
-      filterColumnDefinition: observationEventsFilterConfig.columnDefinitions,
+      filterColumnDefinition: eventsFilterConfig.columnDefinitions,
    },
    currentFilterState: queryFilter.explicitFilterState,
    disabled: hideControls,
@@ -1210,7 +1220,7 @@ export default function ObservationsEventsTable({
              promptId: observation.promptId ?? undefined,
              promptName: observation.promptName ?? undefined,
              promptVersion: observation.promptVersion?.toString() ?? undefined,
-              traceTags: undefined, // TODO: traceTags not available in EventsObservation
+              traceTags: observation.traceTags ?? undefined,
              traceName: observation.traceName ?? undefined,
              timestamp: observation.startTime ?? undefined,
              usageDetails: observation.usageDetails ?? {},
@@ -1262,9 +1272,7 @@ export default function ObservationsEventsTable({
  }, [selectedObservationIds, observations.rows]);

  return (
-    <DataTableControlsProvider
-      tableName={observationEventsFilterConfig.tableName}
-    >
+    <DataTableControlsProvider tableName={eventsFilterConfig.tableName}>
      <div className="flex h-full w-full flex-col">
        {/* Toolbar spanning full width */}
        {!hideControls && (
--- a/Show More
+++ b/Show More