chore: release v2.83.1

revert: next upgrade due to incompatibility issues (#3640 )
chore: remove explicit context propagation for queues (#3622 )
2024-10-08 02:42:25 -07:00 · 2024-10-08 11:25:41 +02:00 · 2024-10-08 09:16:29 +02:00 · 2024-10-08 05:27:30 +00:00 · 2024-10-08 05:15:27 +00:00 · 2024-10-08 05:03:19 +00:00
65 changed files with 3882 additions and 3934 deletions
@@ -53,4 +53,4 @@ LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 

 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
@@ -36,6 +36,10 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # PORT=3000
 # HOSTNAME=localhost

+# Opentelemetry, optional
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+OTEL_SERVICE_NAME="langfuse"
+
 # Default role for users who sign up, optional, can be org or org+project
 # LANGFUSE_DEFAULT_ORG_ID=
 # LANGFUSE_DEFAULT_ORG_ROLE=
@@ -63,6 +67,9 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITLAB_CLIENT_ID=
+# AUTH_GITLAB_CLIENT_SECRET=
+# AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
 # AUTH_AZURE_AD_CLIENT_ID=
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
@@ -22,7 +22,7 @@ jobs:

    steps:
      - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Start containers
        run: docker compose -f "docker-compose.yml" up -d --build
@@ -31,7 +31,7 @@ jobs:
        uses: pnpm/action-setup@v2.2.4

      - name: Setup Node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
        with:
          node-version: 18

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main", "production", "v*"]
+  pull_request:
+    branches: ["main", "production", "v*"]
+  schedule:
+    - cron: "31 13 * * 0"
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: javascript-typescript
+            build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # If the analyze step fails for one of the languages you are analyzing with
+      # "We were unable to automatically build your code", modify the matrix above
+      # to set the build mode to "manual" for that language. Then modify this step
+      # to build your code.
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      - if: matrix.build-mode == 'manual'
+        shell: bash
+        run: |
+          echo 'If you are using a "manual" build mode for one or more of the' \
+            'languages you are analyzing, replace this with the commands to build' \
+            'your code, for example:'
+          echo '  make bootstrap'
+          echo '  make release'
+          exit 1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"
@@ -14,10 +14,10 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4

      - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
        with:
          node-version: 18

@@ -16,11 +16,11 @@ jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
        with:
          version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "pnpm"
@@ -35,10 +35,11 @@ jobs:
        run: pnpm run lint

  test-docker-build:
+    timeout-minutes: 20
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Set NEXT_PUBLIC_BUILD_ID
        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
@@ -48,14 +49,6 @@ jobs:
          docker compose -f docker-compose.build.yml up -d
          sleep 5 # Wait for PostgreSQL to accept connections

-      - name: Check server health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
-
-      - name: Check worker health
-        run: |
-          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
-
      - name: Ensure no unhealthy status
        run: |
          if docker-compose ps | grep "(unhealthy)"; then
@@ -65,6 +58,14 @@ jobs:
            echo "All services are healthy"
          fi

+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
  tests-web:
    timeout-minutes: 20
    runs-on: ubuntu-latest
@@ -79,12 +80,12 @@ jobs:
        with:
          swap-size-gb: 10

-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
        with:
          version: 9.5.0
      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -144,13 +145,13 @@ jobs:
        with:
          swap-size-gb: 10

-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
        with:
          version: 9.5.0

      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          cache: "pnpm"
@@ -201,11 +202,11 @@ jobs:
  e2e-tests:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
        with:
          version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "pnpm"
@@ -243,11 +244,11 @@ jobs:
  e2e-server-tests:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
        with:
          version: 9.5.0
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "pnpm"
@@ -321,13 +322,13 @@ jobs:
        with:
          version: 9.5.0
      - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache-dependency-path: "pnpm-lock.yaml"

      - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Set NEXT_PUBLIC_BUILD_ID
        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
@@ -22,6 +22,15 @@ services:
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
+      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
+      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
+      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
+      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
+      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
+      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
+      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
    restart: always
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -14,6 +14,15 @@ services:
      - NEXTAUTH_URL=http://localhost:3000
      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
+      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
+      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
+      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
+      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
+      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
+      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}

  db:
    image: postgres
@@ -25,7 +25,8 @@
  },
  "dependencies": {
    "@langfuse/shared": "workspace:*",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "axios": "^1.7.7",
    "next": "^14.2.6",
    "next-auth": "^4.24.7",
    "zod": "^3.23.8"
@@ -39,7 +40,7 @@
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
    "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
    "typescript": "^5.4.5"
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "2.81.2",
+  "version": "2.83.1",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -33,7 +33,7 @@
    "braces": "3.0.3",
    "dotenv-cli": "^7.4.2",
    "husky": "^9.0.11",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
    "release-it": "^17.3.0",
    "turbo": "^1.13.4"
  },
@@ -11,7 +11,7 @@
    "@typescript-eslint/eslint-plugin": "^7.1.0",
    "@typescript-eslint/parser": "^7.12.0",
    "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.6",
+    "eslint-config-next": "^14.2.14",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-turbo": "^1.13.4",
    "eslint-plugin-only-warn": "^1.1.0",
@@ -59,16 +59,17 @@
    "@aws-sdk/s3-request-presigner": "^3.554.0",
    "@clickhouse/client": "^1.4.0",
    "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.3",
+    "@langchain/core": "^0.3.7",
    "@langchain/openai": "^0.3.0",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "@prisma/client": "^5.17.0",
+    "@opentelemetry/api": ">=1.0.0 <1.10.0",
+    "@prisma/client": "^5.20.0",
    "@react-email/components": "^0.0.19",
    "@react-email/render": "^0.0.15",
    "@types/bcryptjs": "^2.4.6",
+    "axios": "^1.7.7",
    "bcryptjs": "^2.4.3",
    "bullmq": "^5.12.10",
-    "dd-trace": "^5.19.0",
+    "dd-trace": "^5.23.1",
    "decimal.js": "^10.4.3",
    "exponential-backoff": "^3.1.1",
    "ioredis": "^5.4.1",
@@ -76,37 +77,36 @@
    "langchain": "^0.3.2",
    "lodash": "^4.17.21",
    "next-auth": "^4.24.7",
-    "nodemailer": "^6.9.13",
+    "nodemailer": "^6.9.15",
    "prisma-extension-kysely": "^2.1.0",
    "uuid": "^9.0.1",
    "winston": "^3.14.2",
-    "winston-transport": "^4.7.1",
    "zod": "^3.23.8",
    "zod-to-json-schema": "^3.23.2"
  },
  "devDependencies": {
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
    "@types/node": "^20.11.29",
-    "@types/nodemailer": "^6.4.15",
-    "@types/pg": "^8.11.6",
+    "@types/nodemailer": "^6.4.16",
+    "@types/pg": "^8.11.10",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/parser": "^7.12.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
    "eslint-plugin-prettier": "^5.1.3",
-    "kysely-codegen": "^0.11.0",
-    "nodemon": "^3.1.3",
-    "prettier": "^3.2.5",
-    "prisma": "^5.17.0",
+    "kysely-codegen": "^0.16.8",
+    "nodemon": "^3.1.7",
+    "prettier": "^3.3.3",
+    "prisma": "^5.20.0",
    "prisma-erd-generator": "^1.11.2",
    "prisma-kysely": "^1.8.0",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
    "typescript": "^5.4.5",
-    "vitest": "^1.5.3"
+    "vitest": "^2.1.2"
  },
  "peerDependencies": {
    "@types/react": "~18.2.79",
@@ -73,14 +73,40 @@ export class KyselySingleton {
              createQueryCompiler: () => new PostgresQueryCompiler(),
            },
          }),
-      }),
+      })
    );

    return KyselySingleton.instance;
  }
 }

-export const prisma = PrismaClientSingleton.getInstance();
-export const kyselyPrisma = KyselySingleton.getInstance();
+declare const globalThis: {
+  prismaGlobal: PrismaClient | undefined;
+  kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
+} & typeof global;
+
+if (process.env.NODE_ENV === "development") {
+  globalThis.prismaGlobal ??= new PrismaClient(); // regular instantiation
+  globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    })
+  );
+};
+
+export const prisma =
+  globalThis.prismaGlobal ?? PrismaClientSingleton.getInstance();
+export const kyselyPrisma =
+  globalThis.kyselyPrismaGlobal ?? KyselySingleton.getInstance();

 export * from "@prisma/client";
@@ -63,10 +63,10 @@ export const observationsTableCols: ColumnDefinition[] = [
    internal: '"latency"',
  },
  {
-    name: "Time Per Output Token (s)",
-    id: "timePerOutputToken",
+    name: "Tokens per second",
+    id: "tokensPerSecond",
    type: "number",
-    internal: '"latency" / o."completion_tokens"',
+    internal: 'o."completion_tokens" / "latency"',
    nullable: true,
  },
  {
@@ -12,13 +12,3 @@ Throughout our applications we want to use as much Otel as possible. This helps
 - `web` and `worker` have an instrumentation.ts file, which configures otel for the application.
 - For trpc, we use `@baselime/trpc-opentelemetry-middleware` to enrich spans with trpc inputs and outputs.
 - When building adding new infrastructure, we should search for auto instumentations for our code base.
-
-## Library support
-
- `dd-trace` has a direct dependency of `@opentelemetry/api` version < `1.9` [GH](https://github.com/DataDog/dd-trace-js/blob/ed9b0b30f7b0283579a9bf8c18e1f9deab18fecf/package.json#L81).
- `@prisma/instrumentation` requires at least `@opentelemetry/api` version `1.8` [GH](https://github.com/prisma/prisma/blob/d780290b13754420abcfa5d7592f02049c6cc005/packages/instrumentation/package.json#L25)
- We need to ensure that all other libraries we use are compatible with `@opentelemetry/api` version `1.8`. If we have a package using a later version, this will break the instrumentation and crash the container. We can check the versions with the following command:
-
-```bash
-pnpm --filter=web why @opentelemetry/api
-```
@@ -86,18 +86,6 @@ export function instrumentSync<T>(

 export const getCurrentSpan = () => opentelemetry.trace.getActiveSpan();

-export const addTraceContext = <T extends Record<string, any>>(
-  input: T,
-): T & { _tracecontext?: TCarrier } => {
-  const context = {};
-  opentelemetry.propagation.inject(opentelemetry.context.active(), context);
-
-  return {
-    ...input,
-    _tracecontext: context,
-  };
-};
-
 export const traceException = (
  ex: unknown,
  span?: opentelemetry.Span,
@@ -192,3 +180,15 @@ export const recordHistogram = (
 ) => {
  dd.dogstatsd.histogram(stat, value, tags);
 };
+
+/**
+ * Converts a queue name to the matching datadog metric name.
+ * Consumer only needs to append the relevant suffix.
+ *
+ * Example: `legacy-ingestion-queue` -> `langfuse.queue.legacy_ingestion`
+ */
+export const convertQueueNameToMetricName = (queueName: string): string => {
+  return (
+    "langfuse.queue." + queueName.replace(/-/g, "_").replace(/_queue$/, "")
+  );
+};
@@ -1,26 +1,21 @@
 import { env } from "../env";
 import winston from "winston";
-import Transport from "winston-transport";
 import { getCurrentSpan } from "./instrumentation";

-class TracedTransport extends Transport {
-  constructor(opts: Transport.TransportStreamOptions = {}) {
-    super(opts);
-  }
-
-  log(info: Record<string, any>, callback: () => void): void {
-    setImmediate(() => {
-      this.emit("logged", info);
-    });
-
-    const currentSpan = getCurrentSpan();
-    info.trace_id = currentSpan?.spanContext().traceId;
-    info.span_id = currentSpan?.spanContext().spanId;
-    console.log(JSON.stringify(info));
-
-    callback();
-  }
-}
+const tracingFormat = function () {
+  return winston.format((info) => {
+    const span = getCurrentSpan();
+    if (span) {
+      const { spanId, traceId } = span.spanContext();
+      const traceIdEnd = traceId.slice(traceId.length / 2);
+      info["dd.trace_id"] = BigInt(`0x${traceIdEnd}`).toString();
+      info["dd.span_id"] = BigInt(`0x${spanId}`).toString();
+      info["trace_id"] = traceId;
+      info["span_id"] = spanId;
+    }
+    return info;
+  })();
+};

 const getWinstonLogger = (
  nodeEnv: "development" | "production" | "test",
@@ -39,19 +34,16 @@ const getWinstonLogger = (
  const jsonLoggerFormat = winston.format.combine(
    winston.format.errors({ stack: true }),
    winston.format.timestamp(),
+    tracingFormat(),
    winston.format.json(),
  );

  const format =
    env.LANGFUSE_LOG_FORMAT === "text" ? textLoggerFormat : jsonLoggerFormat;
-  const transport =
-    env.LANGFUSE_LOG_FORMAT === "text"
-      ? new winston.transports.Console()
-      : new TracedTransport();
  return winston.createLogger({
    level: minLevel,
    format: format,
-    transports: [transport],
+    transports: [new winston.transports.Console()],
  });
 };

@@ -19,7 +19,6 @@ interface BatchExportSuccessTemplateProps {
  receiverEmail: string;
  downloadLink: string;
  batchExportName: string;
-  expiresInHours: number;
 }

 export const BatchExportSuccessEmailTemplate = ({
@@ -27,7 +26,6 @@ export const BatchExportSuccessEmailTemplate = ({
  downloadLink,
  userName,
  batchExportName,
-  expiresInHours,
 }: BatchExportSuccessTemplateProps) => {
  const previewText = `Download your data export from Langfuse`;

@@ -56,8 +54,7 @@ export const BatchExportSuccessEmailTemplate = ({
            <Text className="text-sm leading-6 text-black">
              Your data export{" "}
              <span className="font-mono">{batchExportName}</span> is ready to
-              download. The download link will expire in{" "}
-              <strong>{expiresInHours} hours</strong>.
+              download. The download link is valid for a few hours.
            </Text>
            <Text className="text-sm leading-6 text-black">
              Please note data exports do not reflect custom column ordering or
@@ -13,7 +13,6 @@ type SendBatchExportSuccessParams = {
  downloadLink: string;
  userName: string;
  batchExportName: string;
-  expiresInHours: number;
 };

 export const sendBatchExportSuccessEmail = async ({
@@ -22,7 +21,6 @@ export const sendBatchExportSuccessEmail = async ({
  downloadLink,
  userName,
  batchExportName,
-  expiresInHours,
 }: SendBatchExportSuccessParams) => {
  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
    logger.error("Missing environment variables for sending email.");
@@ -37,8 +35,7 @@ export const sendBatchExportSuccessEmail = async ({
        downloadLink,
        userName,
        batchExportName,
-        expiresInHours,
-      }),
+      })
    );

    await mailer.sendMail({
@@ -36,7 +36,7 @@ export const sendMembershipInvitationEmail = async ({
 }: SendMembershipInvitationParams) => {
  if (!env.EMAIL_FROM_ADDRESS || !env.SMTP_CONNECTION_URL) {
    logger.error(
-      "Missing environment variables for sending membership invitation email.",
+      "Missing environment variables for sending membership invitation email."
    );
    return;
  }
@@ -51,7 +51,7 @@ export const sendMembershipInvitationEmail = async ({
  const authUrl = getAuthURL();
  if (!authUrl) {
    logger.error(
-      "Missing NEXTAUTH_URL or NEXT_PUBLIC_LANGFUSE_CLOUD_REGION environment variable.",
+      "Missing NEXTAUTH_URL or NEXT_PUBLIC_LANGFUSE_CLOUD_REGION environment variable."
    );
    return;
  }
@@ -68,7 +68,7 @@ export const sendMembershipInvitationEmail = async ({
        inviteLink: authUrl,
        emailFromAddress: env.EMAIL_FROM_ADDRESS,
        langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
-      }),
+      })
    );

    await mailer.sendMail({
@@ -106,7 +106,7 @@ RUN apk add --no-cache dumb-init
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs

-RUN npm install -g --no-package-lock --no-save prisma@5.13.0
+RUN npm install -g --no-package-lock --no-save prisma@5.20.0

 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .
@@ -50,8 +50,8 @@ const nextConfig = {
    instrumentationHook: true,
    serverComponentsExternalPackages: [
      "dd-trace",
-      "@opentelemetry/auto-instrumentations-node",
      "@opentelemetry/api",
+      "@appsignal/opentelemetry-instrumentation-bullmq",
    ],
  },
  poweredByHeader: false,
@@ -105,22 +105,22 @@ const nextConfig = {
      // Required to check authentication status from langfuse.com
      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
        ? [
-          {
-            source: "/api/auth/session",
-            headers: [
-              {
-                key: "Access-Control-Allow-Origin",
-                value: "https://langfuse.com",
-              },
-              { key: "Access-Control-Allow-Credentials", value: "true" },
-              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-              {
-                key: "Access-Control-Allow-Headers",
-                value: "Content-Type, Authorization",
-              },
-            ],
-          },
-        ]
+            {
+              source: "/api/auth/session",
+              headers: [
+                {
+                  key: "Access-Control-Allow-Origin",
+                  value: "https://langfuse.com",
+                },
+                { key: "Access-Control-Allow-Credentials", value: "true" },
+                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+                {
+                  key: "Access-Control-Allow-Headers",
+                  value: "Content-Type, Authorization",
+                },
+              ],
+            },
+          ]
        : []),
      // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
      {
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "2.81.2",
+  "version": "2.83.1",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -23,7 +23,7 @@
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
-    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.1",
+    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.2",
    "@baselime/trpc-opentelemetry-middleware": "^0.1.2",
    "@codemirror/lang-json": "^6.0.1",
    "@codemirror/lint": "^6.8.0",
@@ -36,69 +36,66 @@
    "@heroicons/react": "^2.1.3",
    "@hookform/resolvers": "^3.3.4",
    "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.3",
+    "@langchain/core": "^0.3.7",
    "@langchain/openai": "^0.3.0",
    "@langfuse/ee": "workspace:*",
    "@langfuse/shared": "workspace:*",
    "@marsidev/react-turnstile": "^0.5.4",
-    "@mui/x-tree-view": "^7.6.2",
+    "@mui/x-tree-view": "^7.19.0",
    "@next-auth/prisma-adapter": "^1.0.7",
-    "@opentelemetry/api": ">=1.0.0 <1.9.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.44.0",
-    "@opentelemetry/context-async-hooks": "^1.25.1",
-    "@opentelemetry/core": "^1.14.0",
-    "@opentelemetry/instrumentation": "^0.52.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/core": "^1.26.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.53.0",
+    "@opentelemetry/instrumentation": "^0.53.0",
    "@opentelemetry/instrumentation-aws-sdk": "^0.44.0",
-    "@opentelemetry/instrumentation-http": "^0.52.1",
-    "@opentelemetry/instrumentation-ioredis": "^0.42.0",
-    "@opentelemetry/instrumentation-undici": "^0.4.0",
-    "@opentelemetry/instrumentation-winston": "^0.36.0",
-    "@opentelemetry/resources": "^1.25.1",
-    "@opentelemetry/sdk-metrics": "1.23.0",
-    "@opentelemetry/sdk-node": "^0.50.0",
-    "@opentelemetry/sdk-trace-base": "^1.25.1",
-    "@opentelemetry/sdk-trace-node": "^1.25.1",
-    "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@opentelemetry/winston-transport": "^0.2.0",
-    "@prisma/instrumentation": "^5.13.0",
-    "@radix-ui/react-accordion": "^1.2.0",
-    "@radix-ui/react-alert-dialog": "^1.1.1",
-    "@radix-ui/react-avatar": "^1.1.0",
-    "@radix-ui/react-checkbox": "^1.1.1",
-    "@radix-ui/react-collapsible": "^1.1.0",
-    "@radix-ui/react-dialog": "^1.1.1",
-    "@radix-ui/react-dropdown-menu": "^2.1.1",
-    "@radix-ui/react-hover-card": "^1.1.1",
+    "@opentelemetry/instrumentation-http": "^0.53.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.43.0",
+    "@opentelemetry/instrumentation-winston": "^0.40.0",
+    "@opentelemetry/resource-detector-aws": "^1.6.1",
+    "@opentelemetry/resource-detector-container": "^0.4.1",
+    "@opentelemetry/resources": "^1.26.0",
+    "@opentelemetry/sdk-node": "^0.53.0",
+    "@opentelemetry/sdk-trace-base": "^1.26.0",
+    "@opentelemetry/sdk-trace-node": "^1.26.0",
+    "@prisma/instrumentation": "^5.20.0",
+    "@radix-ui/react-accordion": "^1.2.1",
+    "@radix-ui/react-alert-dialog": "^1.1.2",
+    "@radix-ui/react-avatar": "^1.1.1",
+    "@radix-ui/react-checkbox": "^1.1.2",
+    "@radix-ui/react-collapsible": "^1.1.1",
+    "@radix-ui/react-dialog": "^1.1.2",
+    "@radix-ui/react-dropdown-menu": "^2.1.2",
+    "@radix-ui/react-hover-card": "^1.1.2",
    "@radix-ui/react-icons": "^1.3.0",
    "@radix-ui/react-label": "^2.1.0",
-    "@radix-ui/react-popover": "^1.1.1",
-    "@radix-ui/react-scroll-area": "^1.1.0",
-    "@radix-ui/react-select": "^2.1.1",
+    "@radix-ui/react-popover": "^1.1.2",
+    "@radix-ui/react-scroll-area": "^1.2.0",
+    "@radix-ui/react-select": "^2.1.2",
    "@radix-ui/react-separator": "^1.1.0",
-    "@radix-ui/react-slider": "^1.2.0",
+    "@radix-ui/react-slider": "^1.2.1",
    "@radix-ui/react-slot": "^1.1.0",
-    "@radix-ui/react-switch": "^1.1.0",
-    "@radix-ui/react-tabs": "^1.1.0",
+    "@radix-ui/react-switch": "^1.1.1",
+    "@radix-ui/react-tabs": "^1.1.1",
    "@radix-ui/react-toggle": "^1.1.0",
    "@radix-ui/react-toggle-group": "^1.1.0",
-    "@radix-ui/react-tooltip": "^1.1.2",
+    "@radix-ui/react-tooltip": "^1.1.3",
    "@remixicon/react": "^4.2.0",
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "8.26.0",
-    "@t3-oss/env-nextjs": "^0.10.1",
+    "@sentry/nextjs": "^8.33.1",
+    "@t3-oss/env-nextjs": "^0.11.1",
    "@tailwindcss/container-queries": "^0.1.1",
    "@tanstack/react-query": "^4.36.1",
-    "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-table": "^8.20.5",
    "@tremor/react": "3.16.2",
    "@trpc/client": "^10.45.0",
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
    "@uiw/codemirror-theme-github": "^4.23.0",
-    "@uiw/codemirror-theme-tokyo-night": "^4.22.2",
+    "@uiw/codemirror-theme-tokyo-night": "^4.23.5",
    "@uiw/react-codemirror": "^4.21.25",
-    "ai": "^3.0.23",
+    "ai": "^3.4.9",
    "bcryptjs": "^2.4.3",
    "bullmq": "^5.12.10",
    "class-variance-authority": "^0.7.0",
@@ -107,30 +104,30 @@
    "core-js": "^3.38.1",
    "cors": "^2.8.5",
    "date-fns": "^3.3.1",
-    "dd-trace": "^5.21.0",
+    "dd-trace": "^5.23.1",
    "decimal.js": "^10.4.3",
    "dompurify": "^3.1.5",
    "graphql": "^16.9.0",
    "ioredis": "^5.4.1",
    "ip-address": "^9.0.5",
-    "js-tiktoken": "^1.0.12",
+    "js-tiktoken": "^1.0.15",
    "kysely": "^0.27.4",
    "langchain": "^0.3.2",
    "lodash": "^4.17.21",
-    "lucide-react": "^0.436.0",
+    "lucide-react": "^0.447.0",
    "next": "^14.2.6",
    "next-auth": "^4.24.7",
-    "next-query-params": "^5.0.0",
+    "next-query-params": "^5.0.1",
    "next-themes": "^0.3.0",
    "posthog-js": "^1.161.3",
    "posthog-node": "^3.6.3",
    "prexit": "^2.2.0",
-    "prisma": "^5.13.0",
+    "prisma": "^5.20.0",
    "rate-limiter-flexible": "^5.0.3",
    "react": "18.2.0",
    "react-day-picker": "^8.10.1",
    "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.5",
+    "react-hook-form": "^7.53.0",
    "react-icons": "^5.2.1",
    "react-markdown": "^9.0.1",
    "react-resizable-panels": "^2.1.1",
@@ -151,8 +148,8 @@
  },
  "devDependencies": {
    "@jedmao/location": "^3.0.0",
-    "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.43.1",
+    "@mermaid-js/mermaid-cli": "^11.2.0",
+    "@playwright/test": "^1.47.2",
    "@tailwindcss/forms": "^0.5.7",
    "@testing-library/jest-dom": "^6.4.6",
    "@testing-library/react": "^15.0.7",
@@ -161,7 +158,7 @@
    "@types/dompurify": "^3.0.5",
    "@types/eslint": "^8.56.7",
    "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
    "@types/node": "20.10.5",
    "@types/react": "~18.2.79",
    "@types/react-dom": "~18.2.25",
@@ -177,12 +174,12 @@
    "jest-environment-jsdom": "^29.7.0",
    "node-mocks-http": "^1.14.1",
    "postcss": "^8.4.38",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.3",
    "prettier-plugin-tailwindcss": "^0.6.6",
-    "tailwindcss": "^3.4.10",
+    "tailwindcss": "^3.4.13",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.19.0",
+    "tsx": "^4.19.1",
    "typescript": "^5.4.5",
    "wait-for-expect": "^3.0.2"
  },
@@ -39,6 +39,8 @@ describe("RateLimitService", () => {
      rateLimitOverrides: [],
    };

+    expect(redis).toBeDefined();
+
    const rateLimitService = new RateLimitService(redis!);
    const result = await rateLimitService.rateLimitRequest(scope, "public-api");

@@ -93,7 +93,7 @@ const Base = (props: {
          "Loading.."
        ) : props.isPublic ? (
          <div
-            className="text-dark-green flex cursor-pointer items-center gap-1"
+            className="flex cursor-pointer items-center gap-1 text-dark-green"
            onClick={() => copyUrl()}
          >
            {isCopied ? "Link copied ..." : "Public"}
@@ -376,9 +376,9 @@ export default function GenerationsTable({
      enableSorting: true,
    },
    {
-      accessorKey: "timePerOutputToken",
-      id: "timePerOutputToken",
-      header: "Time per Output Token",
+      accessorKey: "tokensPerSecond",
+      id: "tokensPerSecond",
+      header: "Tokens per second",
      size: 200,
      cell: ({ row }) => {
        const latency: number | undefined = row.getValue("latency");
@@ -390,9 +390,9 @@ export default function GenerationsTable({
        return latency !== undefined &&
          (usage.completionTokens !== 0 || usage.totalTokens !== 0) ? (
          <span>
-            {usage.completionTokens
-              ? formatIntervalSeconds(latency / usage.completionTokens)
-              : formatIntervalSeconds(latency / usage.totalTokens)}
+            {usage.completionTokens && latency
+              ? Number((usage.completionTokens / latency).toFixed(1))
+              : undefined}
          </span>
        ) : undefined;
      },
@@ -3,7 +3,7 @@ import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { numberFormatter } from "@/src/utils/numbers";
 import { type Observation } from "@langfuse/shared";

-export const TraceAggUsageBadge = (props: {
+export const AggUsageBadge = (props: {
  observations: ObservationReturnType[];
 }) => {
  const usage = {
@@ -15,7 +15,6 @@ import { IOPreview } from "@/src/components/trace/IOPreview";
 import { formatIntervalSeconds } from "@/src/utils/dates";
 import Link from "next/link";
 import { usdFormatter } from "@/src/utils/numbers";
-import { calculateDisplayTotalCost } from "@/src/components/trace";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { withDefault, StringParam, useQueryParam } from "use-query-params";
 import ScoresTable from "@/src/components/table/use-cases/scores";
@@ -24,6 +23,8 @@ import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/compon
 import { AnnotateDrawer } from "@/src/features/scores/components/AnnotateDrawer";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { CommentDrawerButton } from "@/src/features/comments/CommentDrawerButton";
+import { calculateDisplayTotalCost } from "@/src/components/trace/lib/helpers";
+import { useMemo } from "react";

 export const ObservationPreview = (props: {
  observations: Array<ObservationReturnType>;
@@ -51,8 +52,19 @@ export const ObservationPreview = (props: {
    (o) => o.id === props.currentObservationId,
  );

-  const totalCost = calculateDisplayTotalCost(
-    preloadedObservation ? [preloadedObservation] : [],
+  const thisCost = preloadedObservation
+    ? calculateDisplayTotalCost({
+        allObservations: [preloadedObservation],
+      })
+    : undefined;
+
+  const totalCost = useMemo(
+    () =>
+      calculateDisplayTotalCost({
+        allObservations: props.observations,
+        rootObservationId: props.currentObservationId,
+      }),
+    [props.observations, props.currentObservationId],
  );

  if (!preloadedObservation) return <div className="flex-1">Not found</div>;
@@ -142,9 +154,14 @@ export const ObservationPreview = (props: {
              {preloadedObservation.model ? (
                <Badge variant="outline">{preloadedObservation.model}</Badge>
              ) : null}
-              {totalCost ? (
+              {thisCost ? (
                <Badge variant="outline">
-                  {usdFormatter(totalCost.toNumber())}
+                  {usdFormatter(thisCost.toNumber())}
+                </Badge>
+              ) : undefined}
+              {totalCost && totalCost !== thisCost ? (
+                <Badge variant="outline">
+                  ∑ {usdFormatter(totalCost.toNumber())}
                </Badge>
              ) : undefined}

@@ -2,7 +2,7 @@ import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
 import { type APIScore, type Trace, type $Enums } from "@langfuse/shared";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
-import { Fragment } from "react";
+import { Fragment, useMemo } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { LevelColors } from "@/src/components/level-colors";
 import { formatIntervalSeconds } from "@/src/utils/dates";
@@ -11,10 +11,13 @@ import { Toggle } from "@/src/components/ui/toggle";
 import { Button } from "@/src/components/ui/button";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import {
+  calculateDisplayTotalCost,
  nestObservations,
  treeItemColors,
 } from "@/src/components/trace/lib/helpers";
 import { CommentCountIcon } from "@/src/features/comments/CommentCountIcon";
+import { usdFormatter } from "@/src/utils/numbers";
+import Decimal from "decimal.js";

 export const ObservationTree = (props: {
  observations: ObservationReturnType[];
@@ -22,7 +25,7 @@ export const ObservationTree = (props: {
  toggleCollapsedObservation: (id: string) => void;
  collapseAll: () => void;
  expandAll: () => void;
-  trace: Trace;
+  trace: Trace & { latency?: number };
  scores: APIScore[];
  currentObservationId: string | undefined;
  setCurrentObservationId: (id: string | undefined) => void;
@@ -32,7 +35,16 @@ export const ObservationTree = (props: {
  traceCommentCounts?: Map<string, number>;
  className?: string;
 }) => {
-  const nestedObservations = nestObservations(props.observations);
+  const nestedObservations = useMemo(
+    () => nestObservations(props.observations),
+    [props.observations],
+  );
+  const totalCost = useMemo(() => {
+    return calculateDisplayTotalCost({
+      allObservations: props.observations,
+    });
+  }, [props.observations]);
+
  return (
    <div className={props.className}>
      <ObservationTreeTraceNode
@@ -45,6 +57,7 @@ export const ObservationTree = (props: {
        setCurrentObservationId={props.setCurrentObservationId}
        showMetrics={props.showMetrics}
        showScores={props.showScores}
+        totalCost={totalCost}
      />
      <ObservationTreeNode
        observations={nestedObservations}
@@ -57,6 +70,11 @@ export const ObservationTree = (props: {
        setCurrentObservationId={props.setCurrentObservationId}
        showMetrics={props.showMetrics}
        showScores={props.showScores}
+        colorCodeMetrics={props.observations.length >= 3}
+        parentTotalCost={totalCost}
+        parentTotalDuration={
+          props.trace.latency ? props.trace.latency * 1000 : undefined
+        }
      />
    </div>
  );
@@ -72,61 +90,72 @@ const ObservationTreeTraceNode = (props: {
  setCurrentObservationId: (id: string | undefined) => void;
  showMetrics?: boolean;
  showScores?: boolean;
-}) => (
-  <div
-    className={cn(
-      "group mb-0.5 flex cursor-pointer flex-col gap-1 rounded-sm p-1",
-      props.currentObservationId === undefined ||
-        props.currentObservationId === ""
-        ? "bg-muted"
-        : "hover:bg-primary-foreground",
-    )}
-    onClick={() => props.setCurrentObservationId(undefined)}
-  >
-    <div className="flex gap-2">
-      <span className={cn("rounded-sm bg-input px-1 py-0.5 text-xs")}>
-        TRACE
-      </span>
-      <span className="break-all text-sm">{props.trace.name}</span>
-      {props.comments ? (
-        <CommentCountIcon count={props.comments.get(props.trace.id)} />
-      ) : null}
-      <div className="flex flex-1 justify-end">
-        <Button
-          onClick={(ev) => (ev.stopPropagation(), props.expandAll())}
-          size="xs"
-          variant="ghost"
-          title="Expand all"
-        >
-          <PlusCircleIcon className="h-4 w-4" />
-        </Button>
-        <Button
-          onClick={(ev) => (ev.stopPropagation(), props.collapseAll())}
-          size="xs"
-          variant="ghost"
-          title="Collapse all"
-        >
-          <MinusCircle className="h-4 w-4" />
-        </Button>
-      </div>
-    </div>
-
-    {props.showMetrics && props.trace.latency ? (
+  totalCost?: Decimal;
+}) => {
+  return (
+    <div
+      className={cn(
+        "group mb-0.5 flex cursor-pointer flex-col gap-1 rounded-sm p-1",
+        props.currentObservationId === undefined ||
+          props.currentObservationId === ""
+          ? "bg-muted"
+          : "hover:bg-primary-foreground",
+      )}
+      onClick={() => props.setCurrentObservationId(undefined)}
+    >
      <div className="flex gap-2">
-        <span className="text-xs text-muted-foreground">
-          {formatIntervalSeconds(props.trace.latency)}
+        <span className={cn("rounded-sm bg-input px-1 py-0.5 text-xs")}>
+          TRACE
        </span>
+        <span className="break-all text-sm">{props.trace.name}</span>
+        {props.comments ? (
+          <CommentCountIcon count={props.comments.get(props.trace.id)} />
+        ) : null}
+        <div className="flex flex-1 justify-end">
+          <Button
+            onClick={(ev) => (ev.stopPropagation(), props.expandAll())}
+            size="xs"
+            variant="ghost"
+            title="Expand all"
+          >
+            <PlusCircleIcon className="h-4 w-4" />
+          </Button>
+          <Button
+            onClick={(ev) => (ev.stopPropagation(), props.collapseAll())}
+            size="xs"
+            variant="ghost"
+            title="Collapse all"
+          >
+            <MinusCircle className="h-4 w-4" />
+          </Button>
+        </div>
      </div>
-    ) : null}
-    {props.showScores && props.scores.find((s) => s.observationId === null) ? (
-      <div className="flex flex-wrap gap-1">
-        <GroupedScoreBadges
-          scores={props.scores.filter((s) => s.observationId === null)}
-        />
-      </div>
-    ) : null}
-  </div>
-);
+
+      {props.showMetrics && (
+        <div className="flex gap-2">
+          {props.trace.latency ? (
+            <span className="text-xs text-muted-foreground">
+              {formatIntervalSeconds(props.trace.latency)}
+            </span>
+          ) : null}
+          {props.totalCost ? (
+            <span className="text-xs text-muted-foreground">
+              {usdFormatter(props.totalCost.toNumber())}
+            </span>
+          ) : null}
+        </div>
+      )}
+      {props.showScores &&
+      props.scores.find((s) => s.observationId === null) ? (
+        <div className="flex flex-wrap gap-1">
+          <GroupedScoreBadges
+            scores={props.scores.filter((s) => s.observationId === null)}
+          />
+        </div>
+      ) : null}
+    </div>
+  );
+};

 const ObservationTreeNode = (props: {
  observations: NestedObservation[];
@@ -139,6 +168,9 @@ const ObservationTreeNode = (props: {
  setCurrentObservationId: (id: string | undefined) => void;
  showMetrics?: boolean;
  showScores?: boolean;
+  colorCodeMetrics?: boolean;
+  parentTotalCost?: Decimal;
+  parentTotalDuration?: number;
 }) => {
  const capture = usePostHogClientCapture();
  return (
@@ -149,6 +181,13 @@ const ObservationTreeNode = (props: {
          const collapsed = props.collapsedObservations.includes(
            observation.id,
          );
+          const unnestedObservations = unnestObservations(observation);
+          const totalCost = calculateDisplayTotalCost({
+            allObservations: unnestedObservations,
+          });
+          const duration = observation.endTime
+            ? observation.endTime.getTime() - observation.startTime.getTime()
+            : undefined;
          return (
            <Fragment key={observation.id}>
              <div className="flex">
@@ -210,15 +249,22 @@ const ObservationTreeNode = (props: {
                    (observation.promptTokens ||
                      observation.completionTokens ||
                      observation.totalTokens ||
-                      observation.endTime) && (
+                      duration ||
+                      totalCost) && (
                      <div className="flex gap-2">
-                        {observation.endTime ? (
-                          <span className="text-xs text-muted-foreground">
-                            {formatIntervalSeconds(
-                              (observation.endTime.getTime() -
-                                observation.startTime.getTime()) /
-                                1000,
+                        {duration ? (
+                          <span
+                            className={cn(
+                              "text-xs text-muted-foreground",
+                              props.parentTotalDuration &&
+                                props.colorCodeMetrics &&
+                                heatMapTextColor({
+                                  max: props.parentTotalDuration,
+                                  value: duration,
+                                }),
                            )}
+                          >
+                            {formatIntervalSeconds(duration / 1000)}
                          </span>
                        ) : null}
                        {observation.promptTokens ||
@@ -230,6 +276,21 @@ const ObservationTreeNode = (props: {
                            {observation.totalTokens})
                          </span>
                        ) : null}
+                        {totalCost ? (
+                          <span
+                            className={cn(
+                              "text-xs text-muted-foreground",
+                              props.parentTotalCost &&
+                                props.colorCodeMetrics &&
+                                heatMapTextColor({
+                                  max: props.parentTotalCost,
+                                  value: totalCost,
+                                }),
+                            )}
+                          >
+                            {usdFormatter(totalCost.toNumber())}
+                          </span>
+                        ) : null}
                      </div>
                    )}
                  {observation.level !== "DEFAULT" ? (
@@ -271,6 +332,9 @@ const ObservationTreeNode = (props: {
                  setCurrentObservationId={props.setCurrentObservationId}
                  showMetrics={props.showMetrics}
                  showScores={props.showScores}
+                  colorCodeMetrics={props.colorCodeMetrics}
+                  parentTotalCost={props.parentTotalCost}
+                  parentTotalDuration={props.parentTotalDuration}
                />
              )}
            </Fragment>
@@ -294,3 +358,41 @@ const ColorCodedObservationType = (props: {
    </span>
  );
 };
+
+const unnestObservations = (nestedObservation: NestedObservation) => {
+  const unnestedObservations = [];
+  const { children, ...observation } = nestedObservation;
+  unnestedObservations.push(observation);
+  children.forEach((child) => {
+    unnestedObservations.push(...unnestObservations(child));
+  });
+  return unnestedObservations;
+};
+
+const heatMapTextColor = (p: {
+  min?: Decimal | number;
+  max: Decimal | number;
+  value: Decimal | number;
+}) => {
+  const { min, max, value } = p;
+  const minDecimal = min ? new Decimal(min) : new Decimal(0);
+  const maxDecimal = new Decimal(max);
+  const valueDecimal = new Decimal(value);
+
+  const cutOffs: [number, string][] = [
+    [0.75, "text-dark-red"], // 75%
+    [0.5, "text-dark-yellow"], // 50%
+  ];
+  const standardizedValueOnStartEndScale = valueDecimal
+    .sub(minDecimal)
+    .div(maxDecimal.sub(minDecimal));
+  const ratio = standardizedValueOnStartEndScale.toNumber();
+
+  // pick based on ratio if threshold is exceeded
+  for (const [threshold, color] of cutOffs) {
+    if (ratio >= threshold) {
+      return color;
+    }
+  }
+  return "";
+};
@@ -8,7 +8,7 @@ import {
  CardHeader,
  CardTitle,
 } from "@/src/components/ui/card";
-import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
+import { AggUsageBadge } from "@/src/components/token-usage-badge";
 import { Badge } from "@/src/components/ui/badge";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { IOPreview } from "@/src/components/trace/IOPreview";
@@ -21,6 +21,9 @@ import { ScoresPreview } from "@/src/components/trace/ScoresPreview";
 import { AnnotateDrawer } from "@/src/features/scores/components/AnnotateDrawer";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { CommentDrawerButton } from "@/src/features/comments/CommentDrawerButton";
+import { useMemo } from "react";
+import { usdFormatter } from "@/src/utils/numbers";
+import { calculateDisplayTotalCost } from "@/src/components/trace/lib/helpers";

 export const TracePreview = ({
  trace,
@@ -50,6 +53,14 @@ export const TracePreview = ({
    return acc;
  }, new Map<ScoreSource, APIScore[]>());

+  const totalCost = useMemo(
+    () =>
+      calculateDisplayTotalCost({
+        allObservations: observations,
+      }),
+    [observations],
+  );
+
  return (
    <Card className="col-span-2 flex max-h-full flex-col overflow-hidden">
      <div className="flex flex-shrink-0 flex-row justify-end gap-2">
@@ -92,13 +103,18 @@ export const TracePreview = ({
                  {formatIntervalSeconds(trace.latency)}
                </Badge>
              )}
-              <TraceAggUsageBadge observations={observations} />
+              <AggUsageBadge observations={observations} />
              {!!trace.release && (
                <Badge variant="outline">Release: {trace.release}</Badge>
              )}
              {!!trace.version && (
                <Badge variant="outline">Version: {trace.version}</Badge>
              )}
+              {totalCost && (
+                <Badge variant="outline">
+                  ∑ {usdFormatter(totalCost.toNumber())}
+                </Badge>
+              )}
            </div>
          </div>
          <div className="flex flex-wrap gap-2">
@@ -5,7 +5,7 @@ import { TracePreview } from "./TracePreview";

 import Header from "@/src/components/layouts/header";
 import { Badge } from "@/src/components/ui/badge";
-import { TraceAggUsageBadge } from "@/src/components/token-usage-badge";
+import { AggUsageBadge } from "@/src/components/token-usage-badge";
 import { StringParam, useQueryParam, withDefault } from "use-query-params";
 import { PublishTraceSwitch } from "@/src/components/publish-object-switch";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
@@ -26,7 +26,6 @@ import {
  Network,
 } from "lucide-react";
 import { usdFormatter } from "@/src/utils/numbers";
-import Decimal from "decimal.js";
 import { useCallback, useState } from "react";
 import { DeleteButton } from "@/src/components/deleteButton";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -35,6 +34,7 @@ import { TraceTimelineView } from "@/src/components/trace/TraceTimelineView";
 import { type APIScore } from "@langfuse/shared";
 import { useSession } from "next-auth/react";
 import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import { calculateDisplayTotalCost } from "@/src/components/trace/lib/helpers";

 export function Trace(props: {
  observations: Array<ObservationReturnType>;
@@ -253,7 +253,9 @@ export function TracePage({ traceId }: { traceId: string }) {
  const filterOptionTags = traceFilterOptions.data?.tags ?? [];
  const allTags = filterOptionTags.map((t) => t.value);

-  const totalCost = calculateDisplayTotalCost(trace.data?.observations ?? []);
+  const totalCost = calculateDisplayTotalCost({
+    allObservations: trace.data?.observations ?? [],
+  });

  const [selectedTab, setSelectedTab] = useQueryParam(
    "display",
@@ -331,11 +333,9 @@ export function TracePage({ traceId }: { traceId: string }) {
            <Badge>User ID: {trace.data.userId}</Badge>
          </Link>
        ) : null}
-        <TraceAggUsageBadge observations={trace.data.observations} />
+        <AggUsageBadge observations={trace.data.observations} />
        {totalCost ? (
-          <Badge variant="outline">
-            Total cost: {usdFormatter(totalCost.toNumber())}
-          </Badge>
+          <Badge variant="outline">{usdFormatter(totalCost.toNumber())}</Badge>
        ) : undefined}
      </div>
      <div className="mt-3 rounded-lg border bg-card font-semibold text-card-foreground">
@@ -347,6 +347,7 @@ export function TracePage({ traceId }: { traceId: string }) {
            traceId={trace.data.id}
            projectId={trace.data.projectId}
            className="flex-wrap"
+            key={trace.data.id}
          />
        </div>
      </div>
@@ -400,42 +401,3 @@ export function TracePage({ traceId }: { traceId: string }) {
    </FullScreenPage>
  );
 }
-
-export const calculateDisplayTotalCost = (
-  observations: ObservationReturnType[],
-) => {
-  return observations.reduce(
-    (prev: Decimal | undefined, curr: ObservationReturnType) => {
-      // if we don't have any calculated costs, we can't do anything
-      if (
-        !curr.calculatedTotalCost &&
-        !curr.calculatedInputCost &&
-        !curr.calculatedOutputCost
-      )
-        return prev;
-
-      // if we have either input or output cost, but not total cost, we can use that
-      if (
-        !curr.calculatedTotalCost &&
-        (curr.calculatedInputCost || curr.calculatedOutputCost)
-      ) {
-        return prev
-          ? prev.plus(
-              curr.calculatedInputCost ??
-                new Decimal(0).plus(
-                  curr.calculatedOutputCost ?? new Decimal(0),
-                ),
-            )
-          : curr.calculatedInputCost ?? curr.calculatedOutputCost ?? undefined;
-      }
-
-      if (!curr.calculatedTotalCost) return prev;
-
-      // if we have total cost, we can use that
-      return prev
-        ? prev.plus(curr.calculatedTotalCost)
-        : curr.calculatedTotalCost;
-    },
-    undefined,
-  );
-};
@@ -1,13 +1,14 @@
-import { $Enums } from "@langfuse/shared";
+import { ObservationType } from "@langfuse/shared";
 import { type NestedObservation } from "@/src/utils/types";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
+import Decimal from "decimal.js";

-export type TreeItemType = $Enums.ObservationType | "TRACE";
+export type TreeItemType = ObservationType | "TRACE";

 export const treeItemColors: Map<TreeItemType, string> = new Map([
-  [$Enums.ObservationType.SPAN, "bg-muted-blue"],
-  [$Enums.ObservationType.GENERATION, "bg-muted-orange"],
-  [$Enums.ObservationType.EVENT, "bg-muted-green"],
+  [ObservationType.SPAN, "bg-muted-blue"],
+  [ObservationType.GENERATION, "bg-muted-orange"],
+  [ObservationType.EVENT, "bg-muted-green"],
  ["TRACE", "bg-input"],
 ]);

@@ -59,8 +60,72 @@ export function nestObservations(
    obj.children.sort((a, b) => a.startTime.getTime() - b.startTime.getTime());
  }

-  // TODO sum token amounts per level
-
  // Step 5: Return the roots.
  return Array.from(roots.values());
 }
+
+export function calculateDisplayTotalCost(p: {
+  allObservations: ObservationReturnType[];
+  rootObservationId?: string;
+}): Decimal | undefined {
+  // if parentObservationId is provided, only calculate cost for children of that observation
+  // need to be checked recursively for all children and children of children
+  // loop until no more children to be added
+  let observations = p.allObservations;
+
+  if (p.rootObservationId) {
+    observations = observations.filter(
+      (o) =>
+        o.parentObservationId === p.rootObservationId ||
+        o.id === p.rootObservationId,
+    );
+
+    while (true) {
+      const childrenToAdd = p.allObservations.filter(
+        (o) =>
+          o.parentObservationId &&
+          !observations.map((o2) => o2.id).includes(o.id) &&
+          observations.map((o2) => o2.id).includes(o.parentObservationId),
+      );
+      if (childrenToAdd.length === 0) break;
+      observations = [...observations, ...childrenToAdd];
+    }
+  }
+
+  const totalCost = observations.reduce(
+    (prev: Decimal | undefined, curr: ObservationReturnType) => {
+      // if we don't have any calculated costs, we can't do anything
+      if (
+        !curr.calculatedTotalCost &&
+        !curr.calculatedInputCost &&
+        !curr.calculatedOutputCost
+      )
+        return prev;
+
+      // if we have either input or output cost, but not total cost, we can use that
+      if (
+        !curr.calculatedTotalCost &&
+        (curr.calculatedInputCost || curr.calculatedOutputCost)
+      ) {
+        return prev
+          ? prev.plus(
+              curr.calculatedInputCost ??
+                new Decimal(0).plus(
+                  curr.calculatedOutputCost ?? new Decimal(0),
+                ),
+            )
+          : curr.calculatedInputCost ?? curr.calculatedOutputCost ?? undefined;
+      }
+
+      if (!curr.calculatedTotalCost) return prev;
+
+      // if we have total cost, we can use that
+      return prev
+        ? prev.plus(curr.calculatedTotalCost)
+        : curr.calculatedTotalCost;
+    },
+    undefined,
+  );
+
+  return totalCost;
+}
@@ -1 +1 @@
-export const VERSION = "v2.81.2";
+export const VERSION = "v2.83.1";
@@ -1,66 +0,0 @@
-import { registerInstrumentations } from "@opentelemetry/instrumentation";
-import { IORedisInstrumentation } from "@opentelemetry/instrumentation-ioredis";
-import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
-import { PrismaInstrumentation } from "@prisma/instrumentation";
-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks";
-import dd from "dd-trace";
-import opentelemetry from "@opentelemetry/api";
-// import { BullMQInstrumentation } from "@appsignal/opentelemetry-instrumentation-bullmq";
-import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici";
-import { logger } from "@langfuse/shared/src/server";
-import { WinstonInstrumentation } from "@opentelemetry/instrumentation-winston";
-import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk";
-
-if (!process.env.VERCEL && process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
-  logger.info("Initializing otel tracing");
-  const contextManager = new AsyncHooksContextManager().enable();
-
-  opentelemetry.context.setGlobalContextManager(contextManager);
-
-  const tracer = dd.init({
-    runtimeMetrics: true,
-  });
-
-  const { TracerProvider } = tracer;
-
-  const provider = new TracerProvider();
-
-  // correct the ressrouce name for http requests
-  tracer.use("http", {
-    hooks: {
-      request(span, req) {
-        if (span && req) {
-          let url = "path" in req ? req.path : req.url;
-          if (url) {
-            // Remove URL parameters
-            url = url.split("?")[0];
-            // Add wildcard for /_next/static
-            if (url.startsWith("/_next/static")) {
-              url = "/_next/static/*";
-            }
-          }
-          if (url) {
-            const method = req.method;
-            span.setTag("resource.name", method ? `${method} ${url}` : url);
-          }
-        }
-      },
-    },
-  });
-
-  registerInstrumentations({
-    instrumentations: [
-      new IORedisInstrumentation(),
-      new HttpInstrumentation(),
-      new PrismaInstrumentation(),
-      new AwsInstrumentation(),
-      new WinstonInstrumentation({ disableLogSending: true }),
-      getNodeAutoInstrumentations(),
-      new UndiciInstrumentation(),
-      // new BullMQInstrumentation(),
-    ],
-  });
-
-  provider.register();
-}
@@ -28,6 +28,17 @@ export const GithubProviderSchema = base.extend({
    .nullish(),
 });

+export const GitlabProviderSchema = base.extend({
+  authProvider: z.literal("gitlab"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const Auth0ProviderSchema = base.extend({
  authProvider: z.literal("auth0"),
  authConfig: z
@@ -92,6 +103,7 @@ export const CustomProviderSchema = base.extend({

 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type GitlabProviderSchema = z.infer<typeof GitlabProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
@@ -101,6 +113,7 @@ export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
  GoogleProviderSchema,
  GithubProviderSchema,
+  GitlabProviderSchema,
  Auth0ProviderSchema,
  OktaProviderSchema,
  AzureAdProviderSchema,
@@ -1,6 +1,7 @@
 import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
+import GitLabProvider from "next-auth/providers/gitlab";
 import OktaProvider from "next-auth/providers/okta";
 import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
@@ -157,6 +158,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
      ...provider.authConfig,
      clientSecret: decrypt(provider.authConfig.clientSecret),
    });
+  else if (provider.authProvider === "gitlab")
+    return GitLabProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
  else if (provider.authProvider === "auth0")
    return Auth0Provider({
      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
@@ -55,6 +55,9 @@ export const env = createEnv({
    AUTH_GITHUB_CLIENT_ID: z.string().optional(),
    AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),
    AUTH_GITHUB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GITLAB_CLIENT_ID: z.string().optional(),
+    AUTH_GITLAB_CLIENT_SECRET: z.string().optional(),
+    AUTH_GITLAB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
    AUTH_AZURE_AD_CLIENT_ID: z.string().optional(),
    AUTH_AZURE_AD_CLIENT_SECRET: z.string().optional(),
    AUTH_AZURE_AD_TENANT_ID: z.string().optional(),
@@ -111,6 +114,10 @@ export const env = createEnv({
    LANGFUSE_WORKER_PASSWORD: z.string().optional(),
    TURNSTILE_SECRET_KEY: z.string().optional(),

+    // Otel
+    OTEL_EXPORTER_OTLP_ENDPOINT: z.string().default("http://localhost:4318"),
+    OTEL_SERVICE_NAME: z.string().default("web"),
+
    // clickhouse
    CLICKHOUSE_URL: z.string().optional(),
    CLICKHOUSE_USER: z.string().optional(),
@@ -122,7 +129,9 @@ export const env = createEnv({
    LANGFUSE_UI_FEEDBACK_HREF: z.string().url().optional(),
    LANGFUSE_UI_LOGO_LIGHT_MODE_HREF: z.string().url().optional(),
    LANGFUSE_UI_LOGO_DARK_MODE_HREF: z.string().url().optional(),
-    LANGFUSE_UI_DEFAULT_MODEL_ADAPTER: z.enum(["OpenAI", "Anthropic", "Azure"]).optional(),
+    LANGFUSE_UI_DEFAULT_MODEL_ADAPTER: z
+      .enum(["OpenAI", "Anthropic", "Azure"])
+      .optional(),
    LANGFUSE_UI_DEFAULT_BASE_URL_OPENAI: z.string().url().optional(),
    LANGFUSE_UI_DEFAULT_BASE_URL_ANTHROPIC: z.string().url().optional(),
    LANGFUSE_UI_DEFAULT_BASE_URL_AZURE: z.string().url().optional(),
@@ -186,15 +195,42 @@ export const env = createEnv({
    SENTRY_AUTH_TOKEN: z.string().optional(),
    SENTRY_CSP_REPORT_URI: z.string().optional(),
    LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),
-    LANGFUSE_INIT_ORG_ID: z.string().optional(),
-    LANGFUSE_INIT_ORG_NAME: z.string().optional(),
-    LANGFUSE_INIT_PROJECT_ID: z.string().optional(),
-    LANGFUSE_INIT_PROJECT_NAME: z.string().optional(),
-    LANGFUSE_INIT_PROJECT_PUBLIC_KEY: z.string().optional(),
-    LANGFUSE_INIT_PROJECT_SECRET_KEY: z.string().optional(),
-    LANGFUSE_INIT_USER_EMAIL: z.string().email().optional(),
-    LANGFUSE_INIT_USER_NAME: z.string().optional(),
-    LANGFUSE_INIT_USER_PASSWORD: z.string().optional(),
+    LANGFUSE_INIT_ORG_ID: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_ORG_NAME: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_PROJECT_ID: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_PROJECT_NAME: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_PROJECT_PUBLIC_KEY: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_PROJECT_SECRET_KEY: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_USER_EMAIL: z
+      .union([z.string().email(), z.string().length(0)])
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_USER_NAME: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
+    LANGFUSE_INIT_USER_PASSWORD: z
+      .string()
+      .optional()
+      .transform((v) => (v === "" ? undefined : v)),
  },

  /**
@@ -209,11 +245,27 @@ export const env = createEnv({

    // NEXT_PUBLIC_CLIENTVAR: z.string().min(1),
    NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z
-      .enum(["US", "EU", "STAGING", "DEV"])
-      .optional(),
+      .string()
+      .optional()
+      .transform((v) => {
+        // for some reason, empty strings are not being transformed to undefined
+        if (v === undefined) return undefined;
+        if (v === "") return undefined;
+        return v;
+      })
+      .pipe(z.enum(["US", "EU", "STAGING", "DEV"]).optional()),
    NEXT_PUBLIC_DEMO_PROJECT_ID: z.string().optional(),
    NEXT_PUBLIC_DEMO_ORG_ID: z.string().optional(),
-    NEXT_PUBLIC_SIGN_UP_DISABLED: z.enum(["true", "false"]).optional(),
+    NEXT_PUBLIC_SIGN_UP_DISABLED: z
+      .string()
+      .optional()
+      .transform((v) => {
+        // for some reason, empty strings are not being transformed to undefined
+        if (v === undefined) return undefined;
+        if (v === "") return undefined;
+        return v;
+      })
+      .pipe(z.enum(["true", "false"]).optional()),
    NEXT_PUBLIC_TURNSTILE_SITE_KEY: z.string().optional(),
    NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(),
    NEXT_PUBLIC_POSTHOG_HOST: z.string().optional(),
@@ -263,6 +315,10 @@ export const env = createEnv({
    AUTH_GITHUB_CLIENT_SECRET: process.env.AUTH_GITHUB_CLIENT_SECRET,
    AUTH_GITHUB_ALLOW_ACCOUNT_LINKING:
      process.env.AUTH_GITHUB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITLAB_CLIENT_ID: process.env.AUTH_GITLAB_CLIENT_ID,
+    AUTH_GITLAB_CLIENT_SECRET: process.env.AUTH_GITLAB_CLIENT_SECRET,
+    AUTH_GITLAB_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GITLAB_ALLOW_ACCOUNT_LINKING,
    AUTH_AZURE_AD_CLIENT_ID: process.env.AUTH_AZURE_AD_CLIENT_ID,
    AUTH_AZURE_AD_CLIENT_SECRET: process.env.AUTH_AZURE_AD_CLIENT_SECRET,
    AUTH_AZURE_AD_TENANT_ID: process.env.AUTH_AZURE_AD_TENANT_ID,
@@ -298,6 +354,9 @@ export const env = createEnv({
    // Email
    EMAIL_FROM_ADDRESS: process.env.EMAIL_FROM_ADDRESS,
    SMTP_CONNECTION_URL: process.env.SMTP_CONNECTION_URL,
+    // Otel
+    OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+    OTEL_SERVICE_NAME: process.env.OTEL_SERVICE_NAME,
    // S3
    S3_ENDPOINT: process.env.S3_ENDPOINT,
    S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID,
@@ -240,7 +240,7 @@ export function ResetPasswordPage({
          <div className="mx-auto mt-10 max-w-lg text-center text-xs text-muted-foreground">
            You will only receive an email if an account with this email exists
            and you have signed up with email and password. If you used an
-            authentication provider like Google, Okta, or GitHub, please{" "}
+            authentication provider like Google, Gitlab, Okta, or GitHub, please{" "}
            <Link href="/auth/sign-in" className="underline">
              sign in
            </Link>
@@ -1,3 +1,4 @@
+import { useEffect } from "react";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import useLocalStorage from "@/src/components/useLocalStorage";

@@ -19,29 +20,31 @@ function useColumnOrder<TData>(
  localStorageKey: string,
  columns: LangfuseColumnDef<TData>[],
 ) {
-  const initialColumnOrder = () => {
-    const storedColumnOrder = readStoredColumnOrder(localStorageKey);
-    const columnIds = columns.map((c) => c.accessorKey);
-
-    // if new column has been added to table, insert it at it's default position
-    if (columnIds.length > storedColumnOrder.length) {
-      const newColumnOrder = [...storedColumnOrder];
-      columnIds.forEach((id) => {
-        if (!newColumnOrder.includes(id)) {
-          const index = columnIds.indexOf(id);
-          newColumnOrder.splice(index, 0, id);
-        }
-      });
-      return newColumnOrder;
-    }
-    return storedColumnOrder;
-  };
-
  const [columnOrder, setColumnOrder] = useLocalStorage<string[]>(
    localStorageKey,
-    initialColumnOrder(),
+    [],
  );

+  useEffect(() => {
+    const appColumnIds = columns.map((c) => c.accessorKey);
+    const storedColumnIds = readStoredColumnOrder(localStorageKey);
+
+    const finalColumnOrder: string[] = storedColumnIds.filter((id) =>
+      appColumnIds.includes(id),
+    );
+
+    appColumnIds.forEach((id) => {
+      if (!finalColumnOrder.includes(id)) {
+        finalColumnOrder.splice(appColumnIds.indexOf(id), 0, id);
+      }
+    });
+
+    // Compare the new order with the current order to avoid unnecessary updates
+    if (JSON.stringify(finalColumnOrder) !== JSON.stringify(columnOrder)) {
+      setColumnOrder(finalColumnOrder);
+    }
+  }, [columns, localStorageKey, columnOrder, setColumnOrder]);
+
  return [columnOrder, setColumnOrder] as const;
 }

@@ -21,7 +21,7 @@ export const createTracesTimeFilter = (
 export const totalCostDashboardFormatted = (totalCost?: number) => {
  return totalCost
    ? totalCost < 5
-      ? usdFormatter(totalCost, 2, 4)
+      ? usdFormatter(totalCost, 2, 6)
      : usdFormatter(totalCost, 2, 2)
    : usdFormatter(0);
 };
@@ -107,7 +107,7 @@ export class RateLimitService {
    }

    if (res.remainingPoints < 1) {
-      recordIncrement("rate_limit_exceeded", 1, {
+      recordIncrement("langfuse.rate_limit.exceeded", 1, {
        orgId: scope.orgId,
        plan: scope.plan,
        resource: resource,
@@ -268,17 +268,17 @@ export class ApiAuthService {
    const redisApiKey = await this.fetchApiKeyFromRedis(hash);

    if (redisApiKey === API_KEY_NON_EXISTENT) {
-      recordIncrement("api_key_cache_hit", 1);
+      recordIncrement("langfuse.api_key.cache_hit", 1);
      throw new Error("Invalid credentials");
    }

    // if we found something, return the object.
    if (redisApiKey) {
-      recordIncrement("api_key_cache_hit", 1);
+      recordIncrement("langfuse.api_key.cache_hit", 1);
      return redisApiKey;
    }

-    recordIncrement("api_key_cache_miss", 1);
+    recordIncrement("langfuse.api_key.cache_miss", 1);

    // if redis not available or object not found, try the database
    const apiKeyAndOrganisation = await this.prisma.apiKey.findUnique({
@@ -9,7 +9,7 @@ export async function register() {
      : true;

  if (process.env.NEXT_RUNTIME === "nodejs" && isInitLoadingEnabled) {
-    await import("./datadog.server.config");
+    await import("./observability.config");
    await import("./initialize");
  }
 }
@@ -0,0 +1,64 @@
+import dd from "dd-trace";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
+import { IORedisInstrumentation } from "@opentelemetry/instrumentation-ioredis";
+import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import { WinstonInstrumentation } from "@opentelemetry/instrumentation-winston";
+import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk";
+import { BullMQInstrumentation } from "@appsignal/opentelemetry-instrumentation-bullmq";
+import {
+  envDetector,
+  processDetector,
+  Resource,
+} from "@opentelemetry/resources";
+import { awsEcsDetectorSync } from "@opentelemetry/resource-detector-aws";
+import { containerDetector } from "@opentelemetry/resource-detector-container";
+import { env } from "@/src/env.mjs";
+
+if (!process.env.VERCEL && process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
+  dd.init({
+    runtimeMetrics: true,
+    plugins: false,
+  });
+
+  const sdk = new NodeSDK({
+    resource: new Resource({
+      "service.name": env.OTEL_SERVICE_NAME,
+    }),
+    traceExporter: new OTLPTraceExporter({
+      url: `${env.OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
+    }),
+    instrumentations: [
+      new IORedisInstrumentation(),
+      new HttpInstrumentation({
+        requireParentforOutgoingSpans: true,
+        ignoreOutgoingRequestHook: (req) => {
+          return req.host === "127.0.0.1";
+        },
+        requestHook: (span, req: any) => {
+          const url = "path" in req ? req?.path : req?.url;
+          let path = new URL(url, `http://${req?.host ?? "localhost"}`)
+            .pathname;
+          if (path.startsWith("/_next/static")) {
+            path = "/_next/static/*";
+          }
+          span.updateName(`${req?.method} ${path}`);
+          span.setAttribute("http.route", path);
+        },
+      }),
+      new PrismaInstrumentation(),
+      new AwsInstrumentation(),
+      new WinstonInstrumentation({ disableLogSending: true }),
+      new BullMQInstrumentation(),
+    ],
+    resourceDetectors: [
+      envDetector,
+      processDetector,
+      awsEcsDetectorSync,
+      containerDetector,
+    ],
+  });
+
+  sdk.start();
+}
@@ -14,7 +14,6 @@ import {
  recordIncrement,
  getCurrentSpan,
  LegacyIngestionQueue,
-  addTraceContext,
  S3StorageService,
  instrumentAsync,
 } from "@langfuse/shared/src/server";
@@ -92,7 +91,7 @@ export default async function handler(
    );

    recordIncrement(
-      "ingestion_event",
+      "langfuse.ingestion.event",
      parsedSchema.success ? parsedSchema.data.batch.length : 0,
    );

@@ -214,12 +213,12 @@ export default async function handler(
        try {
          await queue.add(
            QueueJobs.LegacyIngestionJob,
-            addTraceContext({
+            {
              payload: { data: sortedBatch, authCheck: authCheck },
              id: randomUUID(),
              timestamp: new Date(),
              name: QueueJobs.LegacyIngestionJob as const,
-            }),
+            },
            {
              removeOnFail: 1_000_000,
              removeOnComplete: true,
@@ -13,7 +13,7 @@ import { Input } from "@/src/components/ui/input";
 import { env } from "@/src/env.mjs";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { FcGoogle } from "react-icons/fc";
-import { FaGithub } from "react-icons/fa";
+import { FaGithub, FaGitlab } from "react-icons/fa";
 import { SiOkta, SiAuth0, SiAmazoncognito } from "react-icons/si";
 import { TbBrandAzure, TbBrandOauth } from "react-icons/tb";
 import { signIn } from "next-auth/react";
@@ -46,6 +46,7 @@ export type PageProps = {
    credentials: boolean;
    google: boolean;
    github: boolean;
+    gitlab: boolean;
    okta: boolean;
    azureAd: boolean;
    auth0: boolean;
@@ -73,6 +74,9 @@ export const getServerSideProps: GetServerSideProps<PageProps> = async () => {
        github:
          env.AUTH_GITHUB_CLIENT_ID !== undefined &&
          env.AUTH_GITHUB_CLIENT_SECRET !== undefined,
+        gitlab:
+          env.AUTH_GITLAB_CLIENT_ID !== undefined &&
+          env.AUTH_GITLAB_CLIENT_SECRET !== undefined,
        okta:
          env.AUTH_OKTA_CLIENT_ID !== undefined &&
          env.AUTH_OKTA_CLIENT_SECRET !== undefined &&
@@ -161,6 +165,16 @@ export function SSOButtons({
              Github
            </Button>
          )}
+          {authProviders.gitlab && (
+            <Button
+              onClick={() => handleSignIn("gitlab")}
+              variant="secondary"
+              loading={providerSigningIn === "gitlab"}
+            >
+              <FaGitlab className="mr-3" size={18} />
+              Gitlab
+            </Button>
+          )}
          {authProviders.azureAd && (
            <Button
              onClick={() => handleSignIn("azure-ad")}
@@ -154,7 +154,7 @@ function OverviewTab({ userId, projectId }: TabProps) {
        },
        {
          label: "Total Cost",
-          value: usdFormatter(user.data.sumCalculatedTotalCost, 2, 2),
+          value: usdFormatter(user.data.sumCalculatedTotalCost),
        },
      ]
    : [];
@@ -12,7 +12,6 @@ import {
  type EventBodyType,
  EventName,
  QueueJobs,
-  addTraceContext,
 } from "@langfuse/shared/src/server";
 import { TRPCError } from "@trpc/server";
 import { redis } from "@langfuse/shared/src/server";
@@ -74,15 +73,12 @@ export const batchExportRouter = createTRPCRouter({
        };

        if (redis && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
-          await getBatchExportQueue()?.add(
-            event.name,
-            addTraceContext({
-              id: event.payload.batchExportId, // Use the batchExportId to deduplicate when the same job is sent multiple times
-              name: QueueJobs.BatchExportJob,
-              timestamp: new Date(),
-              payload: event.payload,
-            }),
-          );
+          await getBatchExportQueue()?.add(event.name, {
+            id: event.payload.batchExportId, // Use the batchExportId to deduplicate when the same job is sent multiple times
+            name: QueueJobs.BatchExportJob,
+            timestamp: new Date(),
+            payload: event.payload,
+          });
        } else {
          await new WorkerClient().sendEvent(event);
        }
@@ -17,6 +17,7 @@ import { type Adapter } from "next-auth/adapters";
 import CredentialsProvider from "next-auth/providers/credentials";
 import GoogleProvider, { type GoogleProfile } from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
+import GitLabProvider from "next-auth/providers/gitlab";
 import OktaProvider from "next-auth/providers/okta";
 import EmailProvider from "next-auth/providers/email";
 import Auth0Provider from "next-auth/providers/auth0";
@@ -226,6 +227,16 @@ if (env.AUTH_GITHUB_CLIENT_ID && env.AUTH_GITHUB_CLIENT_SECRET)
    }),
  );

+if (env.AUTH_GITLAB_CLIENT_ID && env.AUTH_GITLAB_CLIENT_SECRET)
+  staticProviders.push(
+    GitLabProvider({
+      clientId: env.AUTH_GITLAB_CLIENT_ID,
+      clientSecret: env.AUTH_GITLAB_CLIENT_SECRET,
+      allowDangerousEmailAccountLinking:
+        env.AUTH_GITLAB_ALLOW_ACCOUNT_LINKING === "true",
+    })
+  );
+
 if (
  env.AUTH_AZURE_AD_CLIENT_ID &&
  env.AUTH_AZURE_AD_CLIENT_SECRET &&
@@ -1,3 +1,5 @@
+import Decimal from "decimal.js";
+
 export const compactNumberFormatter = (number?: number | bigint) => {
  return Intl.NumberFormat("en-US", {
    notation: "compact",
@@ -32,10 +34,11 @@ export const latencyFormatter = (
 };

 export const usdFormatter = (
-  number?: number | bigint,
+  number?: number | bigint | Decimal,
  minimumFractionDigits: number = 2,
-  maximumFractionDigits: number = 4,
+  maximumFractionDigits: number = 6,
 ) => {
+  const numberToFormat = number instanceof Decimal ? number.toNumber() : number;
  return new Intl.NumberFormat("en-US", {
    style: "currency",
    currency: "USD",
@@ -44,7 +47,7 @@ export const usdFormatter = (
    minimumFractionDigits,
    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat/NumberFormat#maximumfractiondigits
    maximumFractionDigits,
-  }).format(number ?? 0);
+  }).format(numberToFormat ?? 0);
 };

 export function randomIntFromInterval(min: number, max: number) {
@@ -1,6 +1,6 @@
 {
  "name": "worker",
-  "version": "2.81.2",
+  "version": "2.83.1",
  "description": "",
  "license": "MIT",
  "private": true,
@@ -19,35 +19,37 @@
  "author": "engineering@langfuse.com",
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
-    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.1",
+    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.2",
    "@clickhouse/client": "^1.4.0",
    "@langfuse/shared": "workspace:*",
-    "@opentelemetry/api": "^1.8.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.44.0",
-    "@opentelemetry/context-async-hooks": "^1.25.1",
-    "@opentelemetry/instrumentation": "^0.52.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.53.0",
+    "@opentelemetry/instrumentation": "^0.53.0",
    "@opentelemetry/instrumentation-aws-sdk": "^0.44.0",
-    "@opentelemetry/instrumentation-express": "^0.41.1",
-    "@opentelemetry/instrumentation-http": "^0.52.1",
-    "@opentelemetry/instrumentation-ioredis": "^0.42.0",
-    "@opentelemetry/instrumentation-undici": "^0.4.0",
-    "@opentelemetry/instrumentation-winston": "^0.36.0",
-    "@prisma/instrumentation": "^5.13.0",
+    "@opentelemetry/instrumentation-express": "^0.42.0",
+    "@opentelemetry/instrumentation-http": "^0.53.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.43.0",
+    "@opentelemetry/instrumentation-winston": "^0.40.0",
+    "@opentelemetry/resource-detector-aws": "^1.6.1",
+    "@opentelemetry/resource-detector-container": "^0.4.1",
+    "@opentelemetry/resources": "^1.26.0",
+    "@opentelemetry/sdk-node": "^0.53.0",
+    "@prisma/instrumentation": "^5.20.0",
    "backoff": "^2.5.0",
    "bullmq": "^5.12.10",
    "cors": "^2.8.5",
-    "dd-trace": "^5.21.0",
+    "dd-trace": "^5.23.1",
    "decimal.js": "^10.4.3",
    "dotenv": "^16.4.5",
-    "express": "^4.19.2",
+    "express": "^4.21.0",
    "express-basic-auth": "^1.2.1",
    "handlebars": "^4.7.8",
    "helmet": "^7.1.0",
    "ioredis": "^5.4.1",
-    "js-tiktoken": "^1.0.12",
+    "js-tiktoken": "^1.0.15",
    "kysely": "^0.27.4",
    "lodash": "^4.17.21",
-    "pg": "^8.11.5",
+    "pg": "^8.13.0",
    "stripe": "^16.8.0",
    "tiktoken": "^1.0.15",
    "uuid": "^9.0.1",
@@ -59,24 +61,23 @@
    "@types/cors": "^2.8.17",
    "@types/express": "^4.17.21",
    "@types/express-serve-static-core": "^4.19.3",
-    "@types/lodash": "^4.17.9",
+    "@types/lodash": "^4.17.10",
    "@types/node": "^20.11.19",
-    "@types/pg": "^8.11.6",
+    "@types/pg": "^8.11.10",
    "@types/uuid": "^9.0.8",
    "@typescript-eslint/parser": "^7.12.0",
    "eslint": "^8.57.0",
    "eslint-config-prettier": "^9.1.0",
    "eslint-config-standard": "^17.1.0",
    "eslint-plugin-prettier": "^5.1.3",
-    "kysely-codegen": "^0.11.0",
-    "msw": "^2.4.1",
-    "nodemon": "^3.1.3",
-    "prettier": "^3.2.5",
+    "kysely-codegen": "^0.16.8",
+    "msw": "^2.4.9",
+    "nodemon": "^3.1.7",
+    "prettier": "^3.3.3",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
-    "tsx": "^4.19.0",
+    "tsx": "^4.19.1",
    "typescript": "^5.4.5",
-    "vite": "^5.2.13",
-    "vitest": "^1.5.3"
+    "vitest": "^2.1.2"
  }
 }
@@ -1 +1 @@
-export const VERSION = "v2.81.2";
+export const VERSION = "v2.83.1";
@@ -72,6 +72,10 @@ const EnvSchema = z.object({
    .default(5),
  STRIPE_SECRET_KEY: z.string().optional(),

+  // Otel
+  OTEL_EXPORTER_OTLP_ENDPOINT: z.string().default("http://localhost:4318"),
+  OTEL_SERVICE_NAME: z.string().default("worker"),
+
  // Flags to toggle queue consumers on or off.
  QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED: z
    .enum(["true", "false"])
@@ -382,7 +382,6 @@ export const handleBatchExportJob = async (
      downloadLink: signedUrl,
      userName: user?.name || "",
      batchExportName: jobDetails.name,
-      expiresInHours: env.BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS,
    });

    logger.info(`Batch export success email sent to user ${user.id}`);
@@ -235,7 +235,7 @@ export const evaluate = async ({
    parsedVariableMapping
  );

-  logger.info(
+  logger.debug(
    `Evaluating job ${event.jobExecutionId} extracted variables ${JSON.stringify(mappingResult)} `
  );

@@ -246,7 +246,7 @@ export const evaluate = async ({
    ),
  });

-  logger.info(
+  logger.debug(
    `Evaluating job ${event.jobExecutionId} compiled prompt ${prompt}`
  );

@@ -1,39 +1,62 @@
-import tracer from "dd-trace";
-import { registerInstrumentations } from "@opentelemetry/instrumentation";
+import dd from "dd-trace";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
 import { IORedisInstrumentation } from "@opentelemetry/instrumentation-ioredis";
 import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
 import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express";
 import { PrismaInstrumentation } from "@prisma/instrumentation";
-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks";
-import opentelemetry from "@opentelemetry/api";
-import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici";
 import { WinstonInstrumentation } from "@opentelemetry/instrumentation-winston";
 import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk";
-// import { BullMQInstrumentation } from "@appsignal/opentelemetry-instrumentation-bullmq";
+import { BullMQInstrumentation } from "@appsignal/opentelemetry-instrumentation-bullmq";
+import {
+  envDetector,
+  processDetector,
+  Resource,
+} from "@opentelemetry/resources";
+import { awsEcsDetectorSync } from "@opentelemetry/resource-detector-aws";
+import { containerDetector } from "@opentelemetry/resource-detector-container";
+import { env } from "./env";

-const contextManager = new AsyncHooksContextManager().enable();
-
-opentelemetry.context.setGlobalContextManager(contextManager);
-
-const { TracerProvider } = tracer.init({
+dd.init({
  runtimeMetrics: true,
+  plugins: false,
 });

-const provider = new TracerProvider();
-
-registerInstrumentations({
+const sdk = new NodeSDK({
+  resource: new Resource({
+    "service.name": env.OTEL_SERVICE_NAME,
+  }),
+  traceExporter: new OTLPTraceExporter({
+    url: `${env.OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
+  }),
  instrumentations: [
    new IORedisInstrumentation(),
-    new HttpInstrumentation(),
+    new HttpInstrumentation({
+      requireParentforOutgoingSpans: true,
+      ignoreOutgoingRequestHook: (req) => {
+        return req.host === "127.0.0.1";
+      },
+      requestHook: (span, req: any) => {
+        const url = "path" in req ? req?.path : req?.url;
+        let path = new URL(url, `http://${req?.host ?? "localhost"}`).pathname;
+        if (path.startsWith("/_next/static")) {
+          path = "/_next/static/*";
+        }
+        span.updateName(`${req?.method} ${path}`);
+      },
+    }),
    new ExpressInstrumentation(),
    new PrismaInstrumentation(),
    new AwsInstrumentation(),
    new WinstonInstrumentation({ disableLogSending: true }),
-    getNodeAutoInstrumentations(),
-    new UndiciInstrumentation(),
-    // new BullMQInstrumentation(),
+    new BullMQInstrumentation(),
+  ],
+  resourceDetectors: [
+    envDetector,
+    processDetector,
+    awsEcsDetectorSync,
+    containerDetector,
  ],
 });

-provider.register();
+sdk.start();
@@ -3,52 +3,38 @@ import { Job } from "bullmq";
 import { BaseError, BatchExportStatus } from "@langfuse/shared";
 import { kyselyPrisma } from "@langfuse/shared/src/db";

-import {
-  traceException,
-  instrumentAsync,
-  logger,
-} from "@langfuse/shared/src/server";
+import { traceException, logger } from "@langfuse/shared/src/server";
 import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";
 import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";
-import { SpanKind } from "@opentelemetry/api";

 export const batchExportQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.BatchExport]>,
 ) => {
-  return instrumentAsync(
-    {
-      name: "batchExportJobExecutor",
-      spanKind: SpanKind.CONSUMER,
-      traceContext: job.data?._tracecontext,
-    },
-    async () => {
-      try {
-        logger.info("Executing Batch Export Job", job.data.payload);
-        await handleBatchExportJob(job.data.payload);
+  try {
+    logger.info("Executing Batch Export Job", job.data.payload);
+    await handleBatchExportJob(job.data.payload);

-        logger.info("Finished Batch Export Job", job.data.payload);
+    logger.info("Finished Batch Export Job", job.data.payload);

-        return true;
-      } catch (e) {
-        const displayError =
-          e instanceof BaseError ? e.message : "An internal error occurred";
+    return true;
+  } catch (e) {
+    const displayError =
+      e instanceof BaseError ? e.message : "An internal error occurred";

-        await kyselyPrisma.$kysely
-          .updateTable("batch_exports")
-          .set("status", BatchExportStatus.FAILED)
-          .set("finished_at", new Date())
-          .set("log", displayError)
-          .where("id", "=", job.data.payload.batchExportId)
-          .where("project_id", "=", job.data.payload.projectId)
-          .execute();
+    await kyselyPrisma.$kysely
+      .updateTable("batch_exports")
+      .set("status", BatchExportStatus.FAILED)
+      .set("finished_at", new Date())
+      .set("log", displayError)
+      .where("id", "=", job.data.payload.batchExportId)
+      .where("project_id", "=", job.data.payload.projectId)
+      .execute();

-        logger.error(
-          `Failed Batch Export job for id ${job.data.payload.batchExportId}`,
-          e,
-        );
-        traceException(e);
-        throw e;
-      }
-    },
-  );
+    logger.error(
+      `Failed Batch Export job for id ${job.data.payload.batchExportId}`,
+      e,
+    );
+    traceException(e);
+    throw e;
+  }
 };
@@ -4,7 +4,6 @@ import {
  logger,
  QueueName,
  QueueJobs,
-  instrumentAsync,
 } from "@langfuse/shared/src/server";
 import { handleCloudUsageMeteringJob } from "../ee/cloudUsageMetering/handleCloudUsageMeteringJob";
 import { env } from "../env";
@@ -36,20 +35,12 @@ if (cloudUsageMeteringQueue) {

 export const cloudUsageMeteringQueueProcessor: Processor = async (job) => {
  if (job.name === QueueJobs.CloudUsageMeteringJob) {
-    return instrumentAsync(
-      {
-        name: "cloudUsageMeteringJobExecutor",
-        traceContext: job.data?._tracecontext,
-      },
-      async () => {
-        logger.info("Executing Cloud Usage Metering Job", job.data);
-        try {
-          return await handleCloudUsageMeteringJob(job);
-        } catch (error) {
-          logger.error("Error executing Cloud Usage Metering Job", error);
-          throw error;
-        }
-      },
-    );
+    logger.info("Executing Cloud Usage Metering Job", job.data);
+    try {
+      return await handleCloudUsageMeteringJob(job);
+    } catch (error) {
+      logger.error("Error executing Cloud Usage Metering Job", error);
+      throw error;
+    }
  }
 };
@@ -9,13 +9,11 @@ import {
  TQueueJobTypes,
  logger,
  traceException,
-  instrumentAsync,
  recordIncrement,
  recordHistogram,
  recordGauge,
  getTraceUpsertQueue,
 } from "@langfuse/shared/src/server";
-import { SpanKind } from "@opentelemetry/api";

 let evalQueue: Queue<TQueueJobTypes[QueueName.EvaluationExecution]> | null =
  null;
@@ -40,123 +38,100 @@ export const getEvalQueue = () => {
 export const evalJobCreatorQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.TraceUpsert]>,
 ) => {
-  return instrumentAsync(
-    {
-      name: "evalJobCreator",
-      rootSpan: true,
-      spanKind: SpanKind.CONSUMER,
-      traceContext: job.data?._tracecontext,
-    },
-    async () => {
-      try {
-        const startTime = Date.now();
+  try {
+    const startTime = Date.now();

-        const waitTime = Date.now() - job.timestamp;
+    const waitTime = Date.now() - job.timestamp;
+    recordIncrement("langfuse.queue.trace_upsert.request");
+    recordHistogram("langfuse.queue.trace_upsert.wait_time", waitTime, {
+      unit: "milliseconds",
+    });

-        recordIncrement("trace_upsert_queue_request");
-        recordHistogram("trace_upsert_queue_wait_time", waitTime, {
-          unit: "milliseconds",
+    await createEvalJobs({ event: job.data.payload });
+
+    await getTraceUpsertQueue()
+      ?.count()
+      .then((count) => {
+        logger.debug(`Eval creation queue length: ${count}`);
+        recordGauge("trace_upsert_queue_length", count, {
+          unit: "records",
        });
-
-        await createEvalJobs({ event: job.data.payload });
-
-        await getTraceUpsertQueue()
-          ?.count()
-          .then((count) => {
-            logger.debug(`Eval creation queue length: ${count}`);
-            recordGauge("trace_upsert_queue_length", count, {
-              unit: "records",
-            });
-            return count;
-          })
-          .catch();
-        recordHistogram(
-          "trace_upsert_queue_processing_time",
-          Date.now() - startTime,
-          { unit: "milliseconds" },
-        );
-        return true;
-      } catch (e) {
-        logger.error(
-          `Failed job Evaluation for traceId ${job.data.payload.traceId}`,
-          e,
-        );
-        traceException(e);
-        throw e;
-      }
-    },
-  );
+        return count;
+      })
+      .catch();
+    recordHistogram(
+      "langfuse.queue.trace_upsert.processing_time",
+      Date.now() - startTime,
+      { unit: "milliseconds" },
+    );
+    return true;
+  } catch (e) {
+    logger.error(
+      `Failed job Evaluation for traceId ${job.data.payload.traceId}`,
+      e,
+    );
+    traceException(e);
+    throw e;
+  }
 };

 export const evalJobExecutorQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.EvaluationExecution]>,
 ) => {
-  return instrumentAsync(
-    {
-      name: "evalJobExecutor",
-      spanKind: SpanKind.CONSUMER,
-      traceContext: job.data?._tracecontext,
-    },
-    async () => {
-      try {
-        logger.info("Executing Evaluation Execution Job", job.data);
-        const startTime = Date.now();
+  try {
+    logger.info("Executing Evaluation Execution Job", job.data);
+    const startTime = Date.now();

-        const waitTime = Date.now() - job.timestamp;
+    const waitTime = Date.now() - job.timestamp;
+    recordIncrement("langfuse.queue.evaluation_execution.request");
+    recordHistogram("langfuse.queue.evaluation_execution.wait_time", waitTime, {
+      unit: "milliseconds",
+    });

-        recordIncrement("eval_execution_queue_request");
-        recordHistogram("eval_execution_queue_wait_time", waitTime, {
-          unit: "milliseconds",
+    await evaluate({ event: job.data.payload });
+
+    await getEvalQueue()
+      ?.count()
+      .then((count) => {
+        logger.debug(`Eval execution queue length: ${count}`);
+        recordGauge("eval_execution_queue_length", count, {
+          unit: "records",
        });
+        return count;
+      })
+      .catch();
+    recordHistogram(
+      "langfuse.queue.evaluation_execution.processing_time",
+      Date.now() - startTime,
+      { unit: "milliseconds" },
+    );

-        await evaluate({ event: job.data.payload });
+    return true;
+  } catch (e) {
+    const displayError =
+      e instanceof BaseError ? e.message : "An internal error occurred";

-        await getEvalQueue()
-          ?.count()
-          .then((count) => {
-            logger.debug(`Eval execution queue length: ${count}`);
-            recordGauge("eval_execution_queue_length", count, {
-              unit: "records",
-            });
-            return count;
-          })
-          .catch();
-        recordHistogram(
-          "eval_execution_queue_processing_time",
-          Date.now() - startTime,
-          { unit: "milliseconds" },
-        );
+    await kyselyPrisma.$kysely
+      .updateTable("job_executions")
+      .set("status", sql`'ERROR'::"JobExecutionStatus"`)
+      .set("end_time", new Date())
+      .set("error", displayError)
+      .where("id", "=", job.data.payload.jobExecutionId)
+      .where("project_id", "=", job.data.payload.projectId)
+      .execute();

-        return true;
-      } catch (e) {
-        const displayError =
-          e instanceof BaseError ? e.message : "An internal error occurred";
+    // do not log expected errors (api failures + missing api keys not provided by the user)
+    if (
+      !(e instanceof ApiError) &&
+      !(e instanceof BaseError && e.message.includes("API key for provider"))
+    ) {
+      traceException(e);
+      logger.error(
+        `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId}`,
+        e,
+      );
+    }

-        await kyselyPrisma.$kysely
-          .updateTable("job_executions")
-          .set("status", sql`'ERROR'::"JobExecutionStatus"`)
-          .set("end_time", new Date())
-          .set("error", displayError)
-          .where("id", "=", job.data.payload.jobExecutionId)
-          .where("project_id", "=", job.data.payload.projectId)
-          .execute();
-
-        // do not log expected errors (api failures + missing api keys not provided by the user)
-        if (
-          !(e instanceof ApiError) &&
-          !(
-            e instanceof BaseError && e.message.includes("API key for provider")
-          )
-        ) {
-          traceException(e);
-          logger.error(
-            `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId}`,
-            e,
-          );
-        }
-
-        throw e;
-      }
-    },
-  );
+    throw e;
+  }
 };
@@ -6,84 +6,73 @@ import {
  clickhouseClient,
  logger,
  getIngestionFlushQueue,
-  instrumentAsync,
  recordIncrement,
  recordGauge,
  recordHistogram,
 } from "@langfuse/shared/src/server";
 import { ClickhouseWriter } from "../services/ClickhouseWriter";
 import { IngestionService } from "../services/IngestionService";
-import { SpanKind } from "@opentelemetry/api";

 const ingestionFlushQueue = getIngestionFlushQueue();

 export const ingestionFlushQueueProcessor: Processor = async (job) => {
-  return instrumentAsync(
-    {
-      name: "flush-ingestion-consumer",
-      spanKind: SpanKind.CONSUMER,
-      traceContext: job.data?._tracecontext,
-    },
-    async () => {
-      if (job.name === QueueJobs.FlushIngestionEntity) {
-        const flushKey = job.id;
-        if (!flushKey) {
-          throw new Error("Flushkey not provided");
-        }
+  if (job.name === QueueJobs.FlushIngestionEntity) {
+    const flushKey = job.id;
+    if (!flushKey) {
+      throw new Error("Flushkey not provided");
+    }

-        // Log wait time
-        const waitTime = Date.now() - job.timestamp;
-        logger.debug(
-          `Received flush request after ${waitTime} ms for ${flushKey}`,
-        );
+    // Log wait time
+    const waitTime = Date.now() - job.timestamp;
+    recordIncrement("langfuse.queue.ingestion_flush.request");
+    recordHistogram("langfuse.queue.ingestion_flush.wait_time", waitTime, {
+      unit: "milliseconds",
+    });

-        recordIncrement("ingestion_processing_request");
-        recordHistogram("ingestion_flush_wait_time", waitTime, {
+    try {
+      // Check dependencies
+      if (!redis) throw new Error("Redis not available");
+      if (!prisma) throw new Error("Prisma not available");
+      if (!ingestionFlushQueue)
+        throw new Error("Ingestion flush queue not available");
+
+      // Flush ingestion buffer
+      const processingStartTime = Date.now();
+
+      await new IngestionService(
+        redis,
+        prisma,
+        ClickhouseWriter.getInstance(),
+        clickhouseClient,
+      ).flush(flushKey);
+
+      // Log processing time
+      const processingTime = Date.now() - processingStartTime;
+      logger.debug(
+        `Prepared and scheduled CH-write in ${processingTime} ms for ${flushKey}`,
+      );
+      recordHistogram(
+        "langfuse.queue.ingestion_flush.processing_time",
+        processingTime,
+        {
          unit: "milliseconds",
-        });
+        },
+      );

-        try {
-          // Check dependencies
-          if (!redis) throw new Error("Redis not available");
-          if (!prisma) throw new Error("Prisma not available");
-          if (!ingestionFlushQueue)
-            throw new Error("Ingestion flush queue not available");
-
-          // Flush ingestion buffer
-          const processingStartTime = Date.now();
-
-          await new IngestionService(
-            redis,
-            prisma,
-            ClickhouseWriter.getInstance(),
-            clickhouseClient,
-          ).flush(flushKey);
-
-          // Log processing time
-          const processingTime = Date.now() - processingStartTime;
-          logger.debug(
-            `Prepared and scheduled CH-write in ${processingTime} ms for ${flushKey}`,
-          );
-          recordHistogram("ingestion_flush_processing_time", processingTime, {
-            unit: "milliseconds",
+      // Log queue size
+      await ingestionFlushQueue
+        .count()
+        .then((count) => {
+          logger.debug(`Ingestion flush queue length: ${count}`);
+          recordGauge("langfuse.queue.ingestion_flush.length", count, {
+            unit: "records",
          });
-
-          // Log queue size
-          await ingestionFlushQueue
-            .count()
-            .then((count) => {
-              logger.debug(`Ingestion flush queue length: ${count}`);
-              recordGauge("ingestion_flush_queue_length", count, {
-                unit: "records",
-              });
-              return count;
-            })
-            .catch();
-        } catch (err) {
-          logger.error(`Error processing flush request for ${flushKey}`, err);
-          throw err;
-        }
-      }
-    },
-  );
+          return count;
+        })
+        .catch();
+    } catch (err) {
+      logger.error(`Error processing flush request for ${flushKey}`, err);
+      throw err;
+    }
+  }
 };
@@ -2,7 +2,6 @@ import { Job, Processor } from "bullmq";
 import {
  traceException,
  LegacyIngestionQueue,
-  instrumentAsync,
  QueueName,
  recordIncrement,
  recordGauge,
@@ -16,88 +15,72 @@ import {
  sendToWorkerIfEnvironmentConfigured,
 } from "@langfuse/shared/src/server";
 import { tokenCount } from "../features/tokenisation/usage";
-import { SpanKind } from "@opentelemetry/api";

 export const legacyIngestionQueueProcessor: Processor = async (
-  job: Job<TQueueJobTypes[QueueName.LegacyIngestionQueue]>
+  job: Job<TQueueJobTypes[QueueName.LegacyIngestionQueue]>,
 ) => {
-  return instrumentAsync(
-    {
-      name: "legacyIngestion",
-      spanKind: SpanKind.CONSUMER,
-      rootSpan: true,
-      traceContext: job.data?._tracecontext,
-    },
-    async () => {
-      try {
-        const startTime = Date.now();
-        logger.info("Processing legacy ingestion", {
-          payload: job.data.payload.data.map(({ body, ...rest }) => {
-            let modifiedBody = body;
-            if (body && "input" in modifiedBody) {
-              // eslint-disable-next-line @typescript-eslint/no-unused-vars
-              const { input, ...restPayload } = modifiedBody || {};
-              modifiedBody = restPayload;
-            }
-            if (body && "output" in modifiedBody) {
-              // eslint-disable-next-line @typescript-eslint/no-unused-vars
-              const { output, ...restPayload } = modifiedBody || {};
-              modifiedBody = restPayload;
-            }
-            return {
-              ...rest,
-              body: modifiedBody,
-            };
-          }),
+  try {
+    const startTime = Date.now();
+    logger.info("Processing legacy ingestion", {
+      payload: job.data.payload.data.map(({ body, ...rest }) => {
+        let modifiedBody = body;
+        if (body && "input" in modifiedBody) {
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const { input, ...restPayload } = modifiedBody || {};
+          modifiedBody = restPayload;
+        }
+        if (body && "output" in modifiedBody) {
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const { output, ...restPayload } = modifiedBody || {};
+          modifiedBody = restPayload;
+        }
+        return {
+          ...rest,
+          body: modifiedBody,
+        };
+      }),
+    });
+
+    const waitTime = Date.now() - job.timestamp;
+    recordIncrement("langfuse.queue.legacy_ingestion.request");
+    recordHistogram("langfuse.queue.legacy_ingestion.wait_time", waitTime, {
+      unit: "milliseconds",
+    });
+
+    const result = await handleBatch(
+      job.data.payload.data,
+      job.data.payload.authCheck,
+      tokenCount,
+    );
+
+    // send out REDIS requests to worker for all trace types
+    await sendToWorkerIfEnvironmentConfigured(
+      result.results,
+      job.data.payload.authCheck.scope.projectId,
+    );
+
+    // Log queue size
+    await LegacyIngestionQueue.getInstance()
+      ?.count()
+      .then((count) => {
+        logger.debug(`Legacy Ingestion flush queue length: ${count}`);
+        recordGauge("langfuse.queue.legacy_ingestion.length", count, {
+          unit: "records",
        });
-
-        // Log wait time
-        const waitTime = Date.now() - job.timestamp;
-        logger.debug(
-          `Received flush request after ${waitTime} ms for ${job.data.payload.authCheck.scope.projectId}`
-        );
-
-        recordIncrement("legacy_ingestion_processing_request");
-        recordHistogram("legacy_ingestion_flush_wait_time", waitTime, {
-          unit: "milliseconds",
-        });
-
-        const result = await handleBatch(
-          job.data.payload.data,
-          job.data.payload.authCheck,
-          tokenCount
-        );
-
-        // send out REDIS requests to worker for all trace types
-        await sendToWorkerIfEnvironmentConfigured(
-          result.results,
-          job.data.payload.authCheck.scope.projectId
-        );
-
-        // Log queue size
-        await LegacyIngestionQueue.getInstance()
-          ?.count()
-          .then((count) => {
-            logger.debug(`Legacy Ingestion flush queue length: ${count}`);
-            recordGauge("legacy_ingestion_flush_queue_length", count, {
-              unit: "records",
-            });
-            return count;
-          })
-          .catch();
-        recordHistogram(
-          "legacy_ingestion_processing_time",
-          Date.now() - startTime,
-          { unit: "milliseconds" }
-        );
-      } catch (e) {
-        logger.error(
-          `Failed job Evaluation for traceId ${job.data.payload}`,
-          e
-        );
-        traceException(e);
-        throw e;
-      }
-    }
-  );
+        return count;
+      })
+      .catch();
+    recordHistogram(
+      "langfuse.queue.legacy_ingestion.processing_time",
+      Date.now() - startTime,
+      { unit: "milliseconds" },
+    );
+  } catch (e) {
+    logger.error(
+      `Failed job legacy ingestion processing for ${job.data.payload.authCheck.scope.projectId}`,
+      e,
+    );
+    traceException(e);
+    throw e;
+  }
 };
@@ -1,5 +1,10 @@
 import { Job, Processor, Worker, WorkerOptions } from "bullmq";
-import { logger, createNewRedisInstance } from "@langfuse/shared/src/server";
+import {
+  logger,
+  createNewRedisInstance,
+  recordIncrement,
+  convertQueueNameToMetricName,
+} from "@langfuse/shared/src/server";

 export class WorkerManager {
  private static workers: { [key: string]: Worker } = {};
@@ -53,9 +58,11 @@ export class WorkerManager {
        `Queue Job ${job?.name} with id ${job?.id} in ${queueName} failed`,
        err,
      );
+      recordIncrement(convertQueueNameToMetricName(queueName + ".failed"));
    });
    worker.on("error", (failedReason: Error) => {
      logger.error(`Queue worker ${queueName} failed: ${failedReason}`);
+      recordIncrement(convertQueueNameToMetricName(queueName + ".error"));
    });
  }
 }
@@ -266,13 +266,13 @@ describe("ClickhouseWriter", () => {
    await vi.advanceTimersByTimeAsync(writer.writeInterval);

    expect(metricsDistributionSpy).toHaveBeenCalledWith(
-      "ingestion_clickhouse_insert_wait_time",
+      "langfuse.queue.clickhouse_writer.wait_time",
      expect.any(Number),
      { unit: "milliseconds" },
    );

    expect(metricsDistributionSpy).toHaveBeenCalledWith(
-      "ingestion_clickhouse_insert_processing_time",
+      "langfuse.queue.clickhouse_writer.processing_time",
      expect.any(Number),
      { unit: "milliseconds" },
    );
@@ -107,7 +107,7 @@ export class ClickhouseWriter {
    // Log wait time
    queueItems.forEach((item) => {
      const waitTime = Date.now() - item.createdAt;
-      recordHistogram("ingestion_clickhouse_insert_wait_time", waitTime, {
+      recordHistogram("langfuse.queue.clickhouse_writer.wait_time", waitTime, {
        unit: "milliseconds",
      });
    });
@@ -122,7 +122,7 @@ export class ClickhouseWriter {

      // Log processing time
      recordHistogram(
-        "ingestion_clickhouse_insert_processing_time",
+        "langfuse.queue.clickhouse_writer.processing_time",
        Date.now() - processingStartTime,
        {
          unit: "milliseconds",