chore: release v2.77.0

fix: reconnect workers to redis on errors (#3211 )
fix(docker): prevent web build oom by allocating more memory to build step in Dockerfile (#3217 )
2024-09-04 10:50:09 +02:00 · 2024-09-04 10:22:59 +02:00 · 2024-09-04 00:18:15 +02:00 · 2024-09-03 23:50:20 +02:00 · 2024-09-03 14:03:57 +00:00 · 2024-09-03 12:42:57 +02:00
67 changed files with 1570 additions and 972 deletions
@@ -204,5 +204,6 @@ ENCRYPTION_KEY="0000000000000000000000000000000000000000000000000000000000000000
 # LANGFUSE_LOG_LEVEL= 
 # LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
 # LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+# QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"

 ## END Langfuse V3 Ingestion
@@ -40,6 +40,9 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v3

+      - name: Set NEXT_PUBLIC_BUILD_ID
+        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
      - name: Build and run both images from compose
        run: |
          docker compose -f docker-compose.build.yml up -d
@@ -107,13 +110,26 @@ jobs:
      - name: Seed DB
        run: |
          pnpm run db:migrate
-          pnpm run db:seed

      - name: Build
        run: pnpm run build
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: ${{ vars.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }}

      - name: Start Langfuse
        run: (pnpm run start&)
+        env:
+          LANGFUSE_PROVISION_ORG_ID: "seed-org-id"
+          LANGFUSE_PROVISION_ORG_NAME: "Seed Org"
+          LANGFUSE_PROVISION_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+          LANGFUSE_PROVISION_PROJECT_NAME: "Seed Project"
+          LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
+          LANGFUSE_PROVISION_PROJECT_SECRET_KEY: "sk-lf-1234567890"
+          LANGFUSE_PROVISION_USER_EMAIL: "demo@langfuse.com"
+          LANGFUSE_PROVISION_USER_NAME: "Demo User"
+          LANGFUSE_PROVISION_USER_PASSWORD: "password"

      - name: run tests
        run: pnpm --filter=web run test
@@ -317,6 +333,9 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v3

+      - name: Set NEXT_PUBLIC_BUILD_ID
+        run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
      - name: Log in to the GitHub Container registry
        uses: docker/login-action@v2
        with:
@@ -27,3 +27,4 @@ jobs:
          PORTER_STACK_NAME: web
          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4060 }}
+          PORTER_NEXT_PUBLIC_BUILD_ID: ${{ steps.vars.outputs.sha_short }}
@@ -27,3 +27,4 @@ jobs:
          PORTER_STACK_NAME: web
          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4037 }}
+          PORTER_NEXT_PUBLIC_BUILD_ID: ${{ steps.vars.outputs.sha_short }}
@@ -27,3 +27,4 @@ jobs:
          PORTER_STACK_NAME: web
          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
          PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_4054 }}
+          PORTER_NEXT_PUBLIC_BUILD_ID: ${{ steps.vars.outputs.sha_short }}
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "2.75.2",
+  "version": "2.77.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -21,6 +21,10 @@
      "import": "./dist/src/server/index.js",
      "require": "./dist/src/server/index.js"
    },
+    "./src/server/auth/apiKeys": {
+      "import": "./dist/src/server/auth/apiKeys.js",
+      "require": "./dist/src/server/auth/apiKeys.js"
+    },
    "./encryption": {
      "import": "./dist/src/encryption/index.js",
      "require": "./dist/src/encryption/index.js"
@@ -81,7 +85,7 @@
  "devDependencies": {
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.7",
    "@types/node": "^20.11.29",
    "@types/nodemailer": "^6.4.15",
    "@types/pg": "^8.11.6",
@@ -38,6 +38,7 @@ const EnvSchema = z.object({
    .number()
    .positive()
    .default(60 * 10),
+  SALT: z.string().optional(), // used by components imported by web package
 });

 export const env = EnvSchema.parse(process.env);
@@ -9,7 +9,7 @@ export * from "./types";
 export * from "./filterToPrisma";
 export * from "./orderByToPrisma";
 export * from "./tracesTable";
-export * from "./server/auth/auth";
+export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
@@ -0,0 +1,81 @@
+import { PrismaClient } from "@prisma/client";
+import { compare, hash } from "bcryptjs";
+import { randomUUID } from "crypto";
+import * as crypto from "crypto";
+import { env } from "../../env";
+
+export function getDisplaySecretKey(secretKey: string) {
+  return secretKey.slice(0, 6) + "..." + secretKey.slice(-4);
+}
+
+export async function hashSecretKey(key: string) {
+  // legacy, uses bcrypt, transformed into hashed key upon first use
+  const hashedKey = await hash(key, 11);
+  return hashedKey;
+}
+
+async function generateKeySet() {
+  return {
+    pk: `pk-lf-${randomUUID()}`,
+    sk: `sk-lf-${randomUUID()}`,
+  };
+}
+
+export async function verifySecretKey(key: string, hashedKey: string) {
+  const isValid = await compare(key, hashedKey);
+  return isValid;
+}
+
+export function createShaHash(privateKey: string, salt: string): string {
+  const hash = crypto
+    .createHash("sha256")
+    .update(privateKey)
+    .update(crypto.createHash("sha256").update(salt, "utf8").digest("hex"))
+    .digest("hex");
+
+  return hash;
+}
+
+export async function createAndAddApiKeysToDb(p: {
+  prisma: PrismaClient;
+  projectId: string;
+  note?: string;
+  predefinedKeys?: {
+    secretKey: string;
+    publicKey: string;
+  };
+}) {
+  const salt = env.SALT;
+  if (!salt) {
+    throw new Error("SALT is not set");
+  }
+
+  const { pk, sk } = p.predefinedKeys
+    ? { pk: p.predefinedKeys.publicKey, sk: p.predefinedKeys.secretKey }
+    : await generateKeySet();
+
+  const hashedSk = await hashSecretKey(sk);
+  const displaySk = getDisplaySecretKey(sk);
+
+  const hashFromProvidedKey = createShaHash(sk, salt);
+
+  const apiKey = await p.prisma.apiKey.create({
+    data: {
+      projectId: p.projectId,
+      publicKey: pk,
+      hashedSecretKey: hashedSk,
+      displaySecretKey: displaySk,
+      fastHashedSecretKey: hashFromProvidedKey,
+      note: p.note,
+    },
+  });
+
+  return {
+    id: apiKey.id,
+    createdAt: apiKey.createdAt,
+    note: apiKey.note,
+    publicKey: apiKey.publicKey,
+    secretKey: sk,
+    displaySecretKey: displaySk,
+  };
+}
@@ -1,81 +0,0 @@
-import { compare, hash } from "bcryptjs";
-import { randomUUID } from "crypto";
-import * as crypto from "crypto";
-import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
-
-export function generateSecretKey() {
-  return `sk-lf-${randomUUID()}`;
-}
-
-export function generatePublicKey() {
-  return `pk-lf-${randomUUID()}`;
-}
-
-export function getDisplaySecretKey(secretKey: string) {
-  return secretKey.slice(0, 6) + "..." + secretKey.slice(-4);
-}
-
-export async function hashSecretKey(key: string) {
-  // legacy, uses bcrypt, transformed into hashed key upon first use
-  const hashedKey = await hash(key, 11);
-  return hashedKey;
-}
-
-export async function generateKeySet() {
-  const pk = generatePublicKey();
-  const sk = generateSecretKey();
-  const hashedSk = await hashSecretKey(sk);
-  const displaySk = getDisplaySecretKey(sk);
-
-  return {
-    pk,
-    sk,
-    hashedSk,
-    displaySk,
-  };
-}
-
-export async function verifySecretKey(key: string, hashedKey: string) {
-  const isValid = await compare(key, hashedKey);
-  return isValid;
-}
-
-export function createShaHash(privateKey: string, salt: string): string {
-  const hash = crypto
-    .createHash("sha256")
-    .update(privateKey)
-    .update(crypto.createHash("sha256").update(salt, "utf8").digest("hex"))
-    .digest("hex");
-
-  return hash;
-}
-
-export interface CustomSSOUser extends Record<string, any> {
-  email: string;
-  id: string;
-  name: string;
-  verified: boolean;
-}
-
-export function CustomSSOProvider<P extends CustomSSOUser>(
-  options: OAuthUserConfig<P>
-): OAuthConfig<P> {
-  return {
-    id: "custom",
-    name: "CustomSSOProvider",
-    type: "oauth",
-    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
-    authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
-    checks: ["pkce", "state"],
-    idToken: true,
-    profile(profile) {
-      return {
-        id: profile.sub,
-        name: profile.name,
-        email: profile.email,
-        image: null,
-      };
-    },
-    options,
-  };
-}
@@ -0,0 +1,31 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+
+interface CustomSSOUser extends Record<string, any> {
+  email: string;
+  id: string;
+  name: string;
+  verified: boolean;
+}
+
+export function CustomSSOProvider<P extends CustomSSOUser>(
+  options: OAuthUserConfig<P>
+): OAuthConfig<P> {
+  return {
+    id: "custom",
+    name: "CustomSSOProvider",
+    type: "oauth",
+    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
+    authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
+    checks: ["pkce", "state"],
+    idToken: true,
+    profile(profile) {
+      return {
+        id: profile.sub,
+        name: profile.name,
+        email: profile.email,
+        image: null,
+      };
+    },
+    options,
+  };
+}
@@ -3,7 +3,8 @@ export * from "./services/email/organizationInvitation/sendMembershipInvitationE
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
-export * from "./auth/auth";
+export * from "./auth/apiKeys";
+export * from "./auth/customSsoProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -2,28 +2,32 @@ import { Queue } from "bullmq";
 import { QueueName, TQueueJobTypes } from "../queues";
 import { createNewRedisInstance } from "./redis";

-let legacyIngestionQueue: Queue<
-  TQueueJobTypes[QueueName.LegacyIngestionQueue]
-> | null = null;
+export class LegacyIngestionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+  > | null = null;

-export const getLegacyIngestionQueue = () => {
-  if (legacyIngestionQueue) return legacyIngestionQueue;
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+  > | null {
+    if (LegacyIngestionQueue.instance) return LegacyIngestionQueue.instance;

-  const newRedis = createNewRedisInstance();
+    const newRedis = createNewRedisInstance({ enableOfflineQueue: false });

-  legacyIngestionQueue = newRedis
-    ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
-        QueueName.LegacyIngestionQueue,
-        {
-          connection: newRedis,
-          defaultJobOptions: {
-            removeOnComplete: true,
-            removeOnFail: 100,
-            attempts: 5,
+    LegacyIngestionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
+          QueueName.LegacyIngestionQueue,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100,
+              attempts: 5,
+            },
          },
-        }
-      )
-    : null;
+        )
+      : null;

-  return legacyIngestionQueue;
-};
+    return LegacyIngestionQueue.instance;
+  }
+}
@@ -1,11 +1,14 @@
-import Redis from "ioredis";
+import Redis, { RedisOptions } from "ioredis";
 import { env } from "../../env";

-export const createNewRedisInstance = () => {
+export const createNewRedisInstance = (
+  additionalOptions: Partial<RedisOptions> = {},
+) => {
  return env.REDIS_CONNECTION_STRING
    ? new Redis(env.REDIS_CONNECTION_STRING, {
        maxRetriesPerRequest: null,
        enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
+        ...additionalOptions,
      })
    : env.REDIS_HOST
      ? new Redis({
@@ -14,6 +17,7 @@ export const createNewRedisInstance = () => {
          password: String(env.REDIS_AUTH),
          maxRetriesPerRequest: null, // Set to `null` to disable retrying
          enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
+          ...additionalOptions,
        })
      : null;
 };
@@ -214,8 +214,8 @@ importers:
        specifier: workspace:*
        version: link:../config-typescript
      '@types/lodash':
-        specifier: ^4.17.5
-        version: 4.17.5
+        specifier: ^4.17.7
+        version: 4.17.7
      '@types/node':
        specifier: ^20.11.29
        version: 20.11.29
@@ -296,7 +296,7 @@ importers:
        version: 1.7.18(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
      '@headlessui/tailwindcss':
        specifier: 0.2.1
-        version: 0.2.1(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+        version: 0.2.1(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      '@heroicons/react':
        specifier: ^2.1.3
        version: 2.1.3(react@18.2.0)
@@ -452,7 +452,7 @@ importers:
        version: 0.10.1(typescript@5.4.5)(zod@3.23.8)
      '@tailwindcss/container-queries':
        specifier: ^0.1.1
-        version: 0.1.1(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+        version: 0.1.1(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      '@tanstack/react-query':
        specifier: ^4.36.1
        version: 4.36.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -461,7 +461,7 @@ importers:
        version: 8.13.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
      '@tremor/react':
        specifier: 3.16.2
-        version: 3.16.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+        version: 3.16.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      '@trpc/client':
        specifier: ^10.45.0
        version: 10.45.2(@trpc/server@10.45.2)
@@ -541,8 +541,8 @@ importers:
        specifier: ^4.17.21
        version: 4.17.21
      lucide-react:
-        specifier: ^0.429.0
-        version: 0.429.0(react@18.2.0)
+        specifier: ^0.436.0
+        version: 0.436.0(react@18.2.0)
      next:
        specifier: ^14.2.6
        version: 14.2.6(@babel/core@7.24.3)(@opentelemetry/api@1.8.0)(@playwright/test@1.43.1)(babel-plugin-macros@3.1.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -620,7 +620,7 @@ importers:
        version: 2.5.2
      tailwindcss-animate:
        specifier: ^1.0.7
-        version: 1.0.7(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+        version: 1.0.7(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      use-query-params:
        specifier: ^2.2.1
        version: 2.2.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -649,7 +649,7 @@ importers:
        version: 1.43.1
      '@tailwindcss/forms':
        specifier: ^0.5.7
-        version: 0.5.7(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+        version: 0.5.7(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      '@testing-library/jest-dom':
        specifier: ^6.4.6
        version: 6.4.6(@jest/globals@29.7.0)(@types/jest@29.5.12)(jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))(vitest@1.5.3(@types/node@20.10.5)(jsdom@20.0.3))
@@ -672,8 +672,8 @@ importers:
        specifier: ^29.5.12
        version: 29.5.12
      '@types/lodash':
-        specifier: ^4.17.5
-        version: 4.17.5
+        specifier: ^4.17.7
+        version: 4.17.7
      '@types/node':
        specifier: 20.10.5
        version: 20.10.5
@@ -726,8 +726,8 @@ importers:
        specifier: ^0.6.6
        version: 0.6.6(prettier@3.2.5)
      tailwindcss:
-        specifier: ^3.4.4
-        version: 3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+        specifier: ^3.4.10
+        version: 3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
      ts-node:
        specifier: ^10.9.2
        version: 10.9.2(@types/node@20.10.5)(typescript@5.4.5)
@@ -735,8 +735,8 @@ importers:
        specifier: ^4.2.0
        version: 4.2.0
      tsx:
-        specifier: ^4.18.0
-        version: 4.18.0
+        specifier: ^4.19.0
+        version: 4.19.0
      typescript:
        specifier: ^5.4.5
        version: 5.4.5
@@ -843,8 +843,8 @@ importers:
        specifier: ^10.2.0
        version: 10.2.0
      pino-pretty:
-        specifier: ^10.3.1
-        version: 10.3.1
+        specifier: ^11.2.2
+        version: 11.2.2
      stripe:
        specifier: ^16.8.0
        version: 16.8.0
@@ -874,8 +874,8 @@ importers:
        specifier: ^4.19.3
        version: 4.19.3
      '@types/lodash':
-        specifier: ^4.17.5
-        version: 4.17.5
+        specifier: ^4.17.7
+        version: 4.17.7
      '@types/node':
        specifier: ^20.11.19
        version: 20.11.29
@@ -904,8 +904,8 @@ importers:
        specifier: ^0.11.0
        version: 0.11.0(kysely@0.27.4)(pg@8.11.5)
      msw:
-        specifier: ^2.3.1
-        version: 2.3.1(typescript@5.4.5)
+        specifier: ^2.4.1
+        version: 2.4.1(graphql@16.9.0)(typescript@5.4.5)
      nodemon:
        specifier: ^3.1.3
        version: 3.1.3
@@ -919,8 +919,8 @@ importers:
        specifier: ^6.2.0
        version: 6.2.0(typescript@5.4.5)
      tsx:
-        specifier: ^4.18.0
-        version: 4.18.0
+        specifier: ^4.19.0
+        version: 4.19.0
      typescript:
        specifier: ^5.4.5
        version: 5.4.5
@@ -1399,6 +1399,9 @@ packages:
  '@bundled-es-modules/statuses@1.0.1':
    resolution: {integrity: sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==}

+  '@bundled-es-modules/tough-cookie@0.1.6':
+    resolution: {integrity: sha512-dvMHbL464C0zI+Yqxbz6kZ5TOEp7GLW+pry/RWndAR8MJQAXZ2rPmIs8tziTZjeIyhSNZgZbCePtfSbdWqStJw==}
+
  '@chevrotain/cst-dts-gen@10.5.0':
    resolution: {integrity: sha512-lhmC/FyqQ2o7pGK4Om+hzuDrm9rhFYIJ/AXoQBeongmn870Xeb0L6oGEiuR8nohFNL5sMaQEJWCxr1oIVIVXrw==}

@@ -2233,10 +2236,6 @@ packages:
    cpu: [x64]
    os: [win32]

-  '@mswjs/cookies@1.1.0':
-    resolution: {integrity: sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==}
-    engines: {node: '>=18'}
-
  '@mswjs/interceptors@0.29.1':
    resolution: {integrity: sha512-3rDakgJZ77+RiQUuSK69t1F0m8BQKA8Vh5DCS5V0DWvNY67zob2JhhQrhCO0AKLGINTRSFd1tBaHcJTkhefoSw==}
    engines: {node: '>=18'}
@@ -4882,8 +4881,8 @@ packages:
  '@types/koa__router@12.0.3':
    resolution: {integrity: sha512-5YUJVv6NwM1z7m6FuYpKfNLTZ932Z6EF6xy2BbtpJSyn13DKNQEkXVffFVSnJHxvwwWh2SAeumpjAYUELqgjyw==}

-  '@types/lodash@4.17.5':
-    resolution: {integrity: sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==}
+  '@types/lodash@4.17.7':
+    resolution: {integrity: sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==}

  '@types/mdast@4.0.4':
    resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
@@ -6638,10 +6637,6 @@ packages:
    engines: {node: '>=18'}
    hasBin: true

-  escalade@3.1.2:
-    resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}
-    engines: {node: '>=6'}
-
  escalade@3.2.0:
    resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==}
    engines: {node: '>=6'}
@@ -7245,9 +7240,6 @@ packages:
    resolution: {integrity: sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==}
    engines: {node: '>= 0.4'}

-  get-tsconfig@4.7.6:
-    resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==}
-
  get-tsconfig@4.8.0:
    resolution: {integrity: sha512-Pgba6TExTZ0FJAn1qkJAjIeKoDJ3CsI2ChuLohJnZl/tTU8MVrq3b+2t5UOPfRa4RMsorClBjJALkJUMjG1PAw==}

@@ -8551,8 +8543,8 @@ packages:
    resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==}
    engines: {node: '>=12'}

-  lucide-react@0.429.0:
-    resolution: {integrity: sha512-548DahFy7Ey+0OPlOyZ6ipnbGJzewv8gq2DxDrnAzWZ4RN4+3XyIwQXhD4AQvuREFjS1EnbAgOMaYB0VQyaK1g==}
+  lucide-react@0.436.0:
+    resolution: {integrity: sha512-N292bIxoqm1aObAg0MzFtvhYwgQE6qnIOWx/GLj5ONgcTPH6N0fD9bVq/GfdeC9ZORBXozt/XeEKDpiB3x3vlQ==}
    peerDependencies:
      react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc

@@ -8900,13 +8892,16 @@ packages:
  msgpackr@1.10.1:
    resolution: {integrity: sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==}

-  msw@2.3.1:
-    resolution: {integrity: sha512-ocgvBCLn/5l3jpl1lssIb3cniuACJLoOfZu01e3n5dbJrpA5PeeWn28jCLgQDNt6d7QT8tF2fYRzm9JoEHtiig==}
+  msw@2.4.1:
+    resolution: {integrity: sha512-HXcoQPzYTwEmVk+BGIcRa0vLabBT+J20SSSeYh/QfajaK5ceA6dlD4ZZjfz2dqGEq4vRNCPLP6eXsB94KllPFg==}
    engines: {node: '>=18'}
    hasBin: true
    peerDependencies:
+      graphql: '>= 16.8.x'
      typescript: '>= 4.7.x'
    peerDependenciesMeta:
+      graphql:
+        optional: true
      typescript:
        optional: true

@@ -9467,17 +9462,14 @@ packages:
    resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==}
    engines: {node: '>=0.10.0'}

-  pino-abstract-transport@1.1.0:
-    resolution: {integrity: sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==}
-
  pino-abstract-transport@1.2.0:
    resolution: {integrity: sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==}

  pino-http@10.2.0:
    resolution: {integrity: sha512-am03BxnV3Ckx68OkbH0iZs3indsrH78wncQ6w1w51KroIbvJZNImBKX2X1wjdY8lSyaJ0UrX/dnO2DY3cTeCRw==}

-  pino-pretty@10.3.1:
-    resolution: {integrity: sha512-az8JbIYeN/1iLj2t0jR9DV48/LQ3RC6hZPpapKPkb84Q+yTidMCpgWxIT3N0flnBDilyBQ1luWNpOeJptjdp/g==}
+  pino-pretty@11.2.2:
+    resolution: {integrity: sha512-2FnyGir8nAJAqD3srROdrF1J5BIcMT4nwj7hHSc60El6Uxlym00UbCCd8pYIterstVBFlMyF1yFV8XdGIPbj4A==}
    hasBin: true

  pino-std-serializers@7.0.0:
@@ -10407,9 +10399,6 @@ packages:
      solid-js: ^1.2
      swr-store: ^0.10

-  sonic-boom@3.8.0:
-    resolution: {integrity: sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==}
-
  sonic-boom@4.0.1:
    resolution: {integrity: sha512-hTSD/6JMLyT4r9zeof6UtuBDpjJ9sO08/nmS5djaA9eozT9oOlNdpXSnzcgj4FTqpk3nkLrs61l4gip9r1HCrQ==}

@@ -10715,8 +10704,8 @@ packages:
    peerDependencies:
      tailwindcss: '>=3.0.0 || insiders'

-  tailwindcss@3.4.4:
-    resolution: {integrity: sha512-ZoyXOdJjISB7/BcLTR6SEsLgKtDStYyYZVLsUtWChO4Ps20CBad7lfJKVDiejocV4ME1hLmyY0WJE3hSDcmQ2A==}
+  tailwindcss@3.4.10:
+    resolution: {integrity: sha512-KWZkVPm7yJRhdu4SRSl9d4AK2wM3a50UsvgHZO7xY77NQr2V+fIrEuoDGQcbvswWvFGbS2f6e+jC/6WJm1Dl0w==}
    engines: {node: '>=14.0.0'}
    hasBin: true

@@ -10835,6 +10824,10 @@ packages:
    resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==}
    engines: {node: '>=6'}

+  tough-cookie@4.1.4:
+    resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}
+    engines: {node: '>=6'}
+
  tr46@0.0.3:
    resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}

@@ -10903,8 +10896,8 @@ packages:
    peerDependencies:
      typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta'

-  tsx@4.18.0:
-    resolution: {integrity: sha512-a1jaKBSVQkd6yEc1/NI7G6yHFfefIcuf3QJST7ZEyn4oQnxLYrZR5uZAM8UrwUa3Ge8suiZHcNS1gNrEvmobqg==}
+  tsx@4.19.0:
+    resolution: {integrity: sha512-bV30kM7bsLZKZIOCHeMNVMJ32/LuJzLVajkQI/qf92J2Qr08ueLQvW00PUZGiuLPP760UINwupgUj8qrSCPUKg==}
    engines: {node: '>=18.0.0'}
    hasBin: true

@@ -11479,11 +11472,6 @@ packages:
    resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
    engines: {node: '>= 6'}

-  yaml@2.4.1:
-    resolution: {integrity: sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==}
-    engines: {node: '>= 14'}
-    hasBin: true
-
  yaml@2.4.5:
    resolution: {integrity: sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==}
    engines: {node: '>= 14'}
@@ -12422,6 +12410,11 @@ snapshots:
    dependencies:
      statuses: 2.0.1

+  '@bundled-es-modules/tough-cookie@0.1.6':
+    dependencies:
+      '@types/tough-cookie': 4.0.5
+      tough-cookie: 4.1.4
+
  '@chevrotain/cst-dts-gen@10.5.0':
    dependencies:
      '@chevrotain/gast': 10.5.0
@@ -12936,9 +12929,9 @@ snapshots:
      react: 18.2.0
      react-dom: 18.2.0(react@18.2.0)

-  '@headlessui/tailwindcss@0.2.1(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
+  '@headlessui/tailwindcss@0.2.1(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
    dependencies:
-      tailwindcss: 3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      tailwindcss: 3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))

  '@heroicons/react@2.1.3(react@18.2.0)':
    dependencies:
@@ -13211,7 +13204,7 @@ snapshots:
  '@jridgewell/gen-mapping@0.3.5':
    dependencies:
      '@jridgewell/set-array': 1.2.1
-      '@jridgewell/sourcemap-codec': 1.4.15
+      '@jridgewell/sourcemap-codec': 1.5.0
      '@jridgewell/trace-mapping': 0.3.25

  '@jridgewell/resolve-uri@3.1.2': {}
@@ -13230,7 +13223,7 @@ snapshots:
  '@jridgewell/trace-mapping@0.3.25':
    dependencies:
      '@jridgewell/resolve-uri': 3.1.2
-      '@jridgewell/sourcemap-codec': 1.4.15
+      '@jridgewell/sourcemap-codec': 1.5.0

  '@jridgewell/trace-mapping@0.3.9':
    dependencies:
@@ -13549,8 +13542,6 @@ snapshots:
  '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3':
    optional: true

-  '@mswjs/cookies@1.1.0': {}
-
  '@mswjs/interceptors@0.29.1':
    dependencies:
      '@open-draft/deferred-promise': 2.2.0
@@ -16527,14 +16518,14 @@ snapshots:
    optionalDependencies:
      typescript: 5.4.5

-  '@tailwindcss/container-queries@0.1.1(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
+  '@tailwindcss/container-queries@0.1.1(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
    dependencies:
-      tailwindcss: 3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      tailwindcss: 3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))

-  '@tailwindcss/forms@0.5.7(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
+  '@tailwindcss/forms@0.5.7(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
    dependencies:
      mini-svg-data-uri: 1.4.4
-      tailwindcss: 3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      tailwindcss: 3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))

  '@tanstack/query-core@4.36.1': {}

@@ -16603,11 +16594,11 @@ snapshots:

  '@tootallnate/quickjs-emscripten@0.23.0': {}

-  '@tremor/react@3.16.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
+  '@tremor/react@3.16.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))':
    dependencies:
      '@floating-ui/react': 0.19.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
      '@headlessui/react': 1.7.19(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
-      '@headlessui/tailwindcss': 0.2.1(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
+      '@headlessui/tailwindcss': 0.2.1(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
      date-fns: 2.30.0
      react: 18.2.0
      react-day-picker: 8.10.1(date-fns@2.30.0)(react@18.2.0)
@@ -16877,7 +16868,7 @@ snapshots:
    dependencies:
      '@types/koa': 2.14.0

-  '@types/lodash@4.17.5': {}
+  '@types/lodash@4.17.7': {}

  '@types/mdast@4.0.4':
    dependencies:
@@ -19044,8 +19035,6 @@ snapshots:
      '@esbuild/win32-ia32': 0.23.1
      '@esbuild/win32-x64': 0.23.1

-  escalade@3.1.2: {}
-
  escalade@3.2.0: {}

  escape-goat@4.0.0: {}
@@ -19127,7 +19116,7 @@ snapshots:
      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
      fast-glob: 3.3.2
-      get-tsconfig: 4.7.6
+      get-tsconfig: 4.8.0
      is-core-module: 2.15.1
      is-glob: 4.0.3
    transitivePeerDependencies:
@@ -19144,7 +19133,7 @@ snapshots:
      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
      fast-glob: 3.3.2
-      get-tsconfig: 4.7.6
+      get-tsconfig: 4.8.0
      is-core-module: 2.15.1
      is-glob: 4.0.3
    transitivePeerDependencies:
@@ -19805,10 +19794,6 @@ snapshots:
      es-errors: 1.3.0
      get-intrinsic: 1.2.4

-  get-tsconfig@4.7.6:
-    dependencies:
-      resolve-pkg-maps: 1.0.0
-
  get-tsconfig@4.8.0:
    dependencies:
      resolve-pkg-maps: 1.0.0
@@ -21426,7 +21411,7 @@ snapshots:

  lru-cache@7.18.3: {}

-  lucide-react@0.429.0(react@18.2.0):
+  lucide-react@0.436.0(react@18.2.0):
    dependencies:
      react: 18.2.0

@@ -21981,18 +21966,17 @@ snapshots:
    optionalDependencies:
      msgpackr-extract: 3.0.3

-  msw@2.3.1(typescript@5.4.5):
+  msw@2.4.1(graphql@16.9.0)(typescript@5.4.5):
    dependencies:
      '@bundled-es-modules/cookie': 2.0.0
      '@bundled-es-modules/statuses': 1.0.1
+      '@bundled-es-modules/tough-cookie': 0.1.6
      '@inquirer/confirm': 3.1.6
-      '@mswjs/cookies': 1.1.0
      '@mswjs/interceptors': 0.29.1
      '@open-draft/until': 2.1.0
      '@types/cookie': 0.6.0
      '@types/statuses': 2.0.5
      chalk: 4.1.2
-      graphql: 16.9.0
      headers-polyfill: 4.0.3
      is-node-process: 1.2.0
      outvariant: 1.4.2
@@ -22001,6 +21985,7 @@ snapshots:
      type-fest: 4.18.2
      yargs: 17.7.2
    optionalDependencies:
+      graphql: 16.9.0
      typescript: 5.4.5

  mustache@4.2.0: {}
@@ -22603,11 +22588,6 @@ snapshots:

  pify@2.3.0: {}

-  pino-abstract-transport@1.1.0:
-    dependencies:
-      readable-stream: 4.5.2
-      split2: 4.2.0
-
  pino-abstract-transport@1.2.0:
    dependencies:
      readable-stream: 4.5.2
@@ -22620,7 +22600,7 @@ snapshots:
      pino-std-serializers: 7.0.0
      process-warning: 3.0.0

-  pino-pretty@10.3.1:
+  pino-pretty@11.2.2:
    dependencies:
      colorette: 2.0.20
      dateformat: 4.6.3
@@ -22630,11 +22610,11 @@ snapshots:
      joycon: 3.1.1
      minimist: 1.2.8
      on-exit-leak-free: 2.1.2
-      pino-abstract-transport: 1.1.0
+      pino-abstract-transport: 1.2.0
      pump: 3.0.0
      readable-stream: 4.5.2
      secure-json-parse: 2.7.0
-      sonic-boom: 3.8.0
+      sonic-boom: 4.0.1
      strip-json-comments: 3.1.1

  pino-std-serializers@7.0.0: {}
@@ -22677,29 +22657,29 @@ snapshots:

  possible-typed-array-names@1.0.0: {}

-  postcss-import@15.1.0(postcss@8.4.38):
+  postcss-import@15.1.0(postcss@8.4.41):
    dependencies:
-      postcss: 8.4.38
+      postcss: 8.4.41
      postcss-value-parser: 4.2.0
      read-cache: 1.0.0
      resolve: 1.22.8

-  postcss-js@4.0.1(postcss@8.4.38):
+  postcss-js@4.0.1(postcss@8.4.41):
    dependencies:
      camelcase-css: 2.0.1
-      postcss: 8.4.38
+      postcss: 8.4.41

-  postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
+  postcss-load-config@4.0.2(postcss@8.4.41)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
    dependencies:
      lilconfig: 3.1.1
-      yaml: 2.4.1
+      yaml: 2.4.5
    optionalDependencies:
-      postcss: 8.4.38
+      postcss: 8.4.41
      ts-node: 10.9.2(@types/node@20.10.5)(typescript@5.4.5)

-  postcss-nested@6.0.1(postcss@8.4.38):
+  postcss-nested@6.0.1(postcss@8.4.41):
    dependencies:
-      postcss: 8.4.38
+      postcss: 8.4.41
      postcss-selector-parser: 6.0.16

  postcss-selector-parser@6.0.16:
@@ -23669,10 +23649,6 @@ snapshots:
      solid-js: 1.8.18
      swr-store: 0.10.6

-  sonic-boom@3.8.0:
-    dependencies:
-      atomic-sleep: 1.0.0
-
  sonic-boom@4.0.1:
    dependencies:
      atomic-sleep: 1.0.0
@@ -23906,7 +23882,7 @@ snapshots:
    dependencies:
      '@jridgewell/gen-mapping': 0.3.5
      commander: 4.1.1
-      glob: 10.3.12
+      glob: 10.4.5
      lines-and-columns: 1.2.4
      mz: 2.7.0
      pirates: 4.0.6
@@ -23985,11 +23961,11 @@ snapshots:

  tailwind-merge@2.5.2: {}

-  tailwindcss-animate@1.0.7(tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))):
+  tailwindcss-animate@1.0.7(tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))):
    dependencies:
-      tailwindcss: 3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      tailwindcss: 3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))

-  tailwindcss@3.4.4(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
+  tailwindcss@3.4.10(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
    dependencies:
      '@alloc/quick-lru': 5.2.0
      arg: 5.0.2
@@ -24005,11 +23981,11 @@ snapshots:
      normalize-path: 3.0.0
      object-hash: 3.0.0
      picocolors: 1.0.1
-      postcss: 8.4.38
-      postcss-import: 15.1.0(postcss@8.4.38)
-      postcss-js: 4.0.1(postcss@8.4.38)
-      postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
-      postcss-nested: 6.0.1(postcss@8.4.38)
+      postcss: 8.4.41
+      postcss-import: 15.1.0(postcss@8.4.41)
+      postcss-js: 4.0.1(postcss@8.4.41)
+      postcss-load-config: 4.0.2(postcss@8.4.41)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      postcss-nested: 6.0.1(postcss@8.4.41)
      postcss-selector-parser: 6.0.16
      resolve: 1.22.8
      sucrase: 3.35.0
@@ -24127,6 +24103,13 @@ snapshots:
      universalify: 0.2.0
      url-parse: 1.5.10

+  tough-cookie@4.1.4:
+    dependencies:
+      psl: 1.9.0
+      punycode: 2.3.1
+      universalify: 0.2.0
+      url-parse: 1.5.10
+
  tr46@0.0.3: {}

  tr46@3.0.0:
@@ -24213,10 +24196,10 @@ snapshots:
      tslib: 1.14.1
      typescript: 5.4.5

-  tsx@4.18.0:
+  tsx@4.19.0:
    dependencies:
      esbuild: 0.23.1
-      get-tsconfig: 4.7.6
+      get-tsconfig: 4.8.0
    optionalDependencies:
      fsevents: 2.3.3

@@ -24401,7 +24384,7 @@ snapshots:
  update-browserslist-db@1.0.13(browserslist@4.23.0):
    dependencies:
      browserslist: 4.23.0
-      escalade: 3.1.2
+      escalade: 3.2.0
      picocolors: 1.0.1

  update-browserslist-db@1.1.0(browserslist@4.23.3):
@@ -24872,8 +24855,6 @@ snapshots:

  yaml@1.10.2: {}

-  yaml@2.4.1: {}
-
  yaml@2.4.5: {}

  yargs-parser@21.1.1: {}
@@ -24881,7 +24862,7 @@ snapshots:
  yargs@17.7.1:
    dependencies:
      cliui: 8.0.1
-      escalade: 3.1.2
+      escalade: 3.2.0
      get-caller-file: 2.0.5
      require-directory: 2.1.1
      string-width: 4.2.3
@@ -24891,7 +24872,7 @@ snapshots:
  yargs@17.7.2:
    dependencies:
      cliui: 8.0.1
-      escalade: 3.1.2
+      escalade: 3.2.0
      get-caller-file: 2.0.5
      require-directory: 2.1.1
      string-width: 4.2.3
@@ -44,6 +44,11 @@ ARG NEXT_PUBLIC_CRISP_WEBSITE_ID
 ARG NEXT_PUBLIC_SENTRY_DSN
 ARG NEXT_LANGFUSE_TRACING_SAMPLE_RATE

+# Accept build id as NEXT_PUBLIC_BUILD_ID or PORTER_NEXT_PUBLIC_BUILD_ID
+ARG PORTER_NEXT_PUBLIC_BUILD_ID
+ARG NEXT_PUBLIC_BUILD_ID
+ENV NEXT_PUBLIC_BUILD_ID=${PORTER_NEXT_PUBLIC_BUILD_ID:-$NEXT_PUBLIC_BUILD_ID}
+
 # Copy source code of isolated subworkspace
 COPY --from=pruner /app/out/full/ .

@@ -56,7 +61,7 @@ RUN rm -f ./web/src/middleware.ts
 ENV NEXT_TELEMETRY_DISABLED 1
 ENV NEXT_MANUAL_SIG_HANDLE true

-RUN turbo run build --filter=web
+RUN NODE_OPTIONS='--max-old-space-size=4096' turbo run build --filter=web

 # Production image, copy all the files and run next
 FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS runner
@@ -149,39 +149,44 @@ const nextConfig = {
  },
 };

-const sentryOptions = {
-  // Additional config options for the Sentry Webpack plugin. Keep in mind that
-  // the following options are set automatically, and overriding them is not
-  // recommended:
-  //   release, url, authToken, configFile, stripPrefix,
-  //   urlPrefix, include, ignore
+export default withSentryConfig(nextConfig, {
+  // For all available options, see:
+  // https://github.com/getsentry/sentry-webpack-plugin#options

  org: process.env.SENTRY_ORG,
  project: process.env.SENTRY_PROJECT,

-  silent: true, // Suppresses all logs
+  authToken: env.SENTRY_AUTH_TOKEN,
+
+  // Only print logs for uploading source maps in CI
+  silent: !process.env.CI,

  // For all available options, see:
-  // https://github.com/getsentry/sentry-webpack-plugin#options.
+  // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/

-  // See the sections below for information on the following options:
-  //   'Configure Source Maps':
-  //     - disableServerWebpackPlugin
-  //     - disableClientWebpackPlugin
-  //     - hideSourceMaps
-  hideSourceMaps: true,
-  //     - widenClientFileUpload
-  //   'Configure Legacy Browser Support':
-  //     - transpileClientSDK
-  //   'Configure Serverside Auto-instrumentation':
-  //     - autoInstrumentServerFunctions
-  //     - excludeServerRoutes
-  //   'Configure Tunneling':
-  //     - tunnelRoute
+  // Upload a larger set of source maps for prettier stack traces (increases build time)
+  widenClientFileUpload: true,

-  // An auth token is required for uploading source maps.
-  authToken: env.SENTRY_AUTH_TOKEN,
+  // Automatically annotate React components to show their full name in breadcrumbs and session replay
+  reactComponentAnnotation: {
+    enabled: true,
+  },
+
+  // Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers.
+  // This can increase your server load as well as your hosting bill.
+  // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client-
+  // side errors will fail.
  tunnelRoute: "/api/monitoring-tunnel",
-};

-export default withSentryConfig(nextConfig, sentryOptions);
+  // Hides source maps from generated client bundles
+  hideSourceMaps: true,
+
+  // Automatically tree-shake Sentry logger statements to reduce bundle size
+  disableLogger: true,
+
+  // Enables automatic instrumentation of Vercel Cron Monitors. (Does not yet work with App Router route handlers.)
+  // See the following for more information:
+  // https://docs.sentry.io/product/crons/
+  // https://vercel.com/docs/cron-jobs
+  automaticVercelMonitors: false,
+});
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "2.75.2",
+  "version": "2.77.0",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -110,7 +110,7 @@
    "kysely": "^0.27.4",
    "langchain": "^0.2.6",
    "lodash": "^4.17.21",
-    "lucide-react": "^0.429.0",
+    "lucide-react": "^0.436.0",
    "next": "^14.2.6",
    "next-auth": "^4.24.7",
    "next-query-params": "^5.0.0",
@@ -154,7 +154,7 @@
    "@types/dompurify": "^3.0.5",
    "@types/eslint": "^8.56.7",
    "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.7",
    "@types/node": "20.10.5",
    "@types/react": "^18.2.79",
    "@types/react-dom": "^18.2.25",
@@ -172,10 +172,10 @@
    "postcss": "^8.4.38",
    "prettier": "^3.2.5",
    "prettier-plugin-tailwindcss": "^0.6.6",
-    "tailwindcss": "^3.4.4",
+    "tailwindcss": "^3.4.10",
    "ts-node": "^10.9.2",
    "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.18.0",
+    "tsx": "^4.19.0",
    "typescript": "^5.4.5",
    "wait-for-expect": "^3.0.2"
  },
@@ -548,6 +548,167 @@ describe("Authenticate API calls", () => {
    });
  });

+  describe("invalidates api keys in redis", () => {
+    const redis = new Redis("redis://:myredissecret@127.0.0.1:6379", {
+      maxRetriesPerRequest: null,
+    });
+
+    beforeEach(async () => {
+      // if we do not remove the key, it will remain in the cache and
+      // calling the test twice will not add the key to the cache
+
+      const keys = await redis.keys("api-key*");
+      console.log("before each deleting keys", keys);
+      if (keys.length > 0) {
+        console.log("before each deleting keys. actually deleting", keys);
+        await redis.del(keys);
+      }
+    });
+
+    afterEach(async () => {
+      // if we do not remove the key, it will remain in the cache and
+      // calling the test twice will not add the key to the cache
+
+      const keys = await redis.keys("api-key*");
+      console.log("after each deleting keys", keys);
+      if (keys.length > 0) {
+        await redis.del(keys);
+      }
+    });
+
+    afterAll(async () => {
+      redis.disconnect();
+    });
+
+    it("should invalidate organization API keys in redis", async () => {
+      await createAPIKey();
+
+      // put keys into cache
+      await new ApiAuthService(prisma, redis).verifyAuthHeaderAndReturnScope(
+        "Basic cGstbGYtMTIzNDU2Nzg5MDpzay1sZi0xMjM0NTY3ODkw",
+      );
+
+      await new ApiAuthService(prisma, redis).verifyAuthHeaderAndReturnScope(
+        "Basic cGstbGYtMTIzNDU2Nzg5MDpzay1sZi0xMjM0NTY3ODkw",
+      );
+
+      const apiKey = await prisma.apiKey.findUnique({
+        where: { publicKey: "pk-lf-1234567890" },
+      });
+      expect(apiKey).not.toBeNull();
+
+      const cachedKey = await redis.get(
+        `api-key:${apiKey?.fastHashedSecretKey}`,
+      );
+      expect(cachedKey).not.toBeNull();
+
+      await new ApiAuthService(prisma, redis).invalidateOrgApiKeys(
+        "seed-org-id",
+      );
+
+      const invalidatedCachedKey = await redis.get(
+        `api-key:${apiKey?.fastHashedSecretKey}`,
+      );
+      expect(invalidatedCachedKey).toBeNull();
+    });
+
+    it("if no keys in redis, invalidating org keys should do nothing", async () => {
+      await createAPIKey();
+
+      await prisma.apiKey.update({
+        where: { publicKey: "pk-lf-1234567890" },
+        data: {
+          fastHashedSecretKey: Math.random().toString(36).substring(2, 15),
+        },
+      });
+
+      await new ApiAuthService(prisma, redis).invalidateOrgApiKeys(
+        "seed-org-id",
+      );
+
+      const keys = await redis.keys("api-key*");
+      expect(keys.length).toBe(0);
+    });
+
+    it("if no keys in redis, invalidating org keys without fast hash should do nothing", async () => {
+      await createAPIKey();
+
+      await new ApiAuthService(prisma, redis).invalidateOrgApiKeys(
+        "seed-org-id",
+      );
+
+      const keys = await redis.keys("api-key*");
+      expect(keys.length).toBe(0);
+    });
+
+    it("should invalidate project API keys in redis", async () => {
+      await createAPIKey();
+
+      // put keys into cache
+      await new ApiAuthService(prisma, redis).verifyAuthHeaderAndReturnScope(
+        "Basic cGstbGYtMTIzNDU2Nzg5MDpzay1sZi0xMjM0NTY3ODkw",
+      );
+      await new ApiAuthService(prisma, redis).verifyAuthHeaderAndReturnScope(
+        "Basic cGstbGYtMTIzNDU2Nzg5MDpzay1sZi0xMjM0NTY3ODkw",
+      );
+
+      const apiKey = await prisma.apiKey.findUnique({
+        where: { publicKey: "pk-lf-1234567890" },
+      });
+      expect(apiKey).not.toBeNull();
+
+      const cachedKey = await redis.get(
+        `api-key:${apiKey?.fastHashedSecretKey}`,
+      );
+      expect(cachedKey).not.toBeNull();
+
+      await new ApiAuthService(prisma, redis).invalidateProjectApiKeys(
+        "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      );
+
+      const invalidatedCachedKey = await redis.get(
+        `api-key:${apiKey?.fastHashedSecretKey}`,
+      );
+      expect(invalidatedCachedKey).toBeNull();
+    });
+
+    it("if no keys in redis, invalidating project keys should do nothing", async () => {
+      await createAPIKey();
+
+      await prisma.apiKey.update({
+        where: { publicKey: "pk-lf-1234567890" },
+        data: {
+          fastHashedSecretKey: Math.random().toString(36).substring(2, 15),
+        },
+      });
+
+      await new ApiAuthService(prisma, redis).invalidateProjectApiKeys(
+        "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      );
+
+      const keys = await redis.keys("api-key*");
+      expect(keys.length).toBe(0);
+    });
+
+    it("if no keys in redis, invalidating project keys without fast hash should do nothing", async () => {
+      await createAPIKey();
+
+      await prisma.apiKey.update({
+        where: { publicKey: "pk-lf-1234567890" },
+        data: {
+          fastHashedSecretKey: Math.random().toString(36).substring(2, 15),
+        },
+      });
+
+      await new ApiAuthService(prisma, redis).invalidateProjectApiKeys(
+        "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      );
+
+      const keys = await redis.keys("api-key*");
+      expect(keys.length).toBe(0);
+    });
+  });
+
  const createAPIKey = async () => {
    const seedApiKey = {
      id: "seed-api-key",
@@ -1,7 +1,4 @@
-import { hash } from "bcryptjs";
-
 import { env } from "@/src/env.mjs";
-import { getDisplaySecretKey, hashSecretKey } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { type z } from "zod";

@@ -24,10 +21,7 @@ export const pruneDatabase = async () => {
  await prisma.llmApiKeys.deleteMany();
 };

-export function createBasicAuthHeader(
-  username: string,
-  password: string,
-): string {
+function createBasicAuthHeader(username: string, password: string): string {
  const base64Credentials = Buffer.from(`${username}:${password}`).toString(
    "base64",
  );
@@ -97,38 +91,3 @@ export async function makeZodVerifiedAPICall<T extends z.ZodTypeAny>(
  }
  return { body: resBody, status };
 }
-
-export const setupUserAndProject = async () => {
-  const user = await prisma.user.create({
-    data: {
-      id: "user-1",
-      name: "Demo User",
-      email: "demo@langfuse.com",
-      password: await hash("password", 12),
-    },
-  });
-
-  const project = await prisma.project.create({
-    data: {
-      id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      name: "llm-app",
-      apiKeys: {
-        create: [
-          {
-            note: "seeded key",
-            hashedSecretKey: await hashSecretKey("sk-lf-1234567890"),
-            displaySecretKey: getDisplaySecretKey("sk-lf-1234567890"),
-            publicKey: "pk-lf-1234567890",
-          },
-        ],
-      },
-      projectMembers: {
-        create: {
-          role: "OWNER",
-          userId: user.id,
-        },
-      },
-    },
-  });
-  return { user, project };
-};
@@ -2,6 +2,7 @@ import {
  BadgeCheck,
  Github,
  HardDriveDownload,
+  Info,
  Map,
  Newspaper,
 } from "lucide-react";
@@ -20,6 +21,7 @@ import { api } from "@/src/utils/api";
 import { Button } from "@/src/components/ui/button";
 import { useIsEeEnabled } from "@/src/ee/utils/useIsEeEnabled";
 import { env } from "@/src/env.mjs";
+import { cn } from "@/src/utils/tailwind";

 export const VersionLabel = ({ className }: { className?: string }) => {
  const checkUpdate = api.public.checkUpdate.useQuery(undefined, {
@@ -30,6 +32,7 @@ export const VersionLabel = ({ className }: { className?: string }) => {
    onError: (error) => console.error("checkUpdate error", error), // do not render default error message
  });
  const isEeVersion = useIsEeEnabled();
+  const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);

  const hasUpdate =
    !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
@@ -46,8 +49,9 @@ export const VersionLabel = ({ className }: { className?: string }) => {
  return (
    <DropdownMenu>
      <DropdownMenuTrigger asChild>
-        <Button variant="ghost" size="xs" className={className}>
+        <Button variant="ghost" size="xs" className={cn("text-xs",className)}>
          {VERSION}
+          {!isLangfuseCloud && (isEeVersion ? " EE" : " OSS")}
          {hasUpdate && <ArrowUp className={`ml-1 h-3 w-3 ${color}`} />}
        </Button>
      </DropdownMenuTrigger>
@@ -96,6 +100,17 @@ export const VersionLabel = ({ className }: { className?: string }) => {
            Roadmap
          </Link>
        </DropdownMenuItem>
+        {!isLangfuseCloud && !isEeVersion && (
+          <DropdownMenuItem asChild>
+            <Link
+              href="https://langfuse.com/docs/deployment/feature-overview"
+              target="_blank"
+            >
+              <Info size={16} className="mr-2" />
+              Compare Versions
+            </Link>
+          </DropdownMenuItem>
+        )}
        {hasUpdate && (
          <>
            <DropdownMenuSeparator />
@@ -39,7 +39,7 @@ export const ROUTES: Route[] = [
    name: "Langfuse",
    pathname: "/",
    icon: LangfuseIcon,
-    label: <VersionLabel />,
+    label: <VersionLabel className="-ml-3" />,
  },
  {
    name: "Projects",
@@ -186,7 +186,6 @@ export const ObservationPreview = (props: {
                source="generation"
                generation={observationWithInputAndOutput.data}
                analyticsEventName="trace_detail:test_in_playground_button_click"
-                fullWidth
              />
            )}
            {observationWithInputAndOutput.data ? (
@@ -1 +1 @@
-export const VERSION = "v2.75.2";
+export const VERSION = "v2.77.0";
@@ -83,7 +83,7 @@ const OrganizationUsageChart = () => {
          <>
            <Text>
              {usage.data.billingPeriod
-                ? `Observations in billing period`
+                ? `Observations in current billing period`
                : "Observations / last 30d"}
            </Text>
            <Metric>{numberFormatter(usage.data.countObservations, 0)}</Metric>
@@ -214,18 +214,6 @@ export const cloudBillingRouter = createTRPCRouter({
        session: ctx.session,
      });

-      const thirtyDaysAgo = new Date();
-      thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
-      thirtyDaysAgo.setHours(0, 0, 0, 0);
-      let billingPeriod: {
-        start: Date;
-        end: Date;
-      } | null = null;
-      let upcomingInvoice: {
-        usdAmount: number;
-        date: Date;
-      } | null = null;
-
      const organization = await ctx.prisma.organization.findUnique({
        where: {
          id: input.orgId,
@@ -249,35 +237,49 @@ export const cloudBillingRouter = createTRPCRouter({
          parsedOrg.cloudConfig.stripe.activeSubscriptionId,
        );
        if (subscription) {
-          billingPeriod = {
+          const billingPeriod = {
            start: new Date(subscription.current_period_start * 1000),
            end: new Date(subscription.current_period_end * 1000),
          };
          const stripeInvoice = await stripeClient.invoices.retrieveUpcoming({
            subscription: parsedOrg.cloudConfig.stripe.activeSubscriptionId,
          });
-          upcomingInvoice = {
+          const upcomingInvoice = {
            usdAmount: stripeInvoice.amount_due / 100,
            date: new Date(stripeInvoice.period_end * 1000),
          };
+          const usage = stripeInvoice.lines.data.reduce((acc, line) => {
+            if (line.quantity) {
+              return acc + line.quantity;
+            }
+            return acc;
+          }, 0);
+          return {
+            countObservations: usage,
+            billingPeriod,
+            upcomingInvoice,
+          };
        }
      }

+      // For non-Stripe subscriptions, we can only get usage from the LangFuse API
+      const thirtyDaysAgo = new Date();
+      thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+      thirtyDaysAgo.setHours(0, 0, 0, 0);
+
      const usage = await ctx.prisma.observation.count({
        where: {
          project: {
            orgId: input.orgId,
          },
          startTime: {
-            gte: billingPeriod?.start ?? thirtyDaysAgo,
+            gte: thirtyDaysAgo,
          },
        },
      });

      return {
        countObservations: usage,
-        billingPeriod,
-        upcomingInvoice,
      };
    }),
 });
@@ -32,7 +32,7 @@ type JumpToPlaygroundButtonProps = (
      analyticsEventName: "trace_detail:test_in_playground_button_click";
    }
 ) & {
-  fullWidth?: boolean;
+  variant?: "outline" | "secondary";
 };

 export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (
@@ -61,17 +61,14 @@ export const JumpToPlaygroundButton: React.FC<JumpToPlaygroundButtonProps> = (

  return (
    <Button
-      variant={props.fullWidth ? "secondary" : "outline"}
+      variant={props.variant ?? "secondary"}
      title="Test in LLM playground"
-      size={!props.fullWidth ? "icon" : undefined}
      onClick={handleClick}
      asChild
    >
      <Link href={`/project/${projectId}/playground`}>
        <Terminal className="h-4 w-4" />
-        {props.fullWidth ? (
-          <span className="ml-2">Test in playground</span>
-        ) : null}
+        <span className="ml-2">Test in playground</span>
      </Link>
    </Button>
  );
@@ -171,6 +171,15 @@ export const env = createEnv({
    SENTRY_AUTH_TOKEN: z.string().optional(),
    SENTRY_CSP_REPORT_URI: z.string().optional(),
    LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),
+    LANGFUSE_PROVISION_ORG_ID: z.string().optional(),
+    LANGFUSE_PROVISION_ORG_NAME: z.string().optional(),
+    LANGFUSE_PROVISION_PROJECT_ID: z.string().optional(),
+    LANGFUSE_PROVISION_PROJECT_NAME: z.string().optional(),
+    LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY: z.string().optional(),
+    LANGFUSE_PROVISION_PROJECT_SECRET_KEY: z.string().optional(),
+    LANGFUSE_PROVISION_USER_EMAIL: z.string().email().optional(),
+    LANGFUSE_PROVISION_USER_NAME: z.string().optional(),
+    LANGFUSE_PROVISION_USER_PASSWORD: z.string().optional(),
  },

  /**
@@ -194,6 +203,7 @@ export const env = createEnv({
    NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(),
    NEXT_PUBLIC_POSTHOG_HOST: z.string().optional(),
    NEXT_PUBLIC_CRISP_WEBSITE_ID: z.string().optional(),
+    NEXT_PUBLIC_BUILD_ID: z.string().optional(),
  },

  /**
@@ -206,6 +216,7 @@ export const env = createEnv({
    NEXT_PUBLIC_DEMO_ORG_ID: process.env.NEXT_PUBLIC_DEMO_ORG_ID,
    DATABASE_URL: process.env.DATABASE_URL,
    NODE_ENV: process.env.NODE_ENV,
+    NEXT_PUBLIC_BUILD_ID: process.env.NEXT_PUBLIC_BUILD_ID,
    NEXTAUTH_SECRET: process.env.NEXTAUTH_SECRET,
    NEXTAUTH_COOKIE_DOMAIN: process.env.NEXTAUTH_COOKIE_DOMAIN,
    NEXTAUTH_URL: process.env.NEXTAUTH_URL,
@@ -321,6 +332,16 @@ export const env = createEnv({
    SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
    SENTRY_CSP_REPORT_URI: process.env.SENTRY_CSP_REPORT_URI,
    LANGFUSE_RATE_LIMITS_ENABLED: process.env.LANGFUSE_RATE_LIMITS_ENABLED,
+    // provisioning
+    LANGFUSE_PROVISION_ORG_ID: process.env.LANGFUSE_PROVISION_ORG_ID,
+    LANGFUSE_PROVISION_ORG_NAME: process.env.LANGFUSE_PROVISION_ORG_NAME,
+    LANGFUSE_PROVISION_PROJECT_ID: process.env.LANGFUSE_PROVISION_PROJECT_ID,
+    LANGFUSE_PROVISION_PROJECT_NAME: process.env.LANGFUSE_PROVISION_PROJECT_NAME,
+    LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY: process.env.LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY,
+    LANGFUSE_PROVISION_PROJECT_SECRET_KEY: process.env.LANGFUSE_PROVISION_PROJECT_SECRET_KEY,
+    LANGFUSE_PROVISION_USER_EMAIL: process.env.LANGFUSE_PROVISION_USER_EMAIL,
+    LANGFUSE_PROVISION_USER_NAME: process.env.LANGFUSE_PROVISION_USER_NAME,
+    LANGFUSE_PROVISION_USER_PASSWORD: process.env.LANGFUSE_PROVISION_USER_PASSWORD,
  },
  // Skip validation in Docker builds
  // DOCKER_BUILD is set in Dockerfile
@@ -8,6 +8,7 @@ export type AuditableResource =
  | "comment"
  | "datasetItem"
  | "dataset"
+  | "datasetRun"
  | "trace"
  | "project"
  | "observation"
@@ -52,8 +52,8 @@ export async function signupApiHandler(
  }

  // EE: check if custom SSO configuration is enabled for this domain
-  const customSsoProvider = await getSsoAuthProviderIdForDomain(domain);
-  if (customSsoProvider) {
+  const multiTenantSsoProvider = await getSsoAuthProviderIdForDomain(domain);
+  if (multiTenantSsoProvider) {
    res.status(422).json({
      message: "You must sign in via SSO for this domain.",
    });
@@ -20,12 +20,23 @@ import {
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { type ScoreAggregate } from "@/src/features/scores/lib/types";
 import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns";
+import { MoreVertical } from "lucide-react";
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuLabel,
+  DropdownMenuTrigger,
+} from "@/src/components/ui/dropdown-menu";
+import { Button } from "@/src/components/ui/button";
+import { DeleteDatasetRunButton } from "@/src/features/datasets/components/DeleteDatasetRunButton";
+
+type DatasetRunRowKey = {
+  id: string;
+  name: string;
+};

 export type DatasetRunRowData = {
-  key: {
-    id: string;
-    name: string;
-  };
+  key: DatasetRunRowKey;
  createdAt: string;
  countRunItems: string;
  avgLatency: number;
@@ -45,6 +56,7 @@ export function DatasetRunsTable(props: {
    pageIndex: withDefault(NumberParam, 0),
    pageSize: withDefault(NumberParam, 50),
  });
+
  const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
    "datasetRuns",
    "s",
@@ -149,6 +161,35 @@ export function DatasetRunsTable(props: {
        ) : null;
      },
    },
+    {
+      id: "actions",
+      accessorKey: "actions",
+      header: "Actions",
+      size: 70,
+      cell: ({ row }) => {
+        const key: DatasetRunRowKey = row.getValue("key");
+        const { id: datasetRunId } = key;
+
+        return (
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button variant="ghost" className="h-8 w-8 p-0">
+                <span className="sr-only [position:relative]">Open menu</span>
+                <MoreVertical className="h-4 w-4" />
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent align="end">
+              <DropdownMenuLabel>Actions</DropdownMenuLabel>
+              <DeleteDatasetRunButton
+                projectId={props.projectId}
+                datasetRunId={datasetRunId}
+                fullWidth
+              />
+            </DropdownMenuContent>
+          </DropdownMenu>
+        );
+      },
+    },
  ];

  const convertToTableRow = (
@@ -0,0 +1,94 @@
+import { Trash } from "lucide-react";
+import { Button } from "@/src/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { api } from "@/src/utils/api";
+import React, { useState } from "react";
+import { useRouter } from "next/router";
+
+export const DeleteDatasetRunButton = ({
+  projectId,
+  datasetRunId,
+  fullWidth = false,
+  redirectUrl,
+}: {
+  projectId: string;
+  datasetRunId: string;
+  fullWidth?: boolean;
+  redirectUrl?: string;
+}) => {
+  const [isDialogOpen, setIsDialogOpen] = useState(false);
+  const capture = usePostHogClientCapture();
+  const hasAccess = useHasProjectAccess({
+    projectId: projectId,
+    scope: "datasets:CUD",
+  });
+  const utils = api.useUtils();
+  const router = useRouter();
+  const mutDelete = api.datasets.deleteDatasetRun.useMutation({
+    onSuccess: () => {
+      redirectUrl ? router.push(redirectUrl) : utils.datasets.invalidate();
+    },
+  });
+
+  const button = fullWidth ? (
+    <Button variant="ghost" className="w-full" disabled={!hasAccess}>
+      <div className="flex w-full flex-row items-center gap-1">
+        <Trash className="h-4 w-4" />
+        <span className="text-sm font-normal">Delete</span>
+      </div>
+    </Button>
+  ) : (
+    <Button variant="outline" size="icon" disabled={!hasAccess}>
+      <Trash className="h-4 w-4" />
+    </Button>
+  );
+
+  return hasAccess ? (
+    <Dialog
+      open={isDialogOpen}
+      onOpenChange={(isOpen) => {
+        if (!mutDelete.isLoading) {
+          setIsDialogOpen(isOpen);
+        }
+      }}
+    >
+      <DialogTrigger asChild>{button}</DialogTrigger>
+      <DialogContent className="sm:max-w-xl">
+        <DialogHeader>
+          <DialogTitle className="mb-4">Please confirm</DialogTitle>
+          <DialogDescription className="text-md p-0">
+            This action cannot be undone and removes all the data associated
+            with this dataset run.
+          </DialogDescription>
+        </DialogHeader>
+        <Button
+          variant="destructive"
+          loading={mutDelete.isLoading}
+          disabled={mutDelete.isLoading}
+          onClick={async (event) => {
+            event.preventDefault();
+            capture("dataset_run:delete_form_open");
+            await mutDelete.mutateAsync({
+              projectId,
+              datasetRunId,
+            });
+            setIsDialogOpen(false);
+          }}
+        >
+          Delete Dataset Run
+        </Button>
+      </DialogContent>
+    </Dialog>
+  ) : (
+    button
+  );
+};
@@ -150,7 +150,7 @@ export const EditDatasetItem = ({
                        field.onChange(v);
                      }}
                      editable={hasAccess}
-                      className="max-h-[600px] overflow-y-auto"
+                      className="max-h-[500px] overflow-y-auto"
                    />
                  </FormControl>
                  <FormMessage />
@@ -171,7 +171,7 @@ export const EditDatasetItem = ({
                        field.onChange(v);
                      }}
                      editable={hasAccess}
-                      className="max-h-[600px] overflow-y-auto"
+                      className="max-h-[500px] overflow-y-auto"
                    />
                  </FormControl>
                  <FormMessage />
@@ -193,7 +193,7 @@ export const EditDatasetItem = ({
                      field.onChange(v);
                    }}
                    editable={hasAccess}
-                    className="max-h-[300px] overflow-y-auto"
+                    className="max-h-[200px] overflow-y-auto"
                  />
                </FormControl>
                <FormMessage />
@@ -801,4 +801,35 @@ export const datasetRouter = createTRPCRouter({
        },
      });
    }),
+  deleteDatasetRun: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        datasetRunId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "datasets:CUD",
+      });
+
+      const deletedDatasetRun = await ctx.prisma.datasetRuns.delete({
+        where: {
+          id_projectId: {
+            id: input.datasetRunId,
+            projectId: input.projectId,
+          },
+        },
+      });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "datasetRun",
+        resourceId: deletedDatasetRun.id,
+        action: "delete",
+        before: deletedDatasetRun,
+      });
+      return deletedDatasetRun;
+    }),
 });
@@ -0,0 +1,38 @@
+import { Button } from "@/src/components/ui/button";
+import { toast } from "sonner";
+
+export const showVersionUpdateToast = () => {
+  toast.custom(
+    () => (
+      <div className="flex justify-between">
+        <div className="flex min-w-[300px] flex-1 flex-col gap-2">
+          <div className="flex items-center gap-2">
+            <div className="m-0 text-sm font-medium leading-tight text-foreground/70">
+              We have released a new version of Langfuse. Please refresh your
+              browser to get the latest update.
+            </div>
+          </div>
+          <Button
+            variant="outline"
+            size={"sm"}
+            className="text-foreground/50"
+            onClick={() => {
+              window.location.reload();
+            }}
+          >
+            Refresh page
+          </Button>
+        </div>
+      </div>
+    ),
+    {
+      duration: Infinity,
+      style: {
+        padding: "1rem",
+        borderRadius: "0.5rem",
+        border: "1px solid hsl(var(--border))",
+        backgroundColor: "hsl(var(--border))",
+      },
+    },
+  );
+};
@@ -95,6 +95,7 @@ const events = {
    "new_from_trace_form_submit",
    "new_from_trace_form_open",
  ],
+  dataset_run: ["delete_form_open"],
  notification: ["click_link", "dismiss_notification"],
  tag: [
    "add_existing_tag",
@@ -83,8 +83,6 @@ export function SetPromptVersionLabels({ prompt }: { prompt: Prompt }) {
    setIsOpen(false);
  };

-  if (!hasAccess) return null;
-
  return (
    <Popover
      key={prompt.id}
@@ -98,8 +96,10 @@ export function SetPromptVersionLabels({ prompt }: { prompt: Prompt }) {
        <Button
          variant="outline"
          size="icon"
+          className="h-7 w-7 px-0"
          aria-label="Set prompt labels"
          title="Set prompt labels"
+          disabled={!hasAccess}
        >
          <TagIcon className="h-4 w-4" />
        </Button>
@@ -160,7 +160,7 @@ export function SetPromptVersionLabels({ prompt }: { prompt: Prompt }) {
            ) : (
              <Button
                variant="ghost"
-                className="mt-2 w-full justify-start px-2 py-1 text-sm  font-normal"
+                className="mt-2 w-full justify-start px-2 py-1 text-sm font-normal"
                onClick={() => setIsAddingLabel(true)}
              >
                <PlusIcon className="mr-2 h-4 w-4" />
@@ -62,6 +62,7 @@ export function DeletePromptVersion({
          variant="outline"
          type="button"
          size="icon"
+          className="h-7 w-7 px-0"
          disabled={!hasAccess}
        >
          <Trash className="h-4 w-4" />
@@ -1,4 +1,3 @@
-import { Pencil } from "lucide-react";
 import Link from "next/link";
 import { useRouter } from "next/router";
 import { NumberParam, useQueryParam } from "use-query-params";
@@ -7,10 +6,8 @@ import Header from "@/src/components/layouts/header";
 import { OpenAiMessageView } from "@/src/components/trace/IOPreview";
 import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { Badge } from "@/src/components/ui/badge";
-import { Button } from "@/src/components/ui/button";
 import { CodeView, JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
-import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
 import { PromptType } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
@@ -18,7 +15,6 @@ import { extractVariables } from "@/src/utils/string";
 import { ScrollArea } from "@radix-ui/react-scroll-area";
 import { TagPromptDetailsPopover } from "@/src/features/tag/components/TagPromptDetailsPopover";
 import { PromptHistoryNode } from "./prompt-history";
-import { SetPromptVersionLabels } from "@/src/features/prompts/components/SetPromptVersionLabels";
 import Generations from "@/src/components/table/use-cases/generations";
 import {
  Accordion,
@@ -26,14 +22,12 @@ import {
  AccordionItem,
  AccordionTrigger,
 } from "@/src/components/ui/accordion";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { JumpToPlaygroundButton } from "@/src/ee/features/playground/page/components/JumpToPlaygroundButton";
 import { ChatMlArraySchema } from "@/src/components/schemas/ChatMlSchema";
 import { CommentList } from "@/src/features/comments/CommentList";

 export const PromptDetail = () => {
  const projectId = useProjectIdFromURL();
-  const capture = usePostHogClientCapture();
  const promptName = decodeURIComponent(useRouter().query.promptName as string);
  const [currentPromptVersion, setCurrentPromptVersion] = useQueryParam(
    "version",
@@ -107,31 +101,11 @@ export const PromptDetail = () => {
            ]}
            actionButtons={
              <>
-                <SetPromptVersionLabels prompt={prompt} />
-
                <JumpToPlaygroundButton
                  source="prompt"
                  prompt={prompt}
                  analyticsEventName="prompt_detail:test_in_playground_button_click"
-                />
-
-                <Button
                  variant="outline"
-                  size="icon"
-                  onClick={() => capture("prompts:update_form_open")}
-                  asChild
-                >
-                  <Link
-                    href={`/project/${projectId}/prompts/new?promptId=${encodeURIComponent(prompt.id)}`}
-                  >
-                    <Pencil className="h-4 w-4" />
-                  </Link>
-                </Button>
-
-                <DeletePromptVersion
-                  promptVersionId={prompt.id}
-                  version={prompt.version}
-                  countVersions={promptHistory.data.totalCount}
                />
                <DetailPageNav
                  key="nav"
@@ -242,6 +216,7 @@ export const PromptDetail = () => {
                prompts={promptHistory.data.promptVersions}
                currentPromptVersion={prompt.version}
                setCurrentPromptVersion={setCurrentPromptVersion}
+                totalCount={promptHistory.data.totalCount}
              />
            </ScrollArea>
          </div>
@@ -1,7 +1,15 @@
 import { StatusBadge } from "@/src/components/layouts/status-badge";
+import { Button } from "@/src/components/ui/button";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
+import { SetPromptVersionLabels } from "@/src/features/prompts/components/SetPromptVersionLabels";
 import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { type RouterOutputs } from "@/src/utils/api";
+import { Pencil, PencilOff } from "lucide-react";
+import Link from "next/link";
 import { type NextRouter, useRouter } from "next/router";
+import { useState } from "react";

 const PromptHistoryTraceNode = (props: {
  index: number;
@@ -10,7 +18,14 @@ const PromptHistoryTraceNode = (props: {
  setCurrentPromptVersion: (version: number | undefined) => void;
  router: NextRouter;
  projectId: string;
+  totalCount: number;
 }) => {
+  const capture = usePostHogClientCapture();
+  const [isHovered, setIsHovered] = useState(false);
+  const hasAccess = useHasProjectAccess({
+    projectId: props.projectId,
+    scope: "prompts:CUD",
+  });
  const { prompt } = props;
  let badges: JSX.Element[] = prompt.labels
    .sort((a, b) =>
@@ -21,7 +36,7 @@ const PromptHistoryTraceNode = (props: {
          : a.localeCompare(b),
    )
    .map((label) => {
-      return <StatusBadge type={label} key={label} />;
+      return <StatusBadge type={label} key={label} className="h-6" />;
    });

  return (
@@ -29,19 +44,63 @@ const PromptHistoryTraceNode = (props: {
      className={`group mb-2 flex cursor-pointer flex-col gap-1 rounded-sm p-2 hover:bg-primary-foreground ${
        props.currentPromptVersion === prompt.version ? "bg-muted" : ""
      }`}
+      onMouseEnter={() => setIsHovered(true)}
+      onMouseLeave={() => setIsHovered(false)}
      onClick={() => {
        props.index === 0
          ? props.setCurrentPromptVersion(undefined)
          : props.setCurrentPromptVersion(prompt.version);
      }}
    >
-      <div className="flex flex-wrap items-center gap-2">
-        <span className="rounded-sm bg-input p-1 text-xs">
-          Version {prompt.version}
-        </span>
-        {badges}
+      <div className="grid grid-cols-[auto,1fr] items-start gap-2">
+        <div
+          className={`grid grid-cols-[auto,1fr] items-start ${isHovered ? "h-full" : "h-7"}`}
+        >
+          <span className="flex h-6 text-nowrap rounded-sm bg-input p-1 text-xs">
+            Version {prompt.version}
+          </span>
+          {Boolean(prompt.labels.length) && (
+            <div className="ml-2 flex h-full flex-wrap gap-1 overflow-auto">
+              {badges}
+            </div>
+          )}
+        </div>
+        {isHovered && (
+          <div className="flex flex-row space-x-1">
+            <SetPromptVersionLabels prompt={prompt} />
+            {hasAccess ? (
+              <Button
+                variant="outline"
+                size="icon"
+                className="h-7 w-7 px-0"
+                onClick={() => {
+                  capture("prompts:update_form_open");
+                }}
+              >
+                <Link
+                  href={`/project/${props.projectId}/prompts/new?promptId=${encodeURIComponent(prompt.id)}`}
+                >
+                  <Pencil className="h-4 w-4" />
+                </Link>
+              </Button>
+            ) : (
+              <Button
+                variant="outline"
+                size="icon"
+                className="h-7 w-7 px-0"
+                disabled
+              >
+                <PencilOff className="h-4 w-4" />
+              </Button>
+            )}
+            <DeletePromptVersion
+              promptVersionId={prompt.id}
+              version={prompt.version}
+              countVersions={props.totalCount}
+            />
+          </div>
+        )}
      </div>
-
      <div className="flex gap-2">
        <span className="text-xs text-muted-foreground">
          {prompt.createdAt.toLocaleString()}
@@ -60,6 +119,7 @@ export const PromptHistoryNode = (props: {
  prompts: RouterOutputs["prompts"]["allVersions"]["promptVersions"];
  currentPromptVersion: number | undefined;
  setCurrentPromptVersion: (id: number | undefined) => void;
+  totalCount: number;
 }) => {
  const router = useRouter();
  const projectId = router.query.projectId as string;
@@ -74,6 +134,7 @@ export const PromptHistoryNode = (props: {
          setCurrentPromptVersion={props.setCurrentPromptVersion}
          router={router}
          projectId={projectId}
+          totalCount={props.totalCount}
        />
      ))}
    </div>
@@ -15,7 +15,7 @@ import {
 import { type NextApiResponse } from "next";

 // Business Logic
-// - rate limit strategy is based on org-id, org plan, and resources. Rate limits are appliead in buckets of minutes.
+// - rate limit strategy is based on org-id, org plan, and resources. Rate limits are applied in buckets of minutes.
 // - rate limits are not applied for self hosters and are also not applied when Redis is not available
 // - infos for rate-limits are taken from the API access scope. Info for this scope is stored alongside API Keys in Redis for efficient access.
 // - isRateLimited returns false for self-hosters
@@ -35,10 +35,18 @@ export class ApiAuthService {
  private async invalidate(apiKeys: ApiKey[], identifier: string) {
    const hashKeys = apiKeys.map((key) => key.fastHashedSecretKey);

+    const filteredHashKeys = hashKeys.filter((hash): hash is string =>
+      Boolean(hash),
+    );
+    if (filteredHashKeys.length === 0) {
+      console.log("No valid keys to invalidate");
+      return;
+    }
+
    if (this.redis) {
      console.log(`Invalidating API keys in redis for ${identifier}`);
      await this.redis.del(
-        hashKeys
+        filteredHashKeys
          .filter((hash): hash is string => Boolean(hash))
          .map((hash) => this.createRedisKey(hash)),
      );
@@ -81,9 +89,7 @@ export class ApiAuthService {

    // if redis is available, delete the key from there as well
    // delete from redis even if caching is disabled via env for consistency
-    if (this.redis && apiKey.fastHashedSecretKey) {
-      await this.redis.del(this.createRedisKey(apiKey.fastHashedSecretKey));
-    }
+    this.invalidate([apiKey], `key ${id}`);

    await this.prisma.apiKey.delete({
      where: {
@@ -1,5 +1,4 @@
 import { auditLog } from "@/src/features/audit-logs/auditLog";
-import { createShaHash, generateKeySet } from "@langfuse/shared/src/server";
 import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import {
  createTRPCRouter,
@@ -8,7 +7,7 @@ import {
 import * as z from "zod";
 import { ApiAuthService } from "@/src/features/public-api/server/apiAuth";
 import { redis } from "@langfuse/shared/src/server";
-import { env } from "@/src/env.mjs";
+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";

 export const apiKeysRouter = createTRPCRouter({
  byProjectId: protectedProjectProcedure
@@ -56,37 +55,20 @@ export const apiKeysRouter = createTRPCRouter({
        scope: "apiKeys:create",
      });

-      const { pk, sk, hashedSk, displaySk } = await generateKeySet();
-
-      const salt = env.SALT;
-      const hashFromProvidedKey = createShaHash(sk, salt);
-
-      const apiKey = await ctx.prisma.apiKey.create({
-        data: {
-          projectId: input.projectId,
-          publicKey: pk,
-          hashedSecretKey: hashedSk,
-          displaySecretKey: displaySk,
-          fastHashedSecretKey: hashFromProvidedKey,
-          note: input.note,
-        },
+      const apiKeyMeta = await createAndAddApiKeysToDb({
+        prisma: ctx.prisma,
+        projectId: input.projectId,
+        note: input.note,
      });

      await auditLog({
        session: ctx.session,
        resourceType: "apiKey",
-        resourceId: apiKey.id,
+        resourceId: apiKeyMeta.id,
        action: "create",
      });

-      return {
-        id: apiKey.id,
-        createdAt: apiKey.createdAt,
-        note: input.note,
-        publicKey: apiKey.publicKey,
-        secretKey: sk,
-        displaySecretKey: displaySk,
-      };
+      return apiKeyMeta;
    }),
  delete: protectedProjectProcedure
    .input(
@@ -13,7 +13,7 @@ export async function telemetry() {
  try {
    // Only run in prod
    if (process.env.NODE_ENV !== "production") return;
-    // Do not run in Lanfuse cloud, separate telemetry is used
+    // Do not run in Langfuse cloud, separate telemetry is used
    if (process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined) return;
    // Check if telemetry is not disabled, except for EE
    if (
@@ -3,5 +3,6 @@
 export async function register() {
  if (process.env.NEXT_RUNTIME === "nodejs") {
    await import("./datadog.server.config");
+    await import("./provisioning");
  }
 }
@@ -3,7 +3,6 @@ import { type NextApiRequest, type NextApiResponse } from "next";
 import { z } from "zod";
 import { LangfuseNotFoundError, InternalServerError } from "@langfuse/shared";
 import {
-  getLegacyIngestionQueue,
  eventTypes,
  ingestionEvent,
  traceException,
@@ -13,6 +12,7 @@ import {
  handleBatch,
  recordIncrement,
  getCurrentSpan,
+  LegacyIngestionQueue,
 } from "@langfuse/shared/src/server";
 import {
  SdkLogProcessor,
@@ -141,43 +141,54 @@ export default async function handler(

    if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
      // this function MUST NOT return but send the HTTP response directly
-      const queue = getLegacyIngestionQueue();
+      const queue = LegacyIngestionQueue.getInstance();

      if (queue) {
        // still need to check auth scope for all events individually

        const failedAccessScope = accessCheckPerEvent(sortedBatch, authCheck);

-        await queue.add(
-          QueueJobs.LegacyIngestionJob,
-          {
-            payload: { data: sortedBatch, authCheck: authCheck },
-            id: randomUUID(),
-            timestamp: new Date(),
-            name: QueueJobs.LegacyIngestionJob as const,
-          },
-          {
-            removeOnFail: 1_000_000,
-            removeOnComplete: true,
-            attempts: 5,
-            backoff: {
-              type: "exponential",
-              delay: 1000,
+        let addToQueueFailed = false;
+        try {
+          await queue.add(
+            QueueJobs.LegacyIngestionJob,
+            {
+              payload: { data: sortedBatch, authCheck: authCheck },
+              id: randomUUID(),
+              timestamp: new Date(),
+              name: QueueJobs.LegacyIngestionJob as const,
            },
-          },
-        );
+            {
+              removeOnFail: 1_000_000,
+              removeOnComplete: true,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 1000,
+              },
+            },
+          );
+        } catch (e: unknown) {
+          console.warn(
+            "Failed to add batch to queue, falling back to sync processing",
+            e,
+          );
+          addToQueueFailed = true;
+        }

-        return handleBatchResult(
-          [
-            ...validationErrors,
-            ...failedAccessScope.map((e) => ({
-              id: e.id,
-              error: "Access Scope Denied",
-            })),
-          ], // we are not sending additional server errors to the client in case of early return
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
-          res,
-        );
+        if (!addToQueueFailed) {
+          return handleBatchResult(
+            [
+              ...validationErrors,
+              ...failedAccessScope.map((e) => ({
+                id: e.id,
+                error: "Access Scope Denied",
+              })),
+            ], // we are not sending additional server errors to the client in case of early return
+            sortedBatch.map((event) => ({ id: event.id, result: event })),
+            res,
+          );
+        }
      } else {
        console.error(
          "Ingestion queue not initialized, falling back to sync processing",
@@ -1,6 +1,7 @@
 import { createNextApiHandler } from "@trpc/server/adapters/next";
 import { createTRPCContext } from "@/src/server/api/trpc";
 import { appRouter } from "@/src/server/api/root";
+import { env } from "@/src/env.mjs";
 import { traceException } from "@langfuse/shared/src/server";

 export const config = {
@@ -15,4 +16,11 @@ export default createNextApiHandler({
    console.error(`❌ tRPC failed on ${path ?? "<no-path>"}: ${error.message}`);
    traceException(error);
  },
+  responseMeta() {
+    return {
+      headers: {
+        "x-build-id": env.NEXT_PUBLIC_BUILD_ID,
+      },
+    };
+  },
 });
@@ -1,6 +1,11 @@
 import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
 import Header from "@/src/components/layouts/header";
 import { Button } from "@/src/components/ui/button";
+import {
+  ResizableHandle,
+  ResizablePanel,
+  ResizablePanelGroup,
+} from "@/src/components/ui/resizable";
 import { DatasetRunItemsTable } from "@/src/features/datasets/components/DatasetRunItemsTable";
 import { EditDatasetItem } from "@/src/features/datasets/components/EditDatasetItem";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
@@ -68,13 +73,28 @@ export default function Dataset() {
          </>
        }
      />
-      <EditDatasetItem projectId={projectId} datasetItem={item.data ?? null} />
-      <Header title="Runs" level="h3" />
-      <DatasetRunItemsTable
-        projectId={projectId}
-        datasetItemId={itemId}
-        datasetId={datasetId}
-      />
+
+      <ResizablePanelGroup direction="vertical">
+        <ResizablePanel
+          minSize={10}
+          defaultSize={50}
+          className="!overflow-y-auto"
+        >
+          <EditDatasetItem
+            projectId={projectId}
+            datasetItem={item.data ?? null}
+          />
+        </ResizablePanel>
+        <ResizableHandle withHandle className="bg-border" />
+        <ResizablePanel minSize={10} className="flex flex-col space-y-4">
+          <Header title="Runs" level="h3" />
+          <DatasetRunItemsTable
+            projectId={projectId}
+            datasetItemId={itemId}
+            datasetId={datasetId}
+          />
+        </ResizablePanel>
+      </ResizablePanelGroup>
    </FullScreenPage>
  );
 }
@@ -2,6 +2,7 @@ import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
 import Header from "@/src/components/layouts/header";
 import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { DatasetRunItemsTable } from "@/src/features/datasets/components/DatasetRunItemsTable";
+import { DeleteDatasetRunButton } from "@/src/features/datasets/components/DeleteDatasetRunButton";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { api } from "@/src/utils/api";
 import { useRouter } from "next/router";
@@ -36,13 +37,20 @@ export default function Dataset() {
          { name: run.data?.name ?? "" },
        ]}
        actionButtons={
-          <DetailPageNav
-            currentId={runId}
-            path={(id) =>
-              `/project/${projectId}/datasets/${datasetId}/runs/${id}`
-            }
-            listKey="datasetRuns"
-          />
+          <>
+            <DeleteDatasetRunButton
+              projectId={projectId}
+              datasetRunId={runId}
+              redirectUrl={`/project/${projectId}/datasets/${datasetId}`}
+            />
+            <DetailPageNav
+              currentId={runId}
+              path={(id) =>
+                `/project/${projectId}/datasets/${datasetId}/runs/${id}`
+              }
+              listKey="datasetRuns"
+            />
+          </>
        }
      />
      <div className="flex flex-col gap-2">
@@ -0,0 +1,99 @@
+import { env } from "@/src/env.mjs";
+import { createUserEmailPassword } from "@/src/features/auth-credentials/lib/credentialsServerUtils";
+import { prisma } from "@langfuse/shared/src/db";
+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";
+
+// Create Organization
+if (env.LANGFUSE_PROVISION_ORG_ID) {
+  const org = await prisma.organization.upsert({
+    where: { id: env.LANGFUSE_PROVISION_ORG_ID },
+    update: {},
+    create: {
+      id: env.LANGFUSE_PROVISION_ORG_ID,
+      name: env.LANGFUSE_PROVISION_ORG_NAME ?? "Provisioned Org",
+    },
+  });
+
+  // Create Project: Org -> Project
+  if (env.LANGFUSE_PROVISION_PROJECT_ID) {
+    await prisma.project.upsert({
+      where: { id: env.LANGFUSE_PROVISION_PROJECT_ID },
+      update: {},
+      create: {
+        id: env.LANGFUSE_PROVISION_PROJECT_ID,
+        name: env.LANGFUSE_PROVISION_PROJECT_NAME ?? "Provisioned Project",
+        orgId: org.id,
+      },
+    });
+
+    // Add API Keys: Project -> API Key
+    if (
+      env.LANGFUSE_PROVISION_PROJECT_SECRET_KEY &&
+      env.LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY
+    ) {
+      const existingApiKey = await prisma.apiKey.findUnique({
+        where: { publicKey: env.LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY },
+      });
+
+      // Delete key if project changed
+      if (
+        existingApiKey &&
+        existingApiKey.projectId !== env.LANGFUSE_PROVISION_PROJECT_ID
+      ) {
+        await prisma.apiKey.delete({
+          where: { publicKey: env.LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY },
+        });
+      }
+
+      // Create new key if it doesn't exist or project changed
+      if (
+        !existingApiKey ||
+        existingApiKey.projectId !== env.LANGFUSE_PROVISION_PROJECT_ID
+      ) {
+        await createAndAddApiKeysToDb({
+          prisma,
+          projectId: env.LANGFUSE_PROVISION_PROJECT_ID,
+          note: "Provisioned API Key",
+          predefinedKeys: {
+            secretKey: env.LANGFUSE_PROVISION_PROJECT_SECRET_KEY,
+            publicKey: env.LANGFUSE_PROVISION_PROJECT_PUBLIC_KEY,
+          },
+        });
+      }
+    }
+  }
+
+  // Create User: Org -> User
+  if (
+    env.LANGFUSE_PROVISION_USER_EMAIL &&
+    env.LANGFUSE_PROVISION_USER_PASSWORD
+  ) {
+    const existingUser = await prisma.user.findUnique({
+      where: { email: env.LANGFUSE_PROVISION_USER_EMAIL },
+    });
+
+    let userId = existingUser?.id;
+
+    // Create user if it doesn't exist yet
+    if (!userId) {
+      userId = await createUserEmailPassword(
+        env.LANGFUSE_PROVISION_USER_EMAIL,
+        env.LANGFUSE_PROVISION_USER_PASSWORD,
+        env.LANGFUSE_PROVISION_USER_NAME ?? "Provisioned User",
+      );
+    }
+
+    // Create OrgMembership: Org -> OrgMembership <- User
+    await prisma.organizationMembership.upsert({
+      where: {
+        orgId_userId: { userId, orgId: org.id },
+      },
+      update: { role: "OWNER" },
+      create: {
+        userId,
+        orgId: org.id,
+        role: "OWNER",
+      },
+    });
+  }
+}
@@ -104,8 +104,9 @@ const staticProviders: Provider[] = [
      }

      // EE: Check custom SSO enforcement
-      const customSsoProvider = await getSsoAuthProviderIdForDomain(domain);
-      if (customSsoProvider) {
+      const multiTenantSsoProvider =
+        await getSsoAuthProviderIdForDomain(domain);
+      if (multiTenantSsoProvider) {
        throw new Error(`You must sign in via SSO for this domain.`);
      }

@@ -416,8 +417,12 @@ export async function getAuthOptions(): Promise<NextAuthOptions> {
          // EE: Check custom SSO enforcement, enforce the specific SSO provider on email domain
          // This also blocks setting a password for an email that is enforced to use SSO via password reset flow
          const domain = email.split("@")[1];
-          const customSsoProvider = await getSsoAuthProviderIdForDomain(domain);
-          if (customSsoProvider && account?.provider !== customSsoProvider) {
+          const multiTenantSsoProvider =
+            await getSsoAuthProviderIdForDomain(domain);
+          if (
+            multiTenantSsoProvider &&
+            account?.provider !== multiTenantSsoProvider
+          ) {
            console.log(
              "Custom SSO provider enforced for domain, user signed in with other provider",
            );
@@ -10,7 +10,10 @@ import {
  httpLink,
  loggerLink,
  splitLink,
+  TRPCClientError,
+  type TRPCLink,
 } from "@trpc/client";
+import { observable } from "@trpc/server/observable";
 import { createTRPCNext } from "@trpc/next";
 import { type inferRouterInputs, type inferRouterOutputs } from "@trpc/server";
 import superjson from "superjson";
@@ -18,6 +21,7 @@ import superjson from "superjson";
 import { type AppRouter } from "@/src/server/api/root";
 import { setUpSuperjson } from "@/src/utils/superjson";
 import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
+import { showVersionUpdateToast } from "@/src/features/notifications/showVersionUpdateToast";

 setUpSuperjson();

@@ -27,6 +31,58 @@ const getBaseUrl = () => {
  return `http://localhost:${process.env.PORT ?? 3000}`; // dev SSR should use localhost
 };

+// global build id used to compare versions to show refresh toast on stale cache hit serving deprecated files
+let buildId: string | null = null;
+
+const CLIENT_STALE_CACHE_CODES = [404, 400];
+
+const handleTrpcError = (error: unknown) => {
+  if (error instanceof TRPCClientError) {
+    const httpStatus: number =
+      typeof error.data?.httpStatus === "number" ? error.data.httpStatus : 500;
+
+    if (CLIENT_STALE_CACHE_CODES.includes(httpStatus)) {
+      if (
+        !!buildId &&
+        !!process.env.NEXT_PUBLIC_BUILD_ID &&
+        buildId !== process.env.NEXT_PUBLIC_BUILD_ID
+      ) {
+        showVersionUpdateToast();
+        return;
+      }
+    }
+  }
+
+  trpcErrorToast(error);
+};
+
+// onError update build id to compare versions
+const buildIdLink = (): TRPCLink<AppRouter> => () => {
+  return ({ next, op }) => {
+    return observable((observer) => {
+      const unsubscribe = next(op).subscribe({
+        next(value) {
+          observer.next(value);
+        },
+        error(err) {
+          if (
+            err.meta &&
+            err.meta.response &&
+            err.meta.response instanceof Response
+          ) {
+            buildId = err.meta.response.headers.get("x-build-id");
+          }
+          observer.error(err);
+        },
+        complete() {
+          observer.complete();
+        },
+      });
+      return unsubscribe;
+    });
+  };
+};
+
 /** A set of type-safe react-query hooks for your tRPC API. */
 export const api = createTRPCNext<AppRouter>({
  config() {
@@ -44,6 +100,7 @@ export const api = createTRPCNext<AppRouter>({
       * @see https://trpc.io/docs/links
       */
      links: [
+        buildIdLink(),
        loggerLink({
          enabled: (opts) =>
            process.env.NODE_ENV === "development" ||
@@ -73,12 +130,12 @@ export const api = createTRPCNext<AppRouter>({
      queryClientConfig: {
        defaultOptions: {
          queries: {
-            onError: (error) => trpcErrorToast(error),
+            onError: (error) => handleTrpcError(error),
            // react query defaults to `online`, but we want to disable it as it caused issues for some users
            networkMode: "always",
          },
          mutations: {
-            onError: (error) => trpcErrorToast(error),
+            onError: (error) => handleTrpcError(error),
            // react query defaults to `online`, but we want to disable it as it caused issues for some users
            networkMode: "always",
          },
@@ -1,6 +1,6 @@
 {
  "name": "worker",
-  "version": "2.75.2",
+  "version": "2.77.0",
  "description": "",
  "license": "MIT",
  "private": true,
@@ -50,7 +50,7 @@
    "pg": "^8.11.5",
    "pino": "^9.2.0",
    "pino-http": "^10.2.0",
-    "pino-pretty": "^10.3.1",
+    "pino-pretty": "^11.2.2",
    "stripe": "^16.8.0",
    "tiktoken": "^1.0.15",
    "uuid": "^9.0.1",
@@ -62,7 +62,7 @@
    "@types/cors": "^2.8.17",
    "@types/express": "^4.17.21",
    "@types/express-serve-static-core": "^4.19.3",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.7",
    "@types/node": "^20.11.19",
    "@types/pg": "^8.11.6",
    "@types/uuid": "^9.0.8",
@@ -72,12 +72,12 @@
    "eslint-config-standard": "^17.1.0",
    "eslint-plugin-prettier": "^5.1.3",
    "kysely-codegen": "^0.11.0",
-    "msw": "^2.3.1",
+    "msw": "^2.4.1",
    "nodemon": "^3.1.3",
    "prettier": "^3.2.5",
    "ts-node": "^10.9.2",
    "tsc-watch": "^6.2.0",
-    "tsx": "^4.18.0",
+    "tsx": "^4.19.0",
    "typescript": "^5.4.5",
    "vite": "^5.2.13",
    "vitest": "^1.5.3"
@@ -1,30 +1,19 @@
-import { expect, test, describe, vi } from "vitest";
+import { expect, test, describe, vi, afterEach } from "vitest";
 import { randomUUID } from "crypto";
-import { z } from "zod";
-import logger from "../logger";
-import { evalJobCreator } from "../queues/evalQueue";
 import {
  getTraceUpsertQueue,
  QueueJobs,
-  TraceUpsertEventSchema,
+  QueueName,
 } from "@langfuse/shared/src/server";
+import { WorkerManager } from "../queues/workerManager";

 describe.sequential("handle redis events", () => {
-  test("handle redis job succeeding", async () => {
-    vi.mock("../eval-service", () => ({
-      createEvalJobs: async ({
-        data,
-      }: {
-        data: z.infer<typeof TraceUpsertEventSchema>;
-      }) => {
-        return true;
-      },
-    }));
+  afterEach(async () => {
+    await WorkerManager.closeWorkers();
+  });

-    // this activates the consumer
-    evalJobCreator?.on("completed", (job, err) => {
-      logger.info(`Eval Job with id ${job?.id} completed`);
-    });
+  test("handle redis job succeeding", async () => {
+    WorkerManager.register(QueueName.TraceUpsert, async () => true);

    const traceUpsertQueue = getTraceUpsertQueue();

@@ -47,10 +36,36 @@ describe.sequential("handle redis events", () => {
      },
      {
        timeout: 20_000,
-      }
+      },
    );
  }, 20_000);

+  test("handle no matching queue worker", async () => {
+    // IngestionFlushQueue worker vs TraceUpsert producer
+    WorkerManager.register(QueueName.IngestionFlushQueue, async () => true);
+
+    const traceUpsertQueue = getTraceUpsertQueue();
+
+    expect(traceUpsertQueue).toBeDefined();
+
+    const job = await traceUpsertQueue?.add(QueueJobs.TraceUpsert, {
+      id: randomUUID(),
+      timestamp: new Date(),
+      payload: {
+        projectId: "project-id",
+        traceId: "trace-id",
+      },
+      name: QueueJobs.TraceUpsert as const,
+    });
+
+    // Wait for 2s
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    // Job should still be waiting as there is no listener
+    const jobState = await traceUpsertQueue?.getJobState(job!.id!);
+    expect(jobState).toEqual("waiting");
+  }, 5000);
+
  // test("handle redis job failing", async () => {
  //   vi.mock("../eval-service", () => ({
  //     createEvalJobs: async ({
@@ -6,18 +6,21 @@ import MessageResponse from "./interfaces/MessageResponse";

 require("dotenv").config();

-import logger from "./logger";
-
-import { evalJobCreator, evalJobExecutor } from "./queues/evalQueue";
-import { batchExportJobExecutor } from "./queues/batchExportQueue";
-import { ingestionQueueExecutor } from "./queues/ingestionFlushQueueExecutor";
-import { repeatQueueExecutor } from "./queues/repeatQueue";
-import { logQueueWorkerError } from "./utils/logQueueWorkerError";
+import {
+  evalJobCreatorQueueProcessor,
+  evalJobExecutorQueueProcessor,
+} from "./queues/evalQueue";
+import { batchExportQueueProcessor } from "./queues/batchExportQueue";
+import { ingestionFlushQueueProcessor } from "./queues/ingestionFlushQueueExecutor";
+import { repeatQueueProcessor } from "./queues/repeatQueue";
 import { onShutdown } from "./utils/shutdown";

 import helmet from "helmet";
-import { legacyIngestionExecutor } from "./queues/legacyIngestionQueue";
-import { cloudUsageMeteringJobExecutor } from "./queues/cloudUsageMeteringQueue";
+import { legacyIngestionQueueProcessor } from "./queues/legacyIngestionQueue";
+import { cloudUsageMeteringQueueProcessor } from "./queues/cloudUsageMeteringQueue";
+import { WorkerManager } from "./queues/workerManager";
+import { QueueName } from "@langfuse/shared/src/server";
+import { env } from "./env";

 const app = express();

@@ -35,34 +38,54 @@ app.use("/api", api);
 app.use(middlewares.notFound);
 app.use(middlewares.errorHandler);

-logger.info("Eval Job Creator started", evalJobCreator?.isRunning());
+WorkerManager.register(QueueName.RepeatQueue, repeatQueueProcessor);

-logger.info("Eval Job Executor started", evalJobExecutor?.isRunning());
-logger.info(
-  "Batch Export Job Executor started",
-  batchExportJobExecutor?.isRunning()
-);
-logger.info("Repeat Queue Executor started", repeatQueueExecutor?.isRunning());
-logger.info(
-  "Flush Ingestion Queue Executor started",
-  ingestionQueueExecutor?.isRunning()
-);
-logger.info(
-  "Legacy Ingestion Executor started",
-  legacyIngestionExecutor?.isRunning()
-);
-logger.info(
-  "Cloud Usage Metering Job Executor started",
-  cloudUsageMeteringJobExecutor?.isRunning()
+WorkerManager.register(QueueName.TraceUpsert, evalJobCreatorQueueProcessor, {
+  concurrency: env.LANGFUSE_EVAL_CREATOR_WORKER_CONCURRENCY,
+});
+
+WorkerManager.register(
+  QueueName.EvaluationExecution,
+  evalJobExecutorQueueProcessor,
+  {
+    concurrency: env.LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY,
+  },
 );

-evalJobCreator?.on("failed", logQueueWorkerError);
-evalJobExecutor?.on("failed", logQueueWorkerError);
-batchExportJobExecutor?.on("failed", logQueueWorkerError);
-repeatQueueExecutor?.on("failed", logQueueWorkerError);
-ingestionQueueExecutor?.on("failed", logQueueWorkerError);
-legacyIngestionExecutor?.on("failed", logQueueWorkerError);
-cloudUsageMeteringJobExecutor?.on("failed", logQueueWorkerError);
+WorkerManager.register(QueueName.BatchExport, batchExportQueueProcessor, {
+  concurrency: 1, // only 1 job at a time
+  limiter: {
+    // execute 1 batch export in 5 seconds to avoid overloading the DB
+    max: 1,
+    duration: 5_000,
+  },
+});
+
+WorkerManager.register(
+  QueueName.IngestionFlushQueue,
+  ingestionFlushQueueProcessor,
+  {
+    concurrency: env.LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY,
+  },
+);
+
+if (env.STRIPE_SECRET_KEY) {
+  WorkerManager.register(
+    QueueName.CloudUsageMeteringQueue,
+    cloudUsageMeteringQueueProcessor,
+    {
+      concurrency: 1,
+    },
+  );
+}
+
+if (env.QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED === "true") {
+  WorkerManager.register(
+    QueueName.LegacyIngestionQueue,
+    legacyIngestionQueueProcessor,
+    { concurrency: env.LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY }, // n ingestion batches at a time
+  );
+}

 process.on("SIGINT", () => onShutdown("SIGINT"));
 process.on("SIGTERM", () => onShutdown("SIGTERM"));
@@ -1 +1 @@
-export const VERSION = "v2.75.2";
+export const VERSION = "v2.77.0";
@@ -75,6 +75,9 @@ const EnvSchema = z.object({
    .positive()
    .default(5),
  STRIPE_SECRET_KEY: z.string().optional(),
+  QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("true"),
 });

 export const env = EnvSchema.parse(process.env);
@@ -1,75 +1,50 @@
-import { Job, Worker } from "bullmq";
+import { Job } from "bullmq";

 import { BaseError, BatchExportStatus } from "@langfuse/shared";
 import { kyselyPrisma } from "@langfuse/shared/src/db";

-import {
-  traceException,
-  instrumentAsync,
-  createNewRedisInstance,
-} from "@langfuse/shared/src/server";
+import { traceException, instrumentAsync } from "@langfuse/shared/src/server";
 import logger from "../logger";
 import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";
 import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";
 import { SpanKind } from "@opentelemetry/api";

-const createBatchExportJobExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker<TQueueJobTypes[QueueName.BatchExport]>(
-      QueueName.BatchExport,
-      async (job: Job<TQueueJobTypes[QueueName.BatchExport]>) => {
-        return instrumentAsync(
-          {
-            name: "batchExportJobExecutor",
-            spanKind: SpanKind.CONSUMER,
-          },
-          async () => {
-            try {
-              logger.info("Executing Batch Export Job", job.data.payload);
-              await handleBatchExportJob(job.data.payload);
+export const batchExportQueueProcessor = async (
+  job: Job<TQueueJobTypes[QueueName.BatchExport]>,
+) => {
+  return instrumentAsync(
+    {
+      name: "batchExportJobExecutor",
+      spanKind: SpanKind.CONSUMER,
+    },
+    async () => {
+      try {
+        logger.info("Executing Batch Export Job", job.data.payload);
+        await handleBatchExportJob(job.data.payload);

-              logger.info("Finished Batch Export Job", job.data.payload);
+        logger.info("Finished Batch Export Job", job.data.payload);

-              return true;
-            } catch (e) {
-              const displayError =
-                e instanceof BaseError
-                  ? e.message
-                  : "An internal error occurred";
+        return true;
+      } catch (e) {
+        const displayError =
+          e instanceof BaseError ? e.message : "An internal error occurred";

-              await kyselyPrisma.$kysely
-                .updateTable("batch_exports")
-                .set("status", BatchExportStatus.FAILED)
-                .set("finished_at", new Date())
-                .set("log", displayError)
-                .where("id", "=", job.data.payload.batchExportId)
-                .where("project_id", "=", job.data.payload.projectId)
-                .execute();
+        await kyselyPrisma.$kysely
+          .updateTable("batch_exports")
+          .set("status", BatchExportStatus.FAILED)
+          .set("finished_at", new Date())
+          .set("log", displayError)
+          .where("id", "=", job.data.payload.batchExportId)
+          .where("project_id", "=", job.data.payload.projectId)
+          .execute();

-              logger.error(
-                e,
-                `Failed Batch Export job for id ${job.data.payload.batchExportId} ${e}`
-              );
-              traceException(e);
-              throw e;
-            }
-          }
+        logger.error(
+          e,
+          `Failed Batch Export job for id ${job.data.payload.batchExportId} ${e}`,
        );
-      },
-      {
-        connection: redisInstance,
-        concurrency: 1, // only 1 job at a time
-        limiter: {
-          // execute 1 batch export in 5 seconds to avoid overloading the DB
-          max: 1,
-          duration: 5_000,
-        },
+        traceException(e);
+        throw e;
      }
-    );
-  }
-
-  return null;
+    },
+  );
 };
-
-export const batchExportJobExecutor = createBatchExportJobExecutor();
@@ -1,11 +1,10 @@
-import { Queue, Worker } from "bullmq";
+import { Processor, Queue } from "bullmq";
 import logger from "../logger";
 import {
  redis,
  QueueName,
  QueueJobs,
  instrumentAsync,
-  createNewRedisInstance,
 } from "@langfuse/shared/src/server";
 import { handleCloudUsageMeteringJob } from "../ee/cloudUsageMetering/handleCloudUsageMeteringJob";
 import { env } from "../env";
@@ -23,7 +22,7 @@ if (cloudUsageMeteringQueue) {
    {},
    {
      repeat: { pattern: "5 * * * *" },
-    }
+    },
  );

  // add a job to the queue to start the job immediately in case we need to catch up
@@ -35,35 +34,19 @@ if (cloudUsageMeteringQueue) {
  });
 }

-const createCloudUsageMeteringJobExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (env.STRIPE_SECRET_KEY && redisInstance) {
-    return new Worker(
-      QueueName.CloudUsageMeteringQueue,
-      async (job) => {
-        if (job.name === QueueJobs.CloudUsageMeteringJob) {
-          return instrumentAsync(
-            { name: "cloudUsageMeteringJobExecutor" },
-            async () => {
-              logger.info("Executing Cloud Usage Metering Job", job.data);
-              try {
-                return await handleCloudUsageMeteringJob(job);
-              } catch (error) {
-                logger.error("Error executing Cloud Usage Metering Job", error);
-                throw error;
-              }
-            }
-          );
+export const cloudUsageMeteringQueueProcessor: Processor = async (job) => {
+  if (job.name === QueueJobs.CloudUsageMeteringJob) {
+    return instrumentAsync(
+      { name: "cloudUsageMeteringJobExecutor" },
+      async () => {
+        logger.info("Executing Cloud Usage Metering Job", job.data);
+        try {
+          return await handleCloudUsageMeteringJob(job);
+        } catch (error) {
+          logger.error("Error executing Cloud Usage Metering Job", error);
+          throw error;
        }
      },
-      {
-        connection: redisInstance,
-        concurrency: 1,
-      }
    );
  }
-  return null;
 };
-
-export const cloudUsageMeteringJobExecutor =
-  createCloudUsageMeteringJobExecutor();
@@ -1,4 +1,4 @@
-import { Job, Queue, Worker } from "bullmq";
+import { Job, Queue } from "bullmq";
 import { ApiError, BaseError } from "@langfuse/shared";
 import { evaluate, createEvalJobs } from "../features/evaluation/eval-service";
 import { kyselyPrisma } from "@langfuse/shared/src/db";
@@ -16,7 +16,6 @@ import {
  getTraceUpsertQueue,
 } from "@langfuse/shared/src/server";
 import { SpanKind } from "@opentelemetry/api";
-import { env } from "../env";

 let evalQueue: Queue<TQueueJobTypes[QueueName.EvaluationExecution]> | null =
  null;
@@ -31,160 +30,131 @@ export const getEvalQueue = () => {
        QueueName.EvaluationExecution,
        {
          connection: connection,
-        }
+        },
      )
    : null;

  return evalQueue;
 };

-const createEvalJobCreator = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker<TQueueJobTypes[QueueName.TraceUpsert]>(
-      QueueName.TraceUpsert,
-      async (job: Job<TQueueJobTypes[QueueName.TraceUpsert]>) => {
-        return instrumentAsync(
-          {
-            name: "evalJobCreator",
-            rootSpan: true,
-            spanKind: SpanKind.CONSUMER,
-          },
-          async () => {
-            try {
-              const startTime = Date.now();
+export const evalJobCreatorQueueProcessor = async (
+  job: Job<TQueueJobTypes[QueueName.TraceUpsert]>,
+) => {
+  return instrumentAsync(
+    {
+      name: "evalJobCreator",
+      rootSpan: true,
+      spanKind: SpanKind.CONSUMER,
+    },
+    async () => {
+      try {
+        const startTime = Date.now();

-              const waitTime = Date.now() - job.timestamp;
+        const waitTime = Date.now() - job.timestamp;

-              recordIncrement("trace_upsert_queue_request");
-              recordHistogram("trace_upsert_queue_wait_time", waitTime, {
-                unit: "milliseconds",
-              });
+        recordIncrement("trace_upsert_queue_request");
+        recordHistogram("trace_upsert_queue_wait_time", waitTime, {
+          unit: "milliseconds",
+        });

-              await createEvalJobs({ event: job.data.payload });
+        await createEvalJobs({ event: job.data.payload });

-              await getTraceUpsertQueue()
-                ?.count()
-                .then((count) => {
-                  logger.info(`Eval creation queue length: ${count}`);
-                  recordGauge("trace_upsert_queue_length", count, {
-                    unit: "records",
-                  });
-                  return count;
-                })
-                .catch();
-              recordHistogram(
-                "trace_upsert_queue_processing_time",
-                Date.now() - startTime,
-                { unit: "milliseconds" }
-              );
-              return true;
-            } catch (e) {
-              logger.error(
-                e,
-                `Failed job Evaluation for traceId ${job.data.payload.traceId} ${e}`
-              );
-              traceException(e);
-              throw e;
-            }
-          }
+        await getTraceUpsertQueue()
+          ?.count()
+          .then((count) => {
+            logger.info(`Eval creation queue length: ${count}`);
+            recordGauge("trace_upsert_queue_length", count, {
+              unit: "records",
+            });
+            return count;
+          })
+          .catch();
+        recordHistogram(
+          "trace_upsert_queue_processing_time",
+          Date.now() - startTime,
+          { unit: "milliseconds" },
        );
-      },
-      {
-        connection: redisInstance,
-        concurrency: env.LANGFUSE_EVAL_CREATOR_WORKER_CONCURRENCY,
+        return true;
+      } catch (e) {
+        logger.error(
+          e,
+          `Failed job Evaluation for traceId ${job.data.payload.traceId} ${e}`,
+        );
+        traceException(e);
+        throw e;
      }
-    );
-  }
-  return null;
+    },
+  );
 };

-export const evalJobCreator = createEvalJobCreator();
+export const evalJobExecutorQueueProcessor = async (
+  job: Job<TQueueJobTypes[QueueName.EvaluationExecution]>,
+) => {
+  return instrumentAsync(
+    {
+      name: "evalJobExecutor",
+      spanKind: SpanKind.CONSUMER,
+    },
+    async () => {
+      try {
+        logger.info("Executing Evaluation Execution Job", job.data);
+        const startTime = Date.now();

-const createEvalJobExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker<TQueueJobTypes[QueueName.EvaluationExecution]>(
-      QueueName.EvaluationExecution,
-      async (job: Job<TQueueJobTypes[QueueName.EvaluationExecution]>) => {
-        return instrumentAsync(
-          {
-            name: "evalJobExecutor",
-            spanKind: SpanKind.CONSUMER,
-          },
-          async () => {
-            try {
-              logger.info("Executing Evaluation Execution Job", job.data);
-              const startTime = Date.now();
+        const waitTime = Date.now() - job.timestamp;

-              const waitTime = Date.now() - job.timestamp;
+        recordIncrement("eval_execution_queue_request");
+        recordHistogram("eval_execution_queue_wait_time", waitTime, {
+          unit: "milliseconds",
+        });

-              recordIncrement("eval_execution_queue_request");
-              recordHistogram("eval_execution_queue_wait_time", waitTime, {
-                unit: "milliseconds",
-              });
+        await evaluate({ event: job.data.payload });

-              await evaluate({ event: job.data.payload });
-
-              await getEvalQueue()
-                ?.count()
-                .then((count) => {
-                  logger.info(`Eval execution queue length: ${count}`);
-                  recordGauge("eval_execution_queue_length", count, {
-                    unit: "records",
-                  });
-                  return count;
-                })
-                .catch();
-              recordHistogram(
-                "eval_execution_queue_processing_time",
-                Date.now() - startTime,
-                { unit: "milliseconds" }
-              );
-
-              return true;
-            } catch (e) {
-              const displayError =
-                e instanceof BaseError
-                  ? e.message
-                  : "An internal error occurred";
-
-              await kyselyPrisma.$kysely
-                .updateTable("job_executions")
-                .set("status", sql`'ERROR'::"JobExecutionStatus"`)
-                .set("end_time", new Date())
-                .set("error", displayError)
-                .where("id", "=", job.data.payload.jobExecutionId)
-                .where("project_id", "=", job.data.payload.projectId)
-                .execute();
-
-              // do not log expected errors (api failures + missing api keys not provided by the user)
-              if (
-                !(e instanceof ApiError) &&
-                !(
-                  e instanceof BaseError &&
-                  e.message.includes("API key for provider")
-                )
-              ) {
-                traceException(e);
-                logger.error(
-                  e,
-                  `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId} ${e}`
-                );
-              }
-
-              throw e;
-            }
-          }
+        await getEvalQueue()
+          ?.count()
+          .then((count) => {
+            logger.info(`Eval execution queue length: ${count}`);
+            recordGauge("eval_execution_queue_length", count, {
+              unit: "records",
+            });
+            return count;
+          })
+          .catch();
+        recordHistogram(
+          "eval_execution_queue_processing_time",
+          Date.now() - startTime,
+          { unit: "milliseconds" },
        );
-      },
-      {
-        connection: redisInstance,
-        concurrency: env.LANGFUSE_EVAL_EXECUTION_WORKER_CONCURRENCY,
-      }
-    );
-  }
-  return null;
-};

-export const evalJobExecutor = createEvalJobExecutor();
+        return true;
+      } catch (e) {
+        const displayError =
+          e instanceof BaseError ? e.message : "An internal error occurred";
+
+        await kyselyPrisma.$kysely
+          .updateTable("job_executions")
+          .set("status", sql`'ERROR'::"JobExecutionStatus"`)
+          .set("end_time", new Date())
+          .set("error", displayError)
+          .where("id", "=", job.data.payload.jobExecutionId)
+          .where("project_id", "=", job.data.payload.projectId)
+          .execute();
+
+        // do not log expected errors (api failures + missing api keys not provided by the user)
+        if (
+          !(e instanceof ApiError) &&
+          !(
+            e instanceof BaseError && e.message.includes("API key for provider")
+          )
+        ) {
+          traceException(e);
+          logger.error(
+            e,
+            `Failed Evaluation_Execution job for id ${job.data.payload.jobExecutionId} ${e}`,
+          );
+        }
+
+        throw e;
+      }
+    },
+  );
+};
@@ -1,10 +1,6 @@
-import { Queue, Worker } from "bullmq";
+import { Processor } from "bullmq";

-import {
-  createNewRedisInstance,
-  QueueJobs,
-  QueueName,
-} from "@langfuse/shared/src/server";
+import { redis, QueueJobs } from "@langfuse/shared/src/server";
 import { prisma } from "@langfuse/shared/src/db";
 import {
  clickhouseClient,
@@ -14,7 +10,6 @@ import {
  recordGauge,
  recordHistogram,
 } from "@langfuse/shared/src/server";
-import { env } from "../env";
 import logger from "../logger";
 import { ClickhouseWriter } from "../services/ClickhouseWriter";
 import { IngestionService } from "../services/IngestionService";
@@ -22,93 +17,72 @@ import { SpanKind } from "@opentelemetry/api";

 const ingestionFlushQueue = getIngestionFlushQueue();

-const createIngestionQueueExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker(
-      QueueName.IngestionFlushQueue,
-      async (job) => {
-        return instrumentAsync(
-          {
-            name: "flush-ingestion-consumer",
-            spanKind: SpanKind.CONSUMER,
-          },
-          async () => {
-            if (job.name === QueueJobs.FlushIngestionEntity) {
-              const flushKey = job.id;
-              if (!flushKey) {
-                throw new Error("Flushkey not provided");
-              }
+export const ingestionFlushQueueProcessor: Processor = async (job) => {
+  return instrumentAsync(
+    {
+      name: "flush-ingestion-consumer",
+      spanKind: SpanKind.CONSUMER,
+    },
+    async () => {
+      if (job.name === QueueJobs.FlushIngestionEntity) {
+        const flushKey = job.id;
+        if (!flushKey) {
+          throw new Error("Flushkey not provided");
+        }

-              // Log wait time
-              const waitTime = Date.now() - job.timestamp;
-              logger.debug(
-                `Received flush request after ${waitTime} ms for ${flushKey}`
-              );
-
-              recordIncrement("ingestion_processing_request");
-              recordHistogram("ingestion_flush_wait_time", waitTime, {
-                unit: "milliseconds",
-              });
-
-              try {
-                // Check dependencies
-                if (!redisInstance) throw new Error("Redis not available");
-                if (!prisma) throw new Error("Prisma not available");
-                if (!ingestionFlushQueue)
-                  throw new Error("Ingestion flush queue not available");
-
-                // Flush ingestion buffer
-                const processingStartTime = Date.now();
-
-                await new IngestionService(
-                  redisInstance,
-                  prisma,
-                  ClickhouseWriter.getInstance(),
-                  clickhouseClient
-                ).flush(flushKey);
-
-                // Log processing time
-                const processingTime = Date.now() - processingStartTime;
-                logger.debug(
-                  `Prepared and scheduled CH-write in ${processingTime} ms for ${flushKey}`
-                );
-                recordHistogram(
-                  "ingestion_flush_processing_time",
-                  processingTime,
-                  { unit: "milliseconds" }
-                );
-
-                // Log queue size
-                await ingestionFlushQueue
-                  .count()
-                  .then((count) => {
-                    logger.debug(`Ingestion flush queue length: ${count}`);
-                    recordGauge("ingestion_flush_queue_length", count, {
-                      unit: "records",
-                    });
-                    return count;
-                  })
-                  .catch();
-              } catch (err) {
-                console.error(
-                  `Error processing flush request for ${flushKey}`,
-                  err
-                );
-
-                throw err;
-              }
-            }
-          }
+        // Log wait time
+        const waitTime = Date.now() - job.timestamp;
+        logger.debug(
+          `Received flush request after ${waitTime} ms for ${flushKey}`,
        );
-      },
-      {
-        connection: redisInstance,
-        concurrency: env.LANGFUSE_INGESTION_FLUSH_PROCESSING_CONCURRENCY,
-      }
-    );
-  }
-  return null;
-};

-export const ingestionQueueExecutor = createIngestionQueueExecutor();
+        recordIncrement("ingestion_processing_request");
+        recordHistogram("ingestion_flush_wait_time", waitTime, {
+          unit: "milliseconds",
+        });
+
+        try {
+          // Check dependencies
+          if (!redis) throw new Error("Redis not available");
+          if (!prisma) throw new Error("Prisma not available");
+          if (!ingestionFlushQueue)
+            throw new Error("Ingestion flush queue not available");
+
+          // Flush ingestion buffer
+          const processingStartTime = Date.now();
+
+          await new IngestionService(
+            redis,
+            prisma,
+            ClickhouseWriter.getInstance(),
+            clickhouseClient,
+          ).flush(flushKey);
+
+          // Log processing time
+          const processingTime = Date.now() - processingStartTime;
+          logger.debug(
+            `Prepared and scheduled CH-write in ${processingTime} ms for ${flushKey}`,
+          );
+          recordHistogram("ingestion_flush_processing_time", processingTime, {
+            unit: "milliseconds",
+          });
+
+          // Log queue size
+          await ingestionFlushQueue
+            .count()
+            .then((count) => {
+              logger.debug(`Ingestion flush queue length: ${count}`);
+              recordGauge("ingestion_flush_queue_length", count, {
+                unit: "records",
+              });
+              return count;
+            })
+            .catch();
+        } catch (err) {
+          logger.error(`Error processing flush request for ${flushKey}`, err);
+          throw err;
+        }
+      }
+    },
+  );
+};
@@ -1,14 +1,13 @@
-import { Job, Worker } from "bullmq";
+import { Job, Processor } from "bullmq";
 import {
  traceException,
-  getLegacyIngestionQueue,
+  LegacyIngestionQueue,
  instrumentAsync,
  QueueName,
  recordIncrement,
  recordGauge,
  recordHistogram,
  TQueueJobTypes,
-  createNewRedisInstance,
 } from "@langfuse/shared/src/server";
 import logger from "../logger";

@@ -17,85 +16,71 @@ import {
  sendToWorkerIfEnvironmentConfigured,
 } from "@langfuse/shared/src/server";
 import { tokenCount } from "../features/tokenisation/usage";
-import { env } from "../env";
 import { SpanKind } from "@opentelemetry/api";

-const createLegacyIngestionExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
-      QueueName.LegacyIngestionQueue,
-      async (job: Job<TQueueJobTypes[QueueName.LegacyIngestionQueue]>) => {
-        return instrumentAsync(
-          {
-            name: "legacyIngestion",
-            spanKind: SpanKind.CONSUMER,
-            rootSpan: true,
-          },
-          async () => {
-            try {
-              const startTime = Date.now();
-              logger.info(
-                `Processing legacy ingestion for payload ${JSON.stringify(job.data.payload)}`
-              );
-
-              // Log wait time
-              const waitTime = Date.now() - job.timestamp;
-              logger.debug(
-                `Received flush request after ${waitTime} ms for ${job.data.payload.authCheck.scope.projectId}`
-              );
-
-              recordIncrement("legacy_ingestion_processing_request");
-              recordHistogram("legacy_ingestion_flush_wait_time", waitTime, {
-                unit: "milliseconds",
-              });
-
-              const result = await handleBatch(
-                job.data.payload.data,
-                job.data.payload.authCheck,
-                tokenCount
-              );
-
-              // send out REDIS requests to worker for all trace types
-              await sendToWorkerIfEnvironmentConfigured(
-                result.results,
-                job.data.payload.authCheck.scope.projectId
-              );
-
-              // Log queue size
-              await getLegacyIngestionQueue()
-                ?.count()
-                .then((count) => {
-                  logger.info(`Legacy Ingestion flush queue length: ${count}`);
-                  recordGauge("legacy_ingestion_flush_queue_length", count, {
-                    unit: "records",
-                  });
-                  return count;
-                })
-                .catch();
-              recordHistogram(
-                "legacy_ingestion_processing_time",
-                Date.now() - startTime,
-                { unit: "milliseconds" }
-              );
-            } catch (e) {
-              logger.error(
-                e,
-                `Failed job Evaluation for traceId ${job.data.payload} ${e}`
-              );
-              traceException(e);
-              throw e;
-            }
-          }
+export const legacyIngestionQueueProcessor: Processor = async (
+  job: Job<TQueueJobTypes[QueueName.LegacyIngestionQueue]>,
+) => {
+  return instrumentAsync(
+    {
+      name: "legacyIngestion",
+      spanKind: SpanKind.CONSUMER,
+      rootSpan: true,
+    },
+    async () => {
+      try {
+        const startTime = Date.now();
+        logger.info(
+          `Processing legacy ingestion for payload ${JSON.stringify(job.data.payload)}`,
        );
-      },
-      {
-        connection: redisInstance,
-        concurrency: env.LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY, // n ingestion batches at a time
-      }
-    );
-  }
-  return null;
-};

-export const legacyIngestionExecutor = createLegacyIngestionExecutor();
+        // Log wait time
+        const waitTime = Date.now() - job.timestamp;
+        logger.debug(
+          `Received flush request after ${waitTime} ms for ${job.data.payload.authCheck.scope.projectId}`,
+        );
+
+        recordIncrement("legacy_ingestion_processing_request");
+        recordHistogram("legacy_ingestion_flush_wait_time", waitTime, {
+          unit: "milliseconds",
+        });
+
+        const result = await handleBatch(
+          job.data.payload.data,
+          job.data.payload.authCheck,
+          tokenCount,
+        );
+
+        // send out REDIS requests to worker for all trace types
+        await sendToWorkerIfEnvironmentConfigured(
+          result.results,
+          job.data.payload.authCheck.scope.projectId,
+        );
+
+        // Log queue size
+        await LegacyIngestionQueue.getInstance()
+          ?.count()
+          .then((count) => {
+            logger.info(`Legacy Ingestion flush queue length: ${count}`);
+            recordGauge("legacy_ingestion_flush_queue_length", count, {
+              unit: "records",
+            });
+            return count;
+          })
+          .catch();
+        recordHistogram(
+          "legacy_ingestion_processing_time",
+          Date.now() - startTime,
+          { unit: "milliseconds" },
+        );
+      } catch (e) {
+        logger.error(
+          e,
+          `Failed job Evaluation for traceId ${job.data.payload} ${e}`,
+        );
+        traceException(e);
+        throw e;
+      }
+    },
+  );
+};
@@ -1,11 +1,6 @@
-import { Queue, Worker } from "bullmq";
+import { Processor, Queue } from "bullmq";

-import {
-  redis,
-  QueueJobs,
-  QueueName,
-  createNewRedisInstance,
-} from "@langfuse/shared/src/server";
+import { redis, QueueJobs, QueueName } from "@langfuse/shared/src/server";
 import { enqueueBatchExportJobs } from "../features/batchExport/enqueueBatchExportJobs";

 export const repeatQueue = redis
@@ -21,26 +16,12 @@ if (repeatQueue) {
    {},
    {
      repeat: { pattern: "*/10 * * * *" },
-    }
+    },
  );
 }

-const createRepeatQueueExecutor = () => {
-  const redisInstance = createNewRedisInstance();
-  if (redisInstance) {
-    return new Worker(
-      QueueName.RepeatQueue,
-      async (job) => {
-        if (job.name === QueueJobs.EnqueueBatchExportJobs) {
-          return enqueueBatchExportJobs();
-        }
-      },
-      {
-        connection: redisInstance,
-      }
-    );
+export const repeatQueueProcessor: Processor = async (job) => {
+  if (job.name === QueueJobs.EnqueueBatchExportJobs) {
+    return enqueueBatchExportJobs();
  }
-  return null;
 };
-
-export const repeatQueueExecutor = createRepeatQueueExecutor();
@@ -0,0 +1,63 @@
+import logger from "../logger";
+
+import { Job, Processor, Worker, WorkerOptions } from "bullmq";
+import { createNewRedisInstance } from "@langfuse/shared/src/server";
+
+export class WorkerManager {
+  private static workers: { [key: string]: Worker } = {};
+
+  public static async closeWorkers(): Promise<void> {
+    await Promise.all(
+      Object.values(WorkerManager.workers).map((worker) => worker.close()),
+    );
+    logger.info("All workers have been closed.");
+  }
+
+  public static register(
+    queueName: string,
+    processor: Processor,
+    additionalOptions: Partial<WorkerOptions> = {},
+  ): void {
+    if (WorkerManager.workers[queueName]) {
+      logger.info(`Worker ${queueName} is already registered`);
+      return;
+    }
+
+    // Create redis connection for queue worker
+    const redisInstance = createNewRedisInstance({
+      retryStrategy: (times: number) => {
+        // https://docs.bullmq.io/guide/going-to-production#retrystrategy
+        // Retries forever. Waits at least 1s and at most 20s between retries.
+        logger.debug(`Connection to redis lost. Retry attempt: ${times}`);
+        return Math.max(Math.min(Math.exp(times), 20000), 1000);
+      },
+      reconnectOnError: (err: Error) => {
+        logger.warn(`Failed to connect to redis: ${err}. Reconnecting...`);
+        return true;
+      },
+    });
+    if (!redisInstance) {
+      logger.error("Failed to initialize redis connection");
+      return;
+    }
+
+    // Register worker
+    const worker = new Worker(queueName, processor, {
+      connection: redisInstance,
+      ...additionalOptions,
+    });
+    WorkerManager.workers[queueName] = worker;
+    logger.info(`${queueName} executor started: ${worker.isRunning()}`);
+
+    // Add error handling
+    worker.on("failed", (job: Job | undefined, err: Error) => {
+      logger.error(
+        err,
+        `Queue Job ${job?.name} with id ${job?.id} in ${queueName} failed with error ${err}`,
+      );
+    });
+    worker.on("error", (failedReason: Error) => {
+      logger.error(`Queue worker ${queueName} failed: ${failedReason}`);
+    });
+  }
+}
@@ -1,11 +0,0 @@
-import { Job } from "bullmq";
-
-import logger from "../logger";
-
-export function logQueueWorkerError(job: Job | undefined, err: Error) {
-  logger.error(
-    err,
-    `Queue Job ${job?.name} with id ${job?.id} failed with error
-    ${err}`
-  );
-}
@@ -2,16 +2,11 @@ import logger from "../logger";

 import { redis } from "@langfuse/shared/src/server";

-import { evalJobCreator, evalJobExecutor } from "../queues/evalQueue";
-import { batchExportJobExecutor } from "../queues/batchExportQueue";
-import { ingestionQueueExecutor } from "../queues/ingestionFlushQueueExecutor";
-import { repeatQueueExecutor } from "../queues/repeatQueue";
 import { ClickhouseWriter } from "../services/ClickhouseWriter";
 import { setSigtermReceived } from "../features/health";
 import { server } from "../index";
-import { legacyIngestionExecutor } from "../queues/legacyIngestionQueue";
-import { cloudUsageMeteringJobExecutor } from "../queues/cloudUsageMeteringQueue";
 import { freeAllTokenizers } from "../features/tokenisation/usage";
+import { WorkerManager } from "../queues/workerManager";

 export const onShutdown: NodeJS.SignalsListener = async (signal) => {
  logger.info(`Received ${signal}, closing server...`);
@@ -22,18 +17,7 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => {
  logger.info("Server has been closed.");

  // Shutdown workers (https://docs.bullmq.io/guide/going-to-production#gracefully-shut-down-workers)
-  const workers = [
-    evalJobCreator,
-    evalJobExecutor,
-    batchExportJobExecutor,
-    ingestionQueueExecutor,
-    repeatQueueExecutor,
-    legacyIngestionExecutor,
-    cloudUsageMeteringJobExecutor,
-  ];
-
-  await Promise.all(workers.map(async (worker) => await worker?.close()));
-  logger.info("All workers have been closed.");
+  await WorkerManager.closeWorkers();

  // Flush all pending writes to Clickhouse AFTER closing ingestion queue worker that is writing to it
  await ClickhouseWriter.getInstance().shutdown();
Author	SHA1	Message	Date
steffen911	11c04f02c8	chore: release v2.77.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests-web (node20, pg12) (push) Waiting to run Details CI/CD / tests-web (node20, pg15) (push) Waiting to run Details CI/CD / tests-worker (node20, pg12) (push) Waiting to run Details CI/CD / tests-worker (node20, pg15) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / e2e-server-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-09-04 10:50:09 +02:00
Steffen SchmitzandGitHub	536aa138bc	fix: reconnect workers to redis on errors (#3211 )	2024-09-04 10:22:59 +02:00
Marc KlingenandGitHub	61b596c46a	fix(docker): prevent web build oom by allocating more memory to build step in Dockerfile (#3217 )	2024-09-04 00:18:15 +02:00
Marc KlingenandGitHub	ddf43c0836	feat(self-host): provision instance with default org/project/user/keys via `LANGFUSE_PROVISION_*` envs (#3216 )	2024-09-03 23:50:20 +02:00
Marc KlingenandGitHub	d5ab03892a	chore(ee): add EE to VersionLabel (#3214 )	2024-09-03 14:03:57 +00:00
marliessophie GitHub Marc Klingen Max Deichmann	918b6643d2	chore(release): nextjs client side refresh after redeployment (#3116 ) Co-authored-by: Marc Klingen <git@marcklingen.com> Co-authored-by: Max Deichmann <m.deichmann@tum.de>	2024-09-03 12:42:57 +02:00
marliessophieandGitHub	937b4c0f0f	feat(ui): deletion of dataset runs (#3209 )	2024-09-03 12:26:02 +02:00
Max Deichmann	9b7f046e71	chore: release v2.76.0 CI/CD / lint (push) Waiting to run Details CI/CD / test-docker-build (push) Waiting to run Details CI/CD / tests-web (node20, pg12) (push) Waiting to run Details CI/CD / tests-web (node20, pg15) (push) Waiting to run Details CI/CD / tests-worker (node20, pg12) (push) Waiting to run Details CI/CD / tests-worker (node20, pg15) (push) Waiting to run Details CI/CD / e2e-tests (push) Waiting to run Details CI/CD / e2e-server-tests (push) Waiting to run Details CI/CD / all-ci-passed (push) Blocked by required conditions Details CI/CD / push-docker-image (push) Blocked by required conditions Details release.yml / release (push) Waiting to run Details	2024-09-03 10:48:10 +02:00
Steffen SchmitzandGitHub	fe63704508	feat: fallback to synchronous ingestion if queue unavailable (#3196 )	2024-09-03 10:42:59 +02:00
marliessophieandGitHub	c39279f677	style(prompts): actions on hover (#3135 )	2024-09-03 08:12:39 +00:00
Marc KlingenandGitHub	4206f5025d	perf(cloud): pull usage data shown in settings from stripe if available (#3206 )	2024-09-02 23:51:13 +00:00
marliessophieandGitHub	0260a4e861	style(dataset_items): make table vs item detail resizeable (#3139 )	2024-09-02 08:01:43 +00:00
Max DeichmannandGitHub	244adf0dc3	feat: disable legacy ingestion queue if required (#3183 )	2024-09-01 14:24:25 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	dc6b98d009	chore(deps-dev): bump tsx from 4.18.0 to 4.19.0 (#3172 ) Bumps [tsx](https://github.com/privatenumber/tsx) from 4.18.0 to 4.19.0. - [Release notes](https://github.com/privatenumber/tsx/releases) - [Changelog](https://github.com/privatenumber/tsx/blob/master/release.config.cjs) - [Commits](https://github.com/privatenumber/tsx/compare/v4.18.0...v4.19.0) --- updated-dependencies: - dependency-name: tsx dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-09-01 10:07:44 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	b20fc5a20b	chore(deps-dev): bump tailwindcss from 3.4.4 to 3.4.10 (#3173 ) Bumps [tailwindcss](https://github.com/tailwindlabs/tailwindcss) from 3.4.4 to 3.4.10. - [Release notes](https://github.com/tailwindlabs/tailwindcss/releases) - [Changelog](https://github.com/tailwindlabs/tailwindcss/blob/v3.4.10/CHANGELOG.md) - [Commits](https://github.com/tailwindlabs/tailwindcss/compare/v3.4.4...v3.4.10) --- updated-dependencies: - dependency-name: tailwindcss dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-08-31 22:36:42 +02:00
Max DeichmannandGitHub	f59ad5fd60	chore: upload sentry sourcemaps (#3179 )	2024-08-31 18:50:48 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	06ef41e372	chore(deps): bump pino-pretty from 10.3.1 to 11.2.2 (#3170 ) Bumps [pino-pretty](https://github.com/pinojs/pino-pretty) from 10.3.1 to 11.2.2. - [Release notes](https://github.com/pinojs/pino-pretty/releases) - [Commits](https://github.com/pinojs/pino-pretty/compare/v10.3.1...v11.2.2) --- updated-dependencies: - dependency-name: pino-pretty dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-08-31 17:04:49 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	e215a2db99	chore(deps-dev): bump @types/lodash from 4.17.5 to 4.17.7 in the lodash group (#3169 ) chore(deps-dev): bump @types/lodash in the lodash group Bumps the lodash group with 1 update: [@types/lodash](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/lodash). Updates `@types/lodash` from 4.17.5 to 4.17.7 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/lodash) --- updated-dependencies: - dependency-name: "@types/lodash" dependency-type: direct:development update-type: version-update:semver-patch dependency-group: lodash ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-08-31 17:00:36 +02:00
Max DeichmannandGitHub	5e247e88c9	fix: do nothing if invalidating org or project without api keys (#3167 ) * also run on main branch * also run on main branch * also run on main branch * also run on main branch * also run on main branch	2024-08-30 22:57:41 +02:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	8e1e27c868	chore(deps-dev): bump msw from 2.3.1 to 2.4.1 (#3150 ) Bumps [msw](https://github.com/mswjs/msw) from 2.3.1 to 2.4.1. - [Release notes](https://github.com/mswjs/msw/releases) - [Changelog](https://github.com/mswjs/msw/blob/main/CHANGELOG.md) - [Commits](https://github.com/mswjs/msw/compare/v2.3.1...v2.4.1) --- updated-dependencies: - dependency-name: msw dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-08-30 21:52:41 +02:00
dependabot[bot]andGitHub	4648a5be8b	chore(deps): bump lucide-react from 0.429.0 to 0.436.0 (#3154 )	2024-08-30 21:02:28 +02:00